1.1.0/nn.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta name="robots" content="noindex">
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.nn &mdash; PyTorch master documentation</title>
  

    <link rel="canonical" href="https://pytorch.org/docs/stable/nn.html"/>
  

  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="torch.optim" href="optim.html" />
    <link rel="prev" title="torch.Storage" href="storage.html" /> 

  
  <script src="_static/js/modernizr.min.js"></script>
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   
    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

                <div class="version">
                  <a href='http://pytorch.org/docs/versions.html'>1.1.0 &#x25BC</a>
                </div>
              
            
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>
<p class="caption"><span class="caption-text">Package Reference</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="storage.html">torch.Storage</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="#torch-nn-functional">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="#torch-nn-init">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="multiprocessing.html">torch.multiprocessing</a></li>
<li class="toctree-l1"><a class="reference internal" href="bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensorboard.html">torch.utils.tensorboard (experimental)</a></li>
<li class="toctree-l1"><a class="reference internal" href="onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="__config__.html">torch.__config__</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed_deprecated.html">torch.distributed.deprecated</a></li>
</ul>
<p class="caption"><span class="caption-text">torchvision Reference</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="torchvision/index.html">torchvision</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          

<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
      <li>torch.nn</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="_sources/nn.rst.txt" rel="nofollow"><img src="_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <div class="section" id="module-torch.nn">
<span id="torch-nn"></span><h1>torch.nn<a class="headerlink" href="#module-torch.nn" title="Permalink to this headline">¶</a></h1>
<div class="section" id="parameters">
<h2>Parameters<a class="headerlink" href="#parameters" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="torch.nn.Parameter">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Parameter</code><a class="reference internal" href="_modules/torch/nn/parameter.html#Parameter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Parameter" title="Permalink to this definition">¶</a></dt>
<dd><p>A kind of Tensor that is to be considered a module parameter.</p>
<p>Parameters are <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code></a> subclasses, that have a
very special property when used with <a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> s - when they’re
assigned as Module attributes they are automatically added to the list of
its parameters, and will appear e.g. in <a class="reference internal" href="#torch.nn.Module.parameters" title="torch.nn.Module.parameters"><code class="xref py py-meth docutils literal notranslate"><span class="pre">parameters()</span></code></a> iterator.
Assigning a Tensor doesn’t have such effect. This is because one might
want to cache some temporary state, like last hidden state of the RNN, in
the model. If there was no such class as <a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><code class="xref py py-class docutils literal notranslate"><span class="pre">Parameter</span></code></a>, these
temporaries would get registered too.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>data</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – parameter tensor.</p></li>
<li><p><strong>requires_grad</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if the parameter requires gradient. See
<a class="reference internal" href="notes/autograd.html#excluding-subgraphs"><span class="std std-ref">Excluding subgraphs from backward</span></a> for more details. Default: <cite>True</cite></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="containers">
<h2>Containers<a class="headerlink" href="#containers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="module">
<h3><span class="hidden-section">Module</span><a class="headerlink" href="#module" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Module">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Module</code><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module" title="Permalink to this definition">¶</a></dt>
<dd><p>Base class for all neural network modules.</p>
<p>Your models should also subclass this class.</p>
<p>Modules can also contain other Modules, allowing to nest them in
a tree structure. You can assign the submodules as regular attributes:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
<span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>

<span class="k">class</span> <span class="nc">Model</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Model</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
       <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv1</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
       <span class="k">return</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv2</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
</pre></div>
</div>
<p>Submodules assigned in this way will be registered, and will have their
parameters converted too when you call <a class="reference internal" href="#torch.nn.Module.to" title="torch.nn.Module.to"><code class="xref py py-meth docutils literal notranslate"><span class="pre">to()</span></code></a>, etc.</p>
<dl class="method">
<dt id="torch.nn.Module.add_module">
<code class="descname">add_module</code><span class="sig-paren">(</span><em>name</em>, <em>module</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.add_module"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.add_module" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a child module to the current module.</p>
<p>The module can be accessed as an attribute using the given name.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>name</strong> (<em>string</em>) – name of the child module. The child module can be
accessed from this module using the given name</p></li>
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – child module to be added to the module.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.apply">
<code class="descname">apply</code><span class="sig-paren">(</span><em>fn</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.apply"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.apply" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies <code class="docutils literal notranslate"><span class="pre">fn</span></code> recursively to every submodule (as returned by <code class="docutils literal notranslate"><span class="pre">.children()</span></code>)
as well as self. Typical use includes initializing the parameters of a model
(see also <span class="xref std std-ref">torch-nn-init</span>).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>fn</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> -&gt; None) – function to be applied to each submodule</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>self</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">init_weights</span><span class="p">(</span><span class="n">m</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="nb">print</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">m</span><span class="p">)</span> <span class="o">==</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">:</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">m</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">fill_</span><span class="p">(</span><span class="mf">1.0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="nb">print</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">weight</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">net</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">net</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">init_weights</span><span class="p">)</span>
<span class="go">Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">Parameter containing:</span>
<span class="go">tensor([[ 1.,  1.],</span>
<span class="go">        [ 1.,  1.]])</span>
<span class="go">Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">Parameter containing:</span>
<span class="go">tensor([[ 1.,  1.],</span>
<span class="go">        [ 1.,  1.]])</span>
<span class="go">Sequential(</span>
<span class="go">  (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">  (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">)</span>
<span class="go">Sequential(</span>
<span class="go">  (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">  (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.buffers">
<code class="descname">buffers</code><span class="sig-paren">(</span><em>recurse=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.buffers"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.buffers" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over module buffers.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>recurse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – if True, then yields buffers of this module
and all submodules. Otherwise, yields only buffers that
are direct members of this module.</p>
</dd>
<dt class="field-even">Yields</dt>
<dd class="field-even"><p><em>torch.Tensor</em> – module buffer</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">buf</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">buffers</span><span class="p">():</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="nb">print</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">buf</span><span class="o">.</span><span class="n">data</span><span class="p">),</span> <span class="n">buf</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">&lt;class &#39;torch.FloatTensor&#39;&gt; (20L,)</span>
<span class="go">&lt;class &#39;torch.FloatTensor&#39;&gt; (20L, 1L, 5L, 5L)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.children">
<code class="descname">children</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.children"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.children" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over immediate children modules.</p>
<dl class="field-list simple">
<dt class="field-odd">Yields</dt>
<dd class="field-odd"><p><em>Module</em> – a child module</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.cpu">
<code class="descname">cpu</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.cpu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.cpu" title="Permalink to this definition">¶</a></dt>
<dd><p>Moves all model parameters and buffers to the CPU.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>self</p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.cuda">
<code class="descname">cuda</code><span class="sig-paren">(</span><em>device=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.cuda"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.cuda" title="Permalink to this definition">¶</a></dt>
<dd><p>Moves all model parameters and buffers to the GPU.</p>
<p>This also makes associated parameters and buffers different objects. So
it should be called before constructing optimizer if the module will
live on GPU while being optimized.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>device</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – if specified, all parameters will be
copied to that device</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>self</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.double">
<code class="descname">double</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.double"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.double" title="Permalink to this definition">¶</a></dt>
<dd><p>Casts all floating point parameters and buffers to <code class="docutils literal notranslate"><span class="pre">double</span></code> datatype.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>self</p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
</dd></dl>

<dl class="attribute">
<dt id="torch.nn.Module.dump_patches">
<code class="descname">dump_patches</code><em class="property"> = False</em><a class="headerlink" href="#torch.nn.Module.dump_patches" title="Permalink to this definition">¶</a></dt>
<dd><p>This allows better BC support for <a class="reference internal" href="#torch.nn.Module.load_state_dict" title="torch.nn.Module.load_state_dict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">load_state_dict()</span></code></a>. In
<a class="reference internal" href="#torch.nn.Module.state_dict" title="torch.nn.Module.state_dict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">state_dict()</span></code></a>, the version number will be saved as in the attribute
<cite>_metadata</cite> of the returned state dict, and thus pickled. <cite>_metadata</cite> is a
dictionary with keys that follow the naming convention of state dict. See
<code class="docutils literal notranslate"><span class="pre">_load_from_state_dict</span></code> on how to use this information in loading.</p>
<p>If new parameters/buffers are added/removed from a module, this number shall
be bumped, and the module’s <cite>_load_from_state_dict</cite> method can compare the
version number and do appropriate changes if the state dict is from before
the change.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.eval">
<code class="descname">eval</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.eval"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.eval" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets the module in evaluation mode.</p>
<p>This has any effect only on certain modules. See documentations of
particular modules for details of their behaviors in training/evaluation
mode, if they are affected, e.g. <a class="reference internal" href="#torch.nn.Dropout" title="torch.nn.Dropout"><code class="xref py py-class docutils literal notranslate"><span class="pre">Dropout</span></code></a>, <code class="xref py py-class docutils literal notranslate"><span class="pre">BatchNorm</span></code>,
etc.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.extra_repr">
<code class="descname">extra_repr</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.extra_repr"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.extra_repr" title="Permalink to this definition">¶</a></dt>
<dd><p>Set the extra representation of the module</p>
<p>To print customized extra information, you should reimplement
this method in your own modules. Both single-line and multi-line
strings are acceptable.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.float">
<code class="descname">float</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.float"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.float" title="Permalink to this definition">¶</a></dt>
<dd><p>Casts all floating point parameters and buffers to float datatype.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>self</p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.forward">
<code class="descname">forward</code><span class="sig-paren">(</span><em>*input</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.forward" title="Permalink to this definition">¶</a></dt>
<dd><p>Defines the computation performed at every call.</p>
<p>Should be overridden by all subclasses.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Although the recipe for forward pass needs to be defined within
this function, one should call the <a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> instance afterwards
instead of this since the former takes care of running the
registered hooks while the latter silently ignores them.</p>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.half">
<code class="descname">half</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.half"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.half" title="Permalink to this definition">¶</a></dt>
<dd><p>Casts all floating point parameters and buffers to <code class="docutils literal notranslate"><span class="pre">half</span></code> datatype.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>self</p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.load_state_dict">
<code class="descname">load_state_dict</code><span class="sig-paren">(</span><em>state_dict</em>, <em>strict=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.load_state_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.load_state_dict" title="Permalink to this definition">¶</a></dt>
<dd><p>Copies parameters and buffers from <a class="reference internal" href="#torch.nn.Module.state_dict" title="torch.nn.Module.state_dict"><code class="xref py py-attr docutils literal notranslate"><span class="pre">state_dict</span></code></a> into
this module and its descendants. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">strict</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, then
the keys of <a class="reference internal" href="#torch.nn.Module.state_dict" title="torch.nn.Module.state_dict"><code class="xref py py-attr docutils literal notranslate"><span class="pre">state_dict</span></code></a> must exactly match the keys returned
by this module’s <a class="reference internal" href="#torch.nn.Module.state_dict" title="torch.nn.Module.state_dict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">state_dict()</span></code></a> function.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>state_dict</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><em>dict</em></a>) – a dict containing parameters and
persistent buffers.</p></li>
<li><p><strong>strict</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – whether to strictly enforce that the keys
in <a class="reference internal" href="#torch.nn.Module.state_dict" title="torch.nn.Module.state_dict"><code class="xref py py-attr docutils literal notranslate"><span class="pre">state_dict</span></code></a> match the keys returned by this module’s
<a class="reference internal" href="#torch.nn.Module.state_dict" title="torch.nn.Module.state_dict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">state_dict()</span></code></a> function. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><ul class="simple">
<li><p><strong>missing_keys</strong> is a list of str containing the missing keys</p></li>
<li><p><strong>unexpected_keys</strong> is a list of str containing the unexpected keys</p></li>
</ul>
</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">NamedTuple</span></code> with <code class="docutils literal notranslate"><span class="pre">missing_keys</span></code> and <code class="docutils literal notranslate"><span class="pre">unexpected_keys</span></code> fields</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.modules">
<code class="descname">modules</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.modules"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.modules" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over all modules in the network.</p>
<dl class="field-list simple">
<dt class="field-odd">Yields</dt>
<dd class="field-odd"><p><em>Module</em> – a module in the network</p>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Duplicate modules are returned only once. In the following
example, <code class="docutils literal notranslate"><span class="pre">l</span></code> will be returned only once.</p>
</div>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">l</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">net</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="n">l</span><span class="p">,</span> <span class="n">l</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">m</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">net</span><span class="o">.</span><span class="n">modules</span><span class="p">()):</span>
<span class="go">        print(idx, &#39;-&gt;&#39;, m)</span>

<span class="go">0 -&gt; Sequential(</span>
<span class="go">  (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">  (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">)</span>
<span class="go">1 -&gt; Linear(in_features=2, out_features=2, bias=True)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.named_buffers">
<code class="descname">named_buffers</code><span class="sig-paren">(</span><em>prefix=''</em>, <em>recurse=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.named_buffers"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.named_buffers" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over module buffers, yielding both the
name of the buffer as well as the buffer itself.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>prefix</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – prefix to prepend to all buffer names.</p></li>
<li><p><strong>recurse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – if True, then yields buffers of this module
and all submodules. Otherwise, yields only buffers that
are direct members of this module.</p></li>
</ul>
</dd>
<dt class="field-even">Yields</dt>
<dd class="field-even"><p><em>(string, torch.Tensor)</em> – Tuple containing the name and buffer</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">buf</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">named_buffers</span><span class="p">():</span>
<span class="gp">&gt;&gt;&gt; </span>   <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;running_var&#39;</span><span class="p">]:</span>
<span class="gp">&gt;&gt;&gt; </span>       <span class="nb">print</span><span class="p">(</span><span class="n">buf</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.named_children">
<code class="descname">named_children</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.named_children"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.named_children" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over immediate children modules, yielding both
the name of the module as well as the module itself.</p>
<dl class="field-list simple">
<dt class="field-odd">Yields</dt>
<dd class="field-odd"><p><em>(string, Module)</em> – Tuple containing a name and child module</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">named_children</span><span class="p">():</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;conv4&#39;</span><span class="p">,</span> <span class="s1">&#39;conv5&#39;</span><span class="p">]:</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="nb">print</span><span class="p">(</span><span class="n">module</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.named_modules">
<code class="descname">named_modules</code><span class="sig-paren">(</span><em>memo=None</em>, <em>prefix=''</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.named_modules"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.named_modules" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over all modules in the network, yielding
both the name of the module as well as the module itself.</p>
<dl class="field-list simple">
<dt class="field-odd">Yields</dt>
<dd class="field-odd"><p><em>(string, Module)</em> – Tuple of name and module</p>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Duplicate modules are returned only once. In the following
example, <code class="docutils literal notranslate"><span class="pre">l</span></code> will be returned only once.</p>
</div>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">l</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">net</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="n">l</span><span class="p">,</span> <span class="n">l</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">m</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">net</span><span class="o">.</span><span class="n">named_modules</span><span class="p">()):</span>
<span class="go">        print(idx, &#39;-&gt;&#39;, m)</span>

<span class="go">0 -&gt; (&#39;&#39;, Sequential(</span>
<span class="go">  (0): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">  (1): Linear(in_features=2, out_features=2, bias=True)</span>
<span class="go">))</span>
<span class="go">1 -&gt; (&#39;0&#39;, Linear(in_features=2, out_features=2, bias=True))</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.named_parameters">
<code class="descname">named_parameters</code><span class="sig-paren">(</span><em>prefix=''</em>, <em>recurse=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.named_parameters"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.named_parameters" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over module parameters, yielding both the
name of the parameter as well as the parameter itself.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>prefix</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – prefix to prepend to all parameter names.</p></li>
<li><p><strong>recurse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – if True, then yields parameters of this module
and all submodules. Otherwise, yields only parameters that
are direct members of this module.</p></li>
</ul>
</dd>
<dt class="field-even">Yields</dt>
<dd class="field-even"><p><em>(string, Parameter)</em> – Tuple containing the name and parameter</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">named_parameters</span><span class="p">():</span>
<span class="gp">&gt;&gt;&gt; </span>   <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;bias&#39;</span><span class="p">]:</span>
<span class="gp">&gt;&gt;&gt; </span>       <span class="nb">print</span><span class="p">(</span><span class="n">param</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.parameters">
<code class="descname">parameters</code><span class="sig-paren">(</span><em>recurse=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.parameters"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.parameters" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns an iterator over module parameters.</p>
<p>This is typically passed to an optimizer.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>recurse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – if True, then yields parameters of this module
and all submodules. Otherwise, yields only parameters that
are direct members of this module.</p>
</dd>
<dt class="field-even">Yields</dt>
<dd class="field-even"><p><em>Parameter</em> – module parameter</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">():</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="nb">print</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">param</span><span class="o">.</span><span class="n">data</span><span class="p">),</span> <span class="n">param</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">&lt;class &#39;torch.FloatTensor&#39;&gt; (20L,)</span>
<span class="go">&lt;class &#39;torch.FloatTensor&#39;&gt; (20L, 1L, 5L, 5L)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.register_backward_hook">
<code class="descname">register_backward_hook</code><span class="sig-paren">(</span><em>hook</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.register_backward_hook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.register_backward_hook" title="Permalink to this definition">¶</a></dt>
<dd><p>Registers a backward hook on the module.</p>
<p>The hook will be called every time the gradients with respect to module
inputs are computed. The hook should have the following signature:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">hook</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">grad_input</span><span class="p">,</span> <span class="n">grad_output</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tensor</span> <span class="ow">or</span> <span class="kc">None</span>
</pre></div>
</div>
<p>The <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_input</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_output</span></code> may be tuples if the
module has multiple inputs or outputs. The hook should not modify its
arguments, but it can optionally return a new gradient with respect to
input that will be used in place of <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_input</span></code> in subsequent
computations.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>a handle that can be used to remove the added hook by calling
<code class="docutils literal notranslate"><span class="pre">handle.remove()</span></code></p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.utils.hooks.RemovableHandle</span></code></p>
</dd>
</dl>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>The current implementation will not have the presented behavior
for complex <a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> that perform many operations.
In some failure cases, <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_input</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">grad_output</span></code> will only
contain the gradients for a subset of the inputs and outputs.
For such <a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a>, you should use <a class="reference internal" href="tensors.html#torch.Tensor.register_hook" title="torch.Tensor.register_hook"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.Tensor.register_hook()</span></code></a>
directly on a specific input or output to get the required gradients.</p>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.register_buffer">
<code class="descname">register_buffer</code><span class="sig-paren">(</span><em>name</em>, <em>tensor</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.register_buffer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.register_buffer" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a persistent buffer to the module.</p>
<p>This is typically used to register a buffer that should not to be
considered a model parameter. For example, BatchNorm’s <code class="docutils literal notranslate"><span class="pre">running_mean</span></code>
is not a parameter, but is part of the persistent state.</p>
<p>Buffers can be accessed as attributes using given names.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>name</strong> (<em>string</em>) – name of the buffer. The buffer can be accessed
from this module using the given name</p></li>
<li><p><strong>tensor</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – buffer to be registered.</p></li>
</ul>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="bp">self</span><span class="o">.</span><span class="n">register_buffer</span><span class="p">(</span><span class="s1">&#39;running_mean&#39;</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">num_features</span><span class="p">))</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.register_forward_hook">
<code class="descname">register_forward_hook</code><span class="sig-paren">(</span><em>hook</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.register_forward_hook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.register_forward_hook" title="Permalink to this definition">¶</a></dt>
<dd><p>Registers a forward hook on the module.</p>
<p>The hook will be called every time after <a class="reference internal" href="#torch.nn.Module.forward" title="torch.nn.Module.forward"><code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code></a> has computed an output.
It should have the following signature:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">hook</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="nb">input</span><span class="p">,</span> <span class="n">output</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span>
</pre></div>
</div>
<p>The hook should not modify the input or output.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>a handle that can be used to remove the added hook by calling
<code class="docutils literal notranslate"><span class="pre">handle.remove()</span></code></p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.utils.hooks.RemovableHandle</span></code></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.register_forward_pre_hook">
<code class="descname">register_forward_pre_hook</code><span class="sig-paren">(</span><em>hook</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.register_forward_pre_hook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.register_forward_pre_hook" title="Permalink to this definition">¶</a></dt>
<dd><p>Registers a forward pre-hook on the module.</p>
<p>The hook will be called every time before <a class="reference internal" href="#torch.nn.Module.forward" title="torch.nn.Module.forward"><code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code></a> is invoked.
It should have the following signature:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">hook</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="nb">input</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span>
</pre></div>
</div>
<p>The hook should not modify the input.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>a handle that can be used to remove the added hook by calling
<code class="docutils literal notranslate"><span class="pre">handle.remove()</span></code></p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.utils.hooks.RemovableHandle</span></code></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.register_parameter">
<code class="descname">register_parameter</code><span class="sig-paren">(</span><em>name</em>, <em>param</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.register_parameter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.register_parameter" title="Permalink to this definition">¶</a></dt>
<dd><p>Adds a parameter to the module.</p>
<p>The parameter can be accessed as an attribute using given name.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>name</strong> (<em>string</em>) – name of the parameter. The parameter can be accessed
from this module using the given name</p></li>
<li><p><strong>param</strong> (<a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><em>Parameter</em></a>) – parameter to be added to the module.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.state_dict">
<code class="descname">state_dict</code><span class="sig-paren">(</span><em>destination=None</em>, <em>prefix=''</em>, <em>keep_vars=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.state_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.state_dict" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns a dictionary containing a whole state of the module.</p>
<p>Both parameters and persistent buffers (e.g. running averages) are
included. Keys are corresponding parameter and buffer names.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>a dictionary containing a whole state of the module</p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)">dict</a></p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">module</span><span class="o">.</span><span class="n">state_dict</span><span class="p">()</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
<span class="go">[&#39;bias&#39;, &#39;weight&#39;]</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.to">
<code class="descname">to</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.to"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.to" title="Permalink to this definition">¶</a></dt>
<dd><p>Moves and/or casts the parameters and buffers.</p>
<p>This can be called as</p>
<dl class="function">
<dt>
<code class="descname">to</code><span class="sig-paren">(</span><em>device=None</em>, <em>dtype=None</em>, <em>non_blocking=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.to"><span class="viewcode-link">[source]</span></a></dt>
<dd></dd></dl>

<dl class="function">
<dt>
<code class="descname">to</code><span class="sig-paren">(</span><em>dtype</em>, <em>non_blocking=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.to"><span class="viewcode-link">[source]</span></a></dt>
<dd></dd></dl>

<dl class="function">
<dt>
<code class="descname">to</code><span class="sig-paren">(</span><em>tensor</em>, <em>non_blocking=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.to"><span class="viewcode-link">[source]</span></a></dt>
<dd></dd></dl>

<p>Its signature is similar to <a class="reference internal" href="tensors.html#torch.Tensor.to" title="torch.Tensor.to"><code class="xref py py-meth docutils literal notranslate"><span class="pre">torch.Tensor.to()</span></code></a>, but only accepts
floating point desired <code class="xref py py-attr docutils literal notranslate"><span class="pre">dtype</span></code> s. In addition, this method will
only cast the floating point parameters and buffers to <code class="xref py py-attr docutils literal notranslate"><span class="pre">dtype</span></code>
(if given). The integral parameters and buffers will be moved
<code class="xref py py-attr docutils literal notranslate"><span class="pre">device</span></code>, if that is given, but with dtypes unchanged. When
<code class="xref py py-attr docutils literal notranslate"><span class="pre">non_blocking</span></code> is set, it tries to convert/move asynchronously
with respect to the host if possible, e.g., moving CPU Tensors with
pinned memory to CUDA devices.</p>
<p>See below for examples.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This method modifies the module in-place.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>device</strong> (<code class="xref py py-class docutils literal notranslate"><span class="pre">torch.device</span></code>) – the desired device of the parameters
and buffers in this module</p></li>
<li><p><strong>dtype</strong> (<code class="xref py py-class docutils literal notranslate"><span class="pre">torch.dtype</span></code>) – the desired floating point type of
the floating point parameters and buffers in this module</p></li>
<li><p><strong>tensor</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>torch.Tensor</em></a>) – Tensor whose dtype and device are the desired
dtype and device for all parameters and buffers in this module</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>self</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span><span class="o">.</span><span class="n">weight</span>
<span class="go">Parameter containing:</span>
<span class="go">tensor([[ 0.1913, -0.3420],</span>
<span class="go">        [-0.5113, -0.2325]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">double</span><span class="p">)</span>
<span class="go">Linear(in_features=2, out_features=2, bias=True)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span><span class="o">.</span><span class="n">weight</span>
<span class="go">Parameter containing:</span>
<span class="go">tensor([[ 0.1913, -0.3420],</span>
<span class="go">        [-0.5113, -0.2325]], dtype=torch.float64)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">gpu1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">&quot;cuda:1&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">gpu1</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">half</span><span class="p">,</span> <span class="n">non_blocking</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="go">Linear(in_features=2, out_features=2, bias=True)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span><span class="o">.</span><span class="n">weight</span>
<span class="go">Parameter containing:</span>
<span class="go">tensor([[ 0.1914, -0.3420],</span>
<span class="go">        [-0.5112, -0.2324]], dtype=torch.float16, device=&#39;cuda:1&#39;)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cpu</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">&quot;cpu&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">cpu</span><span class="p">)</span>
<span class="go">Linear(in_features=2, out_features=2, bias=True)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">linear</span><span class="o">.</span><span class="n">weight</span>
<span class="go">Parameter containing:</span>
<span class="go">tensor([[ 0.1914, -0.3420],</span>
<span class="go">        [-0.5112, -0.2324]], dtype=torch.float16)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.train">
<code class="descname">train</code><span class="sig-paren">(</span><em>mode=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.train"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.train" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets the module in training mode.</p>
<p>This has any effect only on certain modules. See documentations of
particular modules for details of their behaviors in training/evaluation
mode, if they are affected, e.g. <a class="reference internal" href="#torch.nn.Dropout" title="torch.nn.Dropout"><code class="xref py py-class docutils literal notranslate"><span class="pre">Dropout</span></code></a>, <code class="xref py py-class docutils literal notranslate"><span class="pre">BatchNorm</span></code>,
etc.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>self</p>
</dd>
<dt class="field-even">Return type</dt>
<dd class="field-even"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.type">
<code class="descname">type</code><span class="sig-paren">(</span><em>dst_type</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.type"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.type" title="Permalink to this definition">¶</a></dt>
<dd><p>Casts all parameters and buffers to <code class="xref py py-attr docutils literal notranslate"><span class="pre">dst_type</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>dst_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#type" title="(in Python v3.7)"><em>type</em></a><em> or </em><em>string</em>) – the desired type</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>self</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p><a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module">Module</a></p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.Module.zero_grad">
<code class="descname">zero_grad</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/module.html#Module.zero_grad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Module.zero_grad" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets gradients of all model parameters to zero.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="sequential">
<h3><span class="hidden-section">Sequential</span><a class="headerlink" href="#sequential" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Sequential">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Sequential</code><span class="sig-paren">(</span><em>*args</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#Sequential"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Sequential" title="Permalink to this definition">¶</a></dt>
<dd><p>A sequential container.
Modules will be added to it in the order they are passed in the constructor.
Alternatively, an ordered dict of modules can also be passed in.</p>
<p>To make it easier to understand, here is a small example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Example of using Sequential</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
          <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">20</span><span class="p">,</span><span class="mi">5</span><span class="p">),</span>
          <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span>
          <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span><span class="mi">64</span><span class="p">,</span><span class="mi">5</span><span class="p">),</span>
          <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()</span>
        <span class="p">)</span>

<span class="c1"># Example of using Sequential with OrderedDict</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span><span class="n">OrderedDict</span><span class="p">([</span>
          <span class="p">(</span><span class="s1">&#39;conv1&#39;</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">20</span><span class="p">,</span><span class="mi">5</span><span class="p">)),</span>
          <span class="p">(</span><span class="s1">&#39;relu1&#39;</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()),</span>
          <span class="p">(</span><span class="s1">&#39;conv2&#39;</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span><span class="mi">64</span><span class="p">,</span><span class="mi">5</span><span class="p">)),</span>
          <span class="p">(</span><span class="s1">&#39;relu2&#39;</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">())</span>
        <span class="p">]))</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="modulelist">
<h3><span class="hidden-section">ModuleList</span><a class="headerlink" href="#modulelist" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ModuleList">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ModuleList</code><span class="sig-paren">(</span><em>modules=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleList"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleList" title="Permalink to this definition">¶</a></dt>
<dd><p>Holds submodules in a list.</p>
<p><a class="reference internal" href="#torch.nn.ModuleList" title="torch.nn.ModuleList"><code class="xref py py-class docutils literal notranslate"><span class="pre">ModuleList</span></code></a> can be indexed like a regular Python list, but
modules it contains are properly registered, and will be visible by all
<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> methods.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>modules</strong> (<em>iterable</em><em>, </em><em>optional</em>) – an iterable of modules to add</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MyModule</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MyModule</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">linears</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleList</span><span class="p">([</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">)])</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="c1"># ModuleList can act as an iterable, or be indexed using ints</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">linears</span><span class="p">):</span>
            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">linears</span><span class="p">[</span><span class="n">i</span> <span class="o">//</span> <span class="mi">2</span><span class="p">](</span><span class="n">x</span><span class="p">)</span> <span class="o">+</span> <span class="n">l</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.nn.ModuleList.append">
<code class="descname">append</code><span class="sig-paren">(</span><em>module</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleList.append"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleList.append" title="Permalink to this definition">¶</a></dt>
<dd><p>Appends a given module to the end of the list.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>nn.Module</em></a>) – module to append</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ModuleList.extend">
<code class="descname">extend</code><span class="sig-paren">(</span><em>modules</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleList.extend"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleList.extend" title="Permalink to this definition">¶</a></dt>
<dd><p>Appends modules from a Python iterable to the end of the list.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>modules</strong> (<em>iterable</em>) – iterable of modules to append</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ModuleList.insert">
<code class="descname">insert</code><span class="sig-paren">(</span><em>index</em>, <em>module</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleList.insert"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleList.insert" title="Permalink to this definition">¶</a></dt>
<dd><p>Insert a given module before a given index in the list.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>index</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – index to insert.</p></li>
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>nn.Module</em></a>) – module to insert</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="moduledict">
<h3><span class="hidden-section">ModuleDict</span><a class="headerlink" href="#moduledict" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ModuleDict">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ModuleDict</code><span class="sig-paren">(</span><em>modules=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleDict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleDict" title="Permalink to this definition">¶</a></dt>
<dd><p>Holds submodules in a dictionary.</p>
<p><a class="reference internal" href="#torch.nn.ModuleDict" title="torch.nn.ModuleDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ModuleDict</span></code></a> can be indexed like a regular Python dictionary,
but modules it contains are properly registered, and will be visible by all
<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> methods.</p>
<p><a class="reference internal" href="#torch.nn.ModuleDict" title="torch.nn.ModuleDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ModuleDict</span></code></a> is an <strong>ordered</strong> dictionary that respects</p>
<ul class="simple">
<li><p>the order of insertion, and</p></li>
<li><p>in <a class="reference internal" href="#torch.nn.ModuleDict.update" title="torch.nn.ModuleDict.update"><code class="xref py py-meth docutils literal notranslate"><span class="pre">update()</span></code></a>, the order of the merged <code class="docutils literal notranslate"><span class="pre">OrderedDict</span></code>
or another <a class="reference internal" href="#torch.nn.ModuleDict" title="torch.nn.ModuleDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ModuleDict</span></code></a> (the argument to <a class="reference internal" href="#torch.nn.ModuleDict.update" title="torch.nn.ModuleDict.update"><code class="xref py py-meth docutils literal notranslate"><span class="pre">update()</span></code></a>).</p></li>
</ul>
<p>Note that <a class="reference internal" href="#torch.nn.ModuleDict.update" title="torch.nn.ModuleDict.update"><code class="xref py py-meth docutils literal notranslate"><span class="pre">update()</span></code></a> with other unordered mapping
types (e.g., Python’s plain <code class="docutils literal notranslate"><span class="pre">dict</span></code>) does not preserve the order of the
merged mapping.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>modules</strong> (<em>iterable</em><em>, </em><em>optional</em>) – a mapping (dictionary) of (string: module)
or an iterable of key-value pairs of type (string, module)</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MyModule</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MyModule</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">choices</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleDict</span><span class="p">({</span>
                <span class="s1">&#39;conv&#39;</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span>
                <span class="s1">&#39;pool&#39;</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool2d</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
        <span class="p">})</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">activations</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ModuleDict</span><span class="p">([</span>
                <span class="p">[</span><span class="s1">&#39;lrelu&#39;</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">LeakyReLU</span><span class="p">()],</span>
                <span class="p">[</span><span class="s1">&#39;prelu&#39;</span><span class="p">,</span> <span class="n">nn</span><span class="o">.</span><span class="n">PReLU</span><span class="p">()]</span>
        <span class="p">])</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">choice</span><span class="p">,</span> <span class="n">act</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">choices</span><span class="p">[</span><span class="n">choice</span><span class="p">](</span><span class="n">x</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">activations</span><span class="p">[</span><span class="n">act</span><span class="p">](</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.nn.ModuleDict.clear">
<code class="descname">clear</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleDict.clear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleDict.clear" title="Permalink to this definition">¶</a></dt>
<dd><p>Remove all items from the ModuleDict.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ModuleDict.items">
<code class="descname">items</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleDict.items"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleDict.items" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an iterable of the ModuleDict key/value pairs.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ModuleDict.keys">
<code class="descname">keys</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleDict.keys"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleDict.keys" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an iterable of the ModuleDict keys.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ModuleDict.pop">
<code class="descname">pop</code><span class="sig-paren">(</span><em>key</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleDict.pop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleDict.pop" title="Permalink to this definition">¶</a></dt>
<dd><p>Remove key from the ModuleDict and return its module.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>key</strong> (<em>string</em>) – key to pop from the ModuleDict</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ModuleDict.update">
<code class="descname">update</code><span class="sig-paren">(</span><em>modules</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleDict.update"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleDict.update" title="Permalink to this definition">¶</a></dt>
<dd><p>Update the <a class="reference internal" href="#torch.nn.ModuleDict" title="torch.nn.ModuleDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ModuleDict</span></code></a> with the key-value pairs from a
mapping or an iterable, overwriting existing keys.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">modules</span></code> is an <code class="docutils literal notranslate"><span class="pre">OrderedDict</span></code>, a <a class="reference internal" href="#torch.nn.ModuleDict" title="torch.nn.ModuleDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ModuleDict</span></code></a>, or
an iterable of key-value pairs, the order of new elements in it is preserved.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>modules</strong> (<em>iterable</em>) – a mapping (dictionary) from string to <a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a>,
or an iterable of key-value pairs of type (string, <a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a>)</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ModuleDict.values">
<code class="descname">values</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ModuleDict.values"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ModuleDict.values" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an iterable of the ModuleDict values.</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="parameterlist">
<h3><span class="hidden-section">ParameterList</span><a class="headerlink" href="#parameterlist" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ParameterList">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ParameterList</code><span class="sig-paren">(</span><em>parameters=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterList"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterList" title="Permalink to this definition">¶</a></dt>
<dd><p>Holds parameters in a list.</p>
<p><a class="reference internal" href="#torch.nn.ParameterList" title="torch.nn.ParameterList"><code class="xref py py-class docutils literal notranslate"><span class="pre">ParameterList</span></code></a> can be indexed like a regular Python
list, but parameters it contains are properly registered, and will be
visible by all <a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> methods.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>parameters</strong> (<em>iterable</em><em>, </em><em>optional</em>) – an iterable of <a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><code class="xref py py-class docutils literal notranslate"><span class="pre">Parameter</span></code></a> to add</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MyModule</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MyModule</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ParameterList</span><span class="p">([</span><span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">)])</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="c1"># ParameterList can act as an iterable, or be indexed using ints</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">p</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">):</span>
            <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="n">i</span> <span class="o">//</span> <span class="mi">2</span><span class="p">]</span><span class="o">.</span><span class="n">mm</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="o">+</span> <span class="n">p</span><span class="o">.</span><span class="n">mm</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.nn.ParameterList.append">
<code class="descname">append</code><span class="sig-paren">(</span><em>parameter</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterList.append"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterList.append" title="Permalink to this definition">¶</a></dt>
<dd><p>Appends a given parameter at the end of the list.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>parameter</strong> (<a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><em>nn.Parameter</em></a>) – parameter to append</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ParameterList.extend">
<code class="descname">extend</code><span class="sig-paren">(</span><em>parameters</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterList.extend"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterList.extend" title="Permalink to this definition">¶</a></dt>
<dd><p>Appends parameters from a Python iterable to the end of the list.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>parameters</strong> (<em>iterable</em>) – iterable of parameters to append</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="parameterdict">
<h3><span class="hidden-section">ParameterDict</span><a class="headerlink" href="#parameterdict" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ParameterDict">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ParameterDict</code><span class="sig-paren">(</span><em>parameters=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterDict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterDict" title="Permalink to this definition">¶</a></dt>
<dd><p>Holds parameters in a dictionary.</p>
<p>ParameterDict can be indexed like a regular Python dictionary, but parameters it
contains are properly registered, and will be visible by all Module methods.</p>
<p><a class="reference internal" href="#torch.nn.ParameterDict" title="torch.nn.ParameterDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ParameterDict</span></code></a> is an <strong>ordered</strong> dictionary that respects</p>
<ul class="simple">
<li><p>the order of insertion, and</p></li>
<li><p>in <a class="reference internal" href="#torch.nn.ParameterDict.update" title="torch.nn.ParameterDict.update"><code class="xref py py-meth docutils literal notranslate"><span class="pre">update()</span></code></a>, the order of the merged <code class="docutils literal notranslate"><span class="pre">OrderedDict</span></code>
or another <a class="reference internal" href="#torch.nn.ParameterDict" title="torch.nn.ParameterDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ParameterDict</span></code></a> (the argument to
<a class="reference internal" href="#torch.nn.ParameterDict.update" title="torch.nn.ParameterDict.update"><code class="xref py py-meth docutils literal notranslate"><span class="pre">update()</span></code></a>).</p></li>
</ul>
<p>Note that <a class="reference internal" href="#torch.nn.ParameterDict.update" title="torch.nn.ParameterDict.update"><code class="xref py py-meth docutils literal notranslate"><span class="pre">update()</span></code></a> with other unordered mapping
types (e.g., Python’s plain <code class="docutils literal notranslate"><span class="pre">dict</span></code>) does not preserve the order of the
merged mapping.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>parameters</strong> (<em>iterable</em><em>, </em><em>optional</em>) – a mapping (dictionary) of
(string : <a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><code class="xref py py-class docutils literal notranslate"><span class="pre">Parameter</span></code></a>) or an iterable of key-value pairs
of type (string, <a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><code class="xref py py-class docutils literal notranslate"><span class="pre">Parameter</span></code></a>)</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MyModule</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MyModule</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">params</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ParameterDict</span><span class="p">({</span>
                <span class="s1">&#39;left&#39;</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">10</span><span class="p">)),</span>
                <span class="s1">&#39;right&#39;</span><span class="p">:</span> <span class="n">nn</span><span class="o">.</span><span class="n">Parameter</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
        <span class="p">})</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">choice</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">params</span><span class="p">[</span><span class="n">choice</span><span class="p">]</span><span class="o">.</span><span class="n">mm</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.nn.ParameterDict.clear">
<code class="descname">clear</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterDict.clear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterDict.clear" title="Permalink to this definition">¶</a></dt>
<dd><p>Remove all items from the ParameterDict.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ParameterDict.items">
<code class="descname">items</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterDict.items"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterDict.items" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an iterable of the ParameterDict key/value pairs.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ParameterDict.keys">
<code class="descname">keys</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterDict.keys"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterDict.keys" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an iterable of the ParameterDict keys.</p>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ParameterDict.pop">
<code class="descname">pop</code><span class="sig-paren">(</span><em>key</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterDict.pop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterDict.pop" title="Permalink to this definition">¶</a></dt>
<dd><p>Remove key from the ParameterDict and return its parameter.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>key</strong> (<em>string</em>) – key to pop from the ParameterDict</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ParameterDict.update">
<code class="descname">update</code><span class="sig-paren">(</span><em>parameters</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterDict.update"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterDict.update" title="Permalink to this definition">¶</a></dt>
<dd><p>Update the <a class="reference internal" href="#torch.nn.ParameterDict" title="torch.nn.ParameterDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ParameterDict</span></code></a> with the key-value pairs from a
mapping or an iterable, overwriting existing keys.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">parameters</span></code> is an <code class="docutils literal notranslate"><span class="pre">OrderedDict</span></code>, a <a class="reference internal" href="#torch.nn.ParameterDict" title="torch.nn.ParameterDict"><code class="xref py py-class docutils literal notranslate"><span class="pre">ParameterDict</span></code></a>, or
an iterable of key-value pairs, the order of new elements in it is preserved.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>parameters</strong> (<em>iterable</em>) – a mapping (dictionary) from string to
<a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><code class="xref py py-class docutils literal notranslate"><span class="pre">Parameter</span></code></a>, or an iterable of
key-value pairs of type (string, <a class="reference internal" href="#torch.nn.Parameter" title="torch.nn.Parameter"><code class="xref py py-class docutils literal notranslate"><span class="pre">Parameter</span></code></a>)</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.ParameterDict.values">
<code class="descname">values</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/container.html#ParameterDict.values"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ParameterDict.values" title="Permalink to this definition">¶</a></dt>
<dd><p>Return an iterable of the ParameterDict values.</p>
</dd></dl>

</dd></dl>

</div>
</div>
<div class="section" id="convolution-layers">
<h2>Convolution layers<a class="headerlink" href="#convolution-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="conv1d">
<h3><span class="hidden-section">Conv1d</span><a class="headerlink" href="#conv1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Conv1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Conv1d</code><span class="sig-paren">(</span><em>in_channels</em>, <em>out_channels</em>, <em>kernel_size</em>, <em>stride=1</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>groups=1</em>, <em>bias=True</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/conv.html#Conv1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Conv1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D convolution over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size
<span class="math">\((N, C_{\text{in}}, L)\)</span> and output <span class="math">\((N, C_{\text{out}}, L_{\text{out}})\)</span> can be
precisely described as:</p>
<div class="math">
\[\text{out}(N_i, C_{\text{out}_j}) = \text{bias}(C_{\text{out}_j}) +
\sum_{k = 0}^{C_{in} - 1} \text{weight}(C_{\text{out}_j}, k)
\star \text{input}(N_i, k)

\]</div>
<p>where <span class="math">\(\star\)</span> is the valid <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a> operator,
<span class="math">\(N\)</span> is a batch size, <span class="math">\(C\)</span> denotes a number of channels,
<span class="math">\(L\)</span> is a length of signal sequence.</p>
<ul>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the cross-correlation, a single
number or a one-element tuple.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both sides
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also
known as the à trous algorithm. It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a>
has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code> controls the connections between inputs and outputs.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">out_channels</span></code> must both be divisible by
<code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code>. For example,</p>
<blockquote>
<div><ul class="simple">
<li><p>At groups=1, all inputs are convolved to all outputs.</p></li>
<li><p>At groups=2, the operation becomes equivalent to having two conv
layers side by side, each seeing half the input channels,
and producing half the output channels, and both subsequently
concatenated.</p></li>
<li><p>At groups= <code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code>, each input channel is convolved with
its own set of filters,
of size
<span class="math">\(\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor\)</span>.</p></li>
</ul>
</div></blockquote>
</li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Depending of the size of your kernel, several (of the last)
columns of the input might be lost, because it is a valid
<a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>, and not a full <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>.
It is up to the user to add proper padding.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When <cite>groups == in_channels</cite> and <cite>out_channels == K * in_channels</cite>,
where <cite>K</cite> is a positive integer, this operation is also termed in
literature as depthwise convolution.</p>
<p>In other words, for an input of size <span class="math">\((N, C_{in}, L_{in})\)</span>,
a depthwise convolution with a depthwise multiplier <cite>K</cite>, can be constructed by arguments
<span class="math">\((C_\text{in}=C_{in}, C_\text{out}=C_{in} \times K, ..., \text{groups}=C_{in})\)</span>.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels in the input image</p></li>
<li><p><strong>out_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels produced by the convolution</p></li>
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the convolving kernel</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Stride of the convolution. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Zero-padding added to both sides of
the input. Default: 0</p></li>
<li><p><strong>padding_mode</strong> (<em>string</em><em>, </em><em>optional</em>) – <cite>zeros</cite></p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Spacing between kernel
elements. Default: 1</p></li>
<li><p><strong>groups</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Number of blocked connections from input
channels to output channels. Default: 1</p></li>
<li><p><strong>bias</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, adds a learnable bias to the output. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C_{in}, L_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C_{out}, L_{out})\)</span> where</p>
<div class="math">
\[L_{out} = \left\lfloor\frac{L_{in} + 2 \times \text{padding} - \text{dilation}
          \times (\text{kernel\_size} - 1) - 1}{\text{stride}} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~Conv1d.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape
<span class="math">\((\text{out\_channels}, \frac{\text{in\_channels}}{\text{groups}}, \text{kernel\_size})\)</span>.
The values of these weights are sampled from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \text{kernel\_size}}\)</span></p></li>
<li><p><strong>~Conv1d.bias</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable bias of the module of shape
(out_channels). If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, then the values of these weights are
sampled from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \text{kernel\_size}}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv1d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="conv2d">
<h3><span class="hidden-section">Conv2d</span><a class="headerlink" href="#conv2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Conv2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Conv2d</code><span class="sig-paren">(</span><em>in_channels</em>, <em>out_channels</em>, <em>kernel_size</em>, <em>stride=1</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>groups=1</em>, <em>bias=True</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/conv.html#Conv2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Conv2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D convolution over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size
<span class="math">\((N, C_{\text{in}}, H, W)\)</span> and output <span class="math">\((N, C_{\text{out}}, H_{\text{out}}, W_{\text{out}})\)</span>
can be precisely described as:</p>
<div class="math">
\[\text{out}(N_i, C_{\text{out}_j}) = \text{bias}(C_{\text{out}_j}) +
\sum_{k = 0}^{C_{\text{in}} - 1} \text{weight}(C_{\text{out}_j}, k) \star \text{input}(N_i, k)

\]</div>
<p>where <span class="math">\(\star\)</span> is the valid 2D <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a> operator,
<span class="math">\(N\)</span> is a batch size, <span class="math">\(C\)</span> denotes a number of channels,
<span class="math">\(H\)</span> is a height of input planes in pixels, and <span class="math">\(W\)</span> is
width in pixels.</p>
<ul>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the cross-correlation, a single
number or a tuple.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both
sides for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points for each dimension.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also
known as the à trous algorithm. It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a>
has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code> controls the connections between inputs and outputs.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">out_channels</span></code> must both be divisible by
<code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code>. For example,</p>
<blockquote>
<div><ul class="simple">
<li><p>At groups=1, all inputs are convolved to all outputs.</p></li>
<li><p>At groups=2, the operation becomes equivalent to having two conv
layers side by side, each seeing half the input channels,
and producing half the output channels, and both subsequently
concatenated.</p></li>
<li><p>At groups= <code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code>, each input channel is convolved with
its own set of filters, of size:
<span class="math">\(\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor\)</span>.</p></li>
</ul>
</div></blockquote>
</li>
</ul>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the height and width dimension</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of two ints – in which case, the first <cite>int</cite> is used for the height dimension,
and the second <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Depending of the size of your kernel, several (of the last)
columns of the input might be lost, because it is a valid <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>,
and not a full <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>.
It is up to the user to add proper padding.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When <cite>groups == in_channels</cite> and <cite>out_channels == K * in_channels</cite>,
where <cite>K</cite> is a positive integer, this operation is also termed in
literature as depthwise convolution.</p>
<p>In other words, for an input of size <span class="math">\((N, C_{in}, H_{in}, W_{in})\)</span>,
a depthwise convolution with a depthwise multiplier <cite>K</cite>, can be constructed by arguments
<span class="math">\((in\_channels=C_{in}, out\_channels=C_{in} \times K, ..., groups=C_{in})\)</span>.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels in the input image</p></li>
<li><p><strong>out_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels produced by the convolution</p></li>
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the convolving kernel</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Stride of the convolution. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Zero-padding added to both sides of the input. Default: 0</p></li>
<li><p><strong>padding_mode</strong> (<em>string</em><em>, </em><em>optional</em>) – <cite>zeros</cite></p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Spacing between kernel elements. Default: 1</p></li>
<li><p><strong>groups</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Number of blocked connections from input channels to output channels. Default: 1</p></li>
<li><p><strong>bias</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, adds a learnable bias to the output. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C_{out}, H_{out}, W_{out})\)</span> where</p>
<div class="math">
\[H_{out} = \left\lfloor\frac{H_{in}  + 2 \times \text{padding}[0] - \text{dilation}[0]
          \times (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor\frac{W_{in}  + 2 \times \text{padding}[1] - \text{dilation}[1]
          \times (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~Conv2d.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape
<span class="math">\((\text{out\_channels}, \frac{\text{in\_channels}}{\text{groups}},\)</span>
<span class="math">\(\text{kernel\_size[0]}, \text{kernel\_size[1]})\)</span>.
The values of these weights are sampled from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{1}\text{kernel\_size}[i]}\)</span></p></li>
<li><p><strong>~Conv2d.bias</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable bias of the module of shape (out_channels). If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>,
then the values of these weights are
sampled from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{1}\text{kernel\_size}[i]}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With square kernels and equal stride</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># non-square kernels and unequal stride and with padding</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">padding</span><span class="o">=</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># non-square kernels and unequal stride and with padding and dilation</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">padding</span><span class="o">=</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">dilation</span><span class="o">=</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="conv3d">
<h3><span class="hidden-section">Conv3d</span><a class="headerlink" href="#conv3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Conv3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Conv3d</code><span class="sig-paren">(</span><em>in_channels</em>, <em>out_channels</em>, <em>kernel_size</em>, <em>stride=1</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>groups=1</em>, <em>bias=True</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/conv.html#Conv3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Conv3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D convolution over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size <span class="math">\((N, C_{in}, D, H, W)\)</span>
and output <span class="math">\((N, C_{out}, D_{out}, H_{out}, W_{out})\)</span> can be precisely described as:</p>
<div class="math">
\[out(N_i, C_{out_j}) = bias(C_{out_j}) +
                        \sum_{k = 0}^{C_{in} - 1} weight(C_{out_j}, k) \star input(N_i, k)

\]</div>
<p>where <span class="math">\(\star\)</span> is the valid 3D <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a> operator</p>
<ul>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the cross-correlation.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both
sides for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points for each dimension.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also known as the à trous algorithm.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code> controls the connections between inputs and outputs.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">out_channels</span></code> must both be divisible by
<code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code>. For example,</p>
<blockquote>
<div><ul class="simple">
<li><p>At groups=1, all inputs are convolved to all outputs.</p></li>
<li><p>At groups=2, the operation becomes equivalent to having two conv
layers side by side, each seeing half the input channels,
and producing half the output channels, and both subsequently
concatenated.</p></li>
<li><p>At groups= <code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code>, each input channel is convolved with
its own set of filters, of size
<span class="math">\(\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor\)</span>.</p></li>
</ul>
</div></blockquote>
</li>
</ul>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the depth, height and width dimension</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of three ints – in which case, the first <cite>int</cite> is used for the depth dimension,
the second <cite>int</cite> for the height dimension and the third <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Depending of the size of your kernel, several (of the last)
columns of the input might be lost, because it is a valid <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>,
and not a full <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>.
It is up to the user to add proper padding.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When <cite>groups == in_channels</cite> and <cite>out_channels == K * in_channels</cite>,
where <cite>K</cite> is a positive integer, this operation is also termed in
literature as depthwise convolution.</p>
<p>In other words, for an input of size <span class="math">\((N, C_{in}, D_{in}, H_{in}, W_{in})\)</span>,
a depthwise convolution with a depthwise multiplier <cite>K</cite>, can be constructed by arguments
<span class="math">\((in\_channels=C_{in}, out\_channels=C_{in} \times K, ..., groups=C_{in})\)</span>.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels in the input image</p></li>
<li><p><strong>out_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels produced by the convolution</p></li>
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the convolving kernel</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Stride of the convolution. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Zero-padding added to all three sides of the input. Default: 0</p></li>
<li><p><strong>padding_mode</strong> (<em>string</em><em>, </em><em>optional</em>) – <cite>zeros</cite></p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Spacing between kernel elements. Default: 1</p></li>
<li><p><strong>groups</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Number of blocked connections from input channels to output channels. Default: 1</p></li>
<li><p><strong>bias</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, adds a learnable bias to the output. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C_{in}, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C_{out}, D_{out}, H_{out}, W_{out})\)</span> where</p>
<div class="math">
\[D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] - \text{dilation}[0]
      \times (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor

\]</div>
<div class="math">
\[H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] - \text{dilation}[1]
      \times (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] - \text{dilation}[2]
      \times (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~Conv3d.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape
<span class="math">\((\text{out\_channels}, \frac{\text{in\_channels}}{\text{groups}},\)</span>
<span class="math">\(\text{kernel\_size[0]}, \text{kernel\_size[1]}, \text{kernel\_size[2]})\)</span>.
The values of these weights are sampled from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{2}\text{kernel\_size}[i]}\)</span></p></li>
<li><p><strong>~Conv3d.bias</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable bias of the module of shape (out_channels). If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>,
then the values of these weights are
sampled from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{2}\text{kernel\_size}[i]}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With square kernels and equal stride</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv3d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># non-square kernels and unequal stride and with padding</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv3d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">padding</span><span class="o">=</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="convtranspose1d">
<h3><span class="hidden-section">ConvTranspose1d</span><a class="headerlink" href="#convtranspose1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ConvTranspose1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ConvTranspose1d</code><span class="sig-paren">(</span><em>in_channels</em>, <em>out_channels</em>, <em>kernel_size</em>, <em>stride=1</em>, <em>padding=0</em>, <em>output_padding=0</em>, <em>groups=1</em>, <em>bias=True</em>, <em>dilation=1</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/conv.html#ConvTranspose1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ConvTranspose1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D transposed convolution operator over an input image
composed of several input planes.</p>
<p>This module can be seen as the gradient of Conv1d with respect to its input.
It is also known as a fractionally-strided convolution or
a deconvolution (although it is not an actual deconvolution operation).</p>
<ul>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the cross-correlation.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both
sides for <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> number of points. See note
below for details.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> controls the additional size added to one side
of the output shape. See note below for details.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also known as the à trous algorithm.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code> controls the connections between inputs and outputs.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">out_channels</span></code> must both be divisible by
<code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code>. For example,</p>
<blockquote>
<div><ul class="simple">
<li><p>At groups=1, all inputs are convolved to all outputs.</p></li>
<li><p>At groups=2, the operation becomes equivalent to having two conv
layers side by side, each seeing half the input channels,
and producing half the output channels, and both subsequently
concatenated.</p></li>
<li><p>At groups= <code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code>, each input channel is convolved with
its own set of filters (of size
<span class="math">\(\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor\)</span>).</p></li>
</ul>
</div></blockquote>
</li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Depending of the size of your kernel, several (of the last)
columns of the input might be lost, because it is a valid <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>,
and not a full <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>.
It is up to the user to add proper padding.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> argument effectively adds <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code>
amount of zero padding to both sizes of the input. This is set so that
when a <a class="reference internal" href="#torch.nn.Conv1d" title="torch.nn.Conv1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv1d</span></code></a> and a <a class="reference internal" href="#torch.nn.ConvTranspose1d" title="torch.nn.ConvTranspose1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">ConvTranspose1d</span></code></a>
are initialized with same parameters, they are inverses of each other in
regard to the input and output shapes. However, when <code class="docutils literal notranslate"><span class="pre">stride</span> <span class="pre">&gt;</span> <span class="pre">1</span></code>,
<a class="reference internal" href="#torch.nn.Conv1d" title="torch.nn.Conv1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv1d</span></code></a> maps multiple input shapes to the same output
shape. <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> is provided to resolve this ambiguity by
effectively increasing the calculated output shape on one side. Note
that <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> is only used to find output shape, but does
not actually add zero-padding to output.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels in the input image</p></li>
<li><p><strong>out_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels produced by the convolution</p></li>
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the convolving kernel</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Stride of the convolution. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> zero-padding
will be added to both sides of the input. Default: 0</p></li>
<li><p><strong>output_padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Additional size added to one side
of the output shape. Default: 0</p></li>
<li><p><strong>groups</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Number of blocked connections from input channels to output channels. Default: 1</p></li>
<li><p><strong>bias</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, adds a learnable bias to the output. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Spacing between kernel elements. Default: 1</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C_{in}, L_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C_{out}, L_{out})\)</span> where</p>
<div class="math">
\[L_{out} = (L_{in} - 1) \times \text{stride} - 2 \times \text{padding} + \text{dilation}
          \times (\text{kernel\_size} - 1) + \text{output\_padding} + 1

\]</div>
</li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~ConvTranspose1d.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape
<span class="math">\((\text{in\_channels}, \frac{\text{out\_channels}}{\text{groups}},\)</span>
<span class="math">\(\text{kernel\_size})\)</span>.
The values of these weights are sampled from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \text{kernel\_size}}\)</span></p></li>
<li><p><strong>~ConvTranspose1d.bias</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable bias of the module of shape (out_channels).
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, then the values of these weights are
sampled from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \text{kernel\_size}}\)</span></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="convtranspose2d">
<h3><span class="hidden-section">ConvTranspose2d</span><a class="headerlink" href="#convtranspose2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ConvTranspose2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ConvTranspose2d</code><span class="sig-paren">(</span><em>in_channels</em>, <em>out_channels</em>, <em>kernel_size</em>, <em>stride=1</em>, <em>padding=0</em>, <em>output_padding=0</em>, <em>groups=1</em>, <em>bias=True</em>, <em>dilation=1</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/conv.html#ConvTranspose2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ConvTranspose2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D transposed convolution operator over an input image
composed of several input planes.</p>
<p>This module can be seen as the gradient of Conv2d with respect to its input.
It is also known as a fractionally-strided convolution or
a deconvolution (although it is not an actual deconvolution operation).</p>
<ul>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the cross-correlation.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both
sides for <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> number of points. See note
below for details.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> controls the additional size added to one side
of the output shape. See note below for details.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also known as the à trous algorithm.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code> controls the connections between inputs and outputs.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">out_channels</span></code> must both be divisible by
<code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code>. For example,</p>
<blockquote>
<div><ul class="simple">
<li><p>At groups=1, all inputs are convolved to all outputs.</p></li>
<li><p>At groups=2, the operation becomes equivalent to having two conv
layers side by side, each seeing half the input channels,
and producing half the output channels, and both subsequently
concatenated.</p></li>
<li><p>At groups= <code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code>, each input channel is convolved with
its own set of filters (of size
<span class="math">\(\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor\)</span>).</p></li>
</ul>
</div></blockquote>
</li>
</ul>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code>
can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the height and width dimensions</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of two ints – in which case, the first <cite>int</cite> is used for the height dimension,
and the second <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Depending of the size of your kernel, several (of the last)
columns of the input might be lost, because it is a valid <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>,
and not a full <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>.
It is up to the user to add proper padding.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> argument effectively adds <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code>
amount of zero padding to both sizes of the input. This is set so that
when a <a class="reference internal" href="#torch.nn.Conv2d" title="torch.nn.Conv2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv2d</span></code></a> and a <a class="reference internal" href="#torch.nn.ConvTranspose2d" title="torch.nn.ConvTranspose2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">ConvTranspose2d</span></code></a>
are initialized with same parameters, they are inverses of each other in
regard to the input and output shapes. However, when <code class="docutils literal notranslate"><span class="pre">stride</span> <span class="pre">&gt;</span> <span class="pre">1</span></code>,
<a class="reference internal" href="#torch.nn.Conv2d" title="torch.nn.Conv2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv2d</span></code></a> maps multiple input shapes to the same output
shape. <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> is provided to resolve this ambiguity by
effectively increasing the calculated output shape on one side. Note
that <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> is only used to find output shape, but does
not actually add zero-padding to output.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels in the input image</p></li>
<li><p><strong>out_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels produced by the convolution</p></li>
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the convolving kernel</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Stride of the convolution. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> zero-padding
will be added to both sides of each dimension in the input. Default: 0</p></li>
<li><p><strong>output_padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Additional size added to one side
of each dimension in the output shape. Default: 0</p></li>
<li><p><strong>groups</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Number of blocked connections from input channels to output channels. Default: 1</p></li>
<li><p><strong>bias</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, adds a learnable bias to the output. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Spacing between kernel elements. Default: 1</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C_{out}, H_{out}, W_{out})\)</span> where</p></li>
</ul>
<div class="math">
\[H_{out} = (H_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] + \text{dilation}[0]
          \times (\text{kernel\_size}[0] - 1) + \text{output\_padding}[0] + 1

\]</div>
<div class="math">
\[W_{out} = (W_{in} - 1) \times \text{stride}[1] - 2 \times \text{padding}[1] + \text{dilation}[1]
          \times (\text{kernel\_size}[1] - 1) + \text{output\_padding}[1] + 1

\]</div>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~ConvTranspose2d.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape
<span class="math">\((\text{in\_channels}, \frac{\text{out\_channels}}{\text{groups}},\)</span>
<span class="math">\(\text{kernel\_size[0]}, \text{kernel\_size[1]})\)</span>.
The values of these weights are sampled from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{1}\text{kernel\_size}[i]}\)</span></p></li>
<li><p><strong>~ConvTranspose2d.bias</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable bias of the module of shape (out_channels)
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, then the values of these weights are
sampled from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{1}\text{kernel\_size}[i]}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With square kernels and equal stride</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConvTranspose2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># non-square kernels and unequal stride and with padding</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConvTranspose2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">padding</span><span class="o">=</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># exact output size can be also specified as an argument</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">12</span><span class="p">,</span> <span class="mi">12</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">downsample</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">upsample</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConvTranspose2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">h</span> <span class="o">=</span> <span class="n">downsample</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">h</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([1, 16, 6, 6])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">upsample</span><span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([1, 16, 12, 12])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="convtranspose3d">
<h3><span class="hidden-section">ConvTranspose3d</span><a class="headerlink" href="#convtranspose3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ConvTranspose3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ConvTranspose3d</code><span class="sig-paren">(</span><em>in_channels</em>, <em>out_channels</em>, <em>kernel_size</em>, <em>stride=1</em>, <em>padding=0</em>, <em>output_padding=0</em>, <em>groups=1</em>, <em>bias=True</em>, <em>dilation=1</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/conv.html#ConvTranspose3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ConvTranspose3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D transposed convolution operator over an input image composed of several input
planes.
The transposed convolution operator multiplies each input value element-wise by a learnable kernel,
and sums over the outputs from all input feature planes.</p>
<p>This module can be seen as the gradient of Conv3d with respect to its input.
It is also known as a fractionally-strided convolution or
a deconvolution (although it is not an actual deconvolution operation).</p>
<ul>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the cross-correlation.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both
sides for <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> number of points. See note
below for details.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> controls the additional size added to one side
of the output shape. See note below for details.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also known as the à trous algorithm.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code> controls the connections between inputs and outputs.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">out_channels</span></code> must both be divisible by
<code class="xref py py-attr docutils literal notranslate"><span class="pre">groups</span></code>. For example,</p>
<blockquote>
<div><ul class="simple">
<li><p>At groups=1, all inputs are convolved to all outputs.</p></li>
<li><p>At groups=2, the operation becomes equivalent to having two conv
layers side by side, each seeing half the input channels,
and producing half the output channels, and both subsequently
concatenated.</p></li>
<li><p>At groups= <code class="xref py py-attr docutils literal notranslate"><span class="pre">in_channels</span></code>, each input channel is convolved with
its own set of filters (of size
<span class="math">\(\left\lfloor\frac{out\_channels}{in\_channels}\right\rfloor\)</span>).</p></li>
</ul>
</div></blockquote>
</li>
</ul>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code>
can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the depth, height and width dimensions</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of three ints – in which case, the first <cite>int</cite> is used for the depth dimension,
the second <cite>int</cite> for the height dimension and the third <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Depending of the size of your kernel, several (of the last)
columns of the input might be lost, because it is a valid <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>,
and not a full <a class="reference external" href="https://en.wikipedia.org/wiki/Cross-correlation">cross-correlation</a>.
It is up to the user to add proper padding.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> argument effectively adds <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code>
amount of zero padding to both sizes of the input. This is set so that
when a <a class="reference internal" href="#torch.nn.Conv3d" title="torch.nn.Conv3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv3d</span></code></a> and a <a class="reference internal" href="#torch.nn.ConvTranspose3d" title="torch.nn.ConvTranspose3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">ConvTranspose3d</span></code></a>
are initialized with same parameters, they are inverses of each other in
regard to the input and output shapes. However, when <code class="docutils literal notranslate"><span class="pre">stride</span> <span class="pre">&gt;</span> <span class="pre">1</span></code>,
<a class="reference internal" href="#torch.nn.Conv3d" title="torch.nn.Conv3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv3d</span></code></a> maps multiple input shapes to the same output
shape. <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> is provided to resolve this ambiguity by
effectively increasing the calculated output shape on one side. Note
that <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_padding</span></code> is only used to find output shape, but does
not actually add zero-padding to output.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels in the input image</p></li>
<li><p><strong>out_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of channels produced by the convolution</p></li>
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the convolving kernel</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Stride of the convolution. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> zero-padding
will be added to both sides of each dimension in the input. Default: 0</p></li>
<li><p><strong>output_padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Additional size added to one side
of each dimension in the output shape. Default: 0</p></li>
<li><p><strong>groups</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Number of blocked connections from input channels to output channels. Default: 1</p></li>
<li><p><strong>bias</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, adds a learnable bias to the output. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – Spacing between kernel elements. Default: 1</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C_{in}, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C_{out}, D_{out}, H_{out}, W_{out})\)</span> where</p></li>
</ul>
<div class="math">
\[D_{out} = (D_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] + \text{dilation}[0]
          \times (\text{kernel\_size}[0] - 1) + \text{output\_padding}[0] + 1

\]</div>
<div class="math">
\[H_{out} = (H_{in} - 1) \times \text{stride}[1] - 2 \times \text{padding}[1] + \text{dilation}[1]
          \times (\text{kernel\_size}[1] - 1) + \text{output\_padding}[1] + 1

\]</div>
<div class="math">
\[W_{out} = (W_{in} - 1) \times \text{stride}[2] - 2 \times \text{padding}[2] + \text{dilation}[2]
          \times (\text{kernel\_size}[2] - 1) + \text{output\_padding}[2] + 1

\]</div>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~ConvTranspose3d.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape
<span class="math">\((\text{in\_channels}, \frac{\text{out\_channels}}{\text{groups}},\)</span>
<span class="math">\(\text{kernel\_size[0]}, \text{kernel\_size[1]}, \text{kernel\_size[2]})\)</span>.
The values of these weights are sampled from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{2}\text{kernel\_size}[i]}\)</span></p></li>
<li><p><strong>~ConvTranspose3d.bias</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable bias of the module of shape (out_channels)
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, then the values of these weights are
sampled from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{C_\text{in} * \prod_{i=0}^{2}\text{kernel\_size}[i]}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With square kernels and equal stride</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConvTranspose3d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># non-square kernels and unequal stride and with padding</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConvTranspose3d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">padding</span><span class="o">=</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="unfold">
<h3><span class="hidden-section">Unfold</span><a class="headerlink" href="#unfold" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Unfold">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Unfold</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>dilation=1</em>, <em>padding=0</em>, <em>stride=1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/fold.html#Unfold"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Unfold" title="Permalink to this definition">¶</a></dt>
<dd><p>Extracts sliding local blocks from a batched input tensor.</p>
<p>Consider an batched <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> tensor of shape <span class="math">\((N, C, *)\)</span>,
where <span class="math">\(N\)</span> is the batch dimension, <span class="math">\(C\)</span> is the channel dimension,
and <span class="math">\(*\)</span> represent arbitrary spatial dimensions. This operation flattens
each sliding <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>-sized block within the spatial dimensions
of <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> into a column (i.e., last dimension) of a 3-D <code class="xref py py-attr docutils literal notranslate"><span class="pre">output</span></code>
tensor of shape <span class="math">\((N, C \times \prod(\text{kernel\_size}), L)\)</span>, where
<span class="math">\(C \times \prod(\text{kernel\_size})\)</span> is the total number of values
within each block (a block has <span class="math">\(\prod(\text{kernel\_size})\)</span> spatial
locations each containing a <span class="math">\(C\)</span>-channeled vector), and <span class="math">\(L\)</span> is
the total number of such blocks:</p>
<div class="math">
\[L = \prod_d \left\lfloor\frac{\text{spatial\_size}[d] + 2 \times \text{padding}[d] %
    - \text{dilation}[d] \times (\text{kernel\_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor,

\]</div>
<p>where <span class="math">\(\text{spatial\_size}\)</span> is formed by the spatial dimensions
of <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> (<span class="math">\(*\)</span> above), and <span class="math">\(d\)</span> is over all spatial
dimensions.</p>
<p>Therefore, indexing <code class="xref py py-attr docutils literal notranslate"><span class="pre">output</span></code> at the last dimension (column dimension)
gives all values within a certain block.</p>
<p>The <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> arguments specify
how the sliding blocks are retrieved.</p>
<ul class="simple">
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the sliding blocks.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both
sides for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points for each dimension before
reshaping.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also known as the à trous algorithm.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
</ul>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the sliding blocks</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – the stride of the sliding blocks in the input
spatial dimensions. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – implicit zero padding to be added on
both sides of input. Default: 0</p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – a parameter that controls the
stride of elements within the
neighborhood. Default: 1</p></li>
</ul>
</dd>
</dl>
<ul class="simple">
<li><p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> or
<code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> is an int or a tuple of length 1, their values will be
replicated across all spatial dimensions.</p></li>
<li><p>For the case of two input spatial dimensions this operation is sometimes
called <code class="docutils literal notranslate"><span class="pre">im2col</span></code>.</p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.Fold" title="torch.nn.Fold"><code class="xref py py-class docutils literal notranslate"><span class="pre">Fold</span></code></a> calculates each combined value in the resulting
large tensor by summing all values from all containing blocks.
<a class="reference internal" href="#torch.nn.Unfold" title="torch.nn.Unfold"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unfold</span></code></a> extracts the values in the local blocks by
copying from the large tensor. So, if the blocks overlap, they are not
inverses of each other.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Currently, only 4-D input tensors (batched image-like tensors) are
supported.</p>
</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, *)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C \times \prod(\text{kernel\_size}), L)\)</span> as described above</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">unfold</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Unfold</span><span class="p">(</span><span class="n">kernel_size</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">unfold</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># each patch contains 30 values (2x3=6 vectors, each of 5 channels)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># 4 blocks (2x3 kernels) in total in the 3x4 input</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([2, 30, 4])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Convolution is equivalent with Unfold + Matrix Multiplication + Fold (or view to output shape)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">12</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inp_unf</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">unfold</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out_unf</span> <span class="o">=</span> <span class="n">inp_unf</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">matmul</span><span class="p">(</span><span class="n">w</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="n">w</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span> <span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">t</span><span class="p">())</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">fold</span><span class="p">(</span><span class="n">out_unf</span><span class="p">,</span> <span class="p">(</span><span class="mi">7</span><span class="p">,</span> <span class="mi">8</span><span class="p">),</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># or equivalently (and avoiding a copy),</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># out = out_unf.view(1, 2, 7, 8)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">conv2d</span><span class="p">(</span><span class="n">inp</span><span class="p">,</span> <span class="n">w</span><span class="p">)</span> <span class="o">-</span> <span class="n">out</span><span class="p">)</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">()</span>
<span class="go">tensor(1.9073e-06)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="fold">
<h3><span class="hidden-section">Fold</span><a class="headerlink" href="#fold" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Fold">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Fold</code><span class="sig-paren">(</span><em>output_size</em>, <em>kernel_size</em>, <em>dilation=1</em>, <em>padding=0</em>, <em>stride=1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/fold.html#Fold"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Fold" title="Permalink to this definition">¶</a></dt>
<dd><p>Combines an array of sliding local blocks into a large containing
tensor.</p>
<p>Consider a batched <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> tensor containing sliding local blocks,
e.g., patches of images, of shape <span class="math">\((N, C \times  \prod(\text{kernel\_size}), L)\)</span>,
where <span class="math">\(N\)</span> is batch dimension, <span class="math">\(C \times \prod(\text{kernel\_size})\)</span>
is the number of values within a block (a block has <span class="math">\(\prod(\text{kernel\_size})\)</span>
spatial locations each containing a <span class="math">\(C\)</span>-channeled vector), and
<span class="math">\(L\)</span> is the total number of blocks. (This is exactly the
same specification as the output shape of <a class="reference internal" href="#torch.nn.Unfold" title="torch.nn.Unfold"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unfold</span></code></a>.) This
operation combines these local blocks into the large <code class="xref py py-attr docutils literal notranslate"><span class="pre">output</span></code> tensor
of shape <span class="math">\((N, C, \text{output\_size}[0], \text{output\_size}[1], \dots)\)</span>
by summing the overlapping values. Similar to <a class="reference internal" href="#torch.nn.Unfold" title="torch.nn.Unfold"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unfold</span></code></a>, the
arguments must satisfy</p>
<div class="math">
\[L = \prod_d \left\lfloor\frac{\text{output\_size}[d] + 2 \times \text{padding}[d] %
    - \text{dilation}[d] \times (\text{kernel\_size}[d] - 1) - 1}{\text{stride}[d]} + 1\right\rfloor,

\]</div>
<p>where <span class="math">\(d\)</span> is over all spatial dimensions.</p>
<ul class="simple">
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code> describes the spatial shape of the large containing
tensor of the sliding local blocks. It is useful to resolve the ambiguity
when multiple input shapes map to same number of sliding blocks, e.g.,
with <code class="docutils literal notranslate"><span class="pre">stride</span> <span class="pre">&gt;</span> <span class="pre">0</span></code>.</p></li>
</ul>
<p>The <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> arguments specify
how the sliding blocks are retrieved.</p>
<ul class="simple">
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> controls the stride for the sliding blocks.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> controls the amount of implicit zero-paddings on both
sides for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points for each dimension before
reshaping.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points; also known as the à trous algorithm.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p></li>
</ul>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the shape of the spatial dimensions of the
output (i.e., <code class="docutils literal notranslate"><span class="pre">output.sizes()[2:]</span></code>)</p></li>
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the sliding blocks</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the stride of the sliding blocks in the input
spatial dimensions. Default: 1</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – implicit zero padding to be added on
both sides of input. Default: 0</p></li>
<li><p><strong>dilation</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a><em>, </em><em>optional</em>) – a parameter that controls the
stride of elements within the
neighborhood. Default: 1</p></li>
</ul>
</dd>
</dl>
<ul class="simple">
<li><p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code>,
<code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> or <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> is an int or a tuple of length 1 then
their values will be replicated across all spatial dimensions.</p></li>
<li><p>For the case of two output spatial dimensions this operation is sometimes
called <code class="docutils literal notranslate"><span class="pre">col2im</span></code>.</p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.Fold" title="torch.nn.Fold"><code class="xref py py-class docutils literal notranslate"><span class="pre">Fold</span></code></a> calculates each combined value in the resulting
large tensor by summing all values from all containing blocks.
<a class="reference internal" href="#torch.nn.Unfold" title="torch.nn.Unfold"><code class="xref py py-class docutils literal notranslate"><span class="pre">Unfold</span></code></a> extracts the values in the local blocks by
copying from the large tensor. So, if the blocks overlap, they are not
inverses of each other.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Currently, only 4-D output tensors (batched image-like tensors) are
supported.</p>
</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C \times \prod(\text{kernel\_size}), L)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, \text{output\_size}[0], \text{output\_size}[1], \dots)\)</span> as described above</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">fold</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Fold</span><span class="p">(</span><span class="n">output_size</span><span class="o">=</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span> <span class="n">kernel_size</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">3</span> <span class="o">*</span> <span class="mi">2</span> <span class="o">*</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">12</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">fold</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([1, 3, 4, 5])</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="pooling-layers">
<h2>Pooling layers<a class="headerlink" href="#pooling-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="maxpool1d">
<h3><span class="hidden-section">MaxPool1d</span><a class="headerlink" href="#maxpool1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MaxPool1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MaxPool1d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>return_indices=False</em>, <em>ceil_mode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#MaxPool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MaxPool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D max pooling over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size <span class="math">\((N, C, L)\)</span>
and output <span class="math">\((N, C, L_{out})\)</span> can be precisely described as:</p>
<div class="math">
\[out(N_i, C_j, k) = \max_{m=0, \ldots, \text{kernel\_size} - 1}
        input(N_i, C_j, stride \times k + m)

\]</div>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> is non-zero, then the input is implicitly zero-padded on both sides
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points. <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window to take a max over</p></li>
<li><p><strong>stride</strong> – the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero padding to be added on both sides</p></li>
<li><p><strong>dilation</strong> – a parameter that controls the stride of elements in the window</p></li>
<li><p><strong>return_indices</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, will return the max indices along with the outputs.
Useful for <a class="reference internal" href="#torch.nn.MaxUnpool1d" title="torch.nn.MaxUnpool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.MaxUnpool1d</span></code></a> later</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, L_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, L_{out})\)</span>, where</p>
<div class="math">
\[L_{out} = \left\lfloor \frac{L_{in} + 2 \times \text{padding} - \text{dilation}
      \times (\text{kernel\_size} - 1) - 1}{\text{stride}} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool1d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="maxpool2d">
<h3><span class="hidden-section">MaxPool2d</span><a class="headerlink" href="#maxpool2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MaxPool2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MaxPool2d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>return_indices=False</em>, <em>ceil_mode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#MaxPool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MaxPool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D max pooling over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size <span class="math">\((N, C, H, W)\)</span>,
output <span class="math">\((N, C, H_{out}, W_{out})\)</span> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> <span class="math">\((kH, kW)\)</span>
can be precisely described as:</p>
<div class="math">
\[\begin{aligned}
    out(N_i, C_j, h, w) ={} & \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} \\
                            & \text{input}(N_i, C_j, \text{stride[0]} \times h + m,
                                           \text{stride[1]} \times w + n)
\end{aligned}

\]</div>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> is non-zero, then the input is implicitly zero-padded on both sides
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points. <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the height and width dimension</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of two ints – in which case, the first <cite>int</cite> is used for the height dimension,
and the second <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window to take a max over</p></li>
<li><p><strong>stride</strong> – the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero padding to be added on both sides</p></li>
<li><p><strong>dilation</strong> – a parameter that controls the stride of elements in the window</p></li>
<li><p><strong>return_indices</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, will return the max indices along with the outputs.
Useful for <a class="reference internal" href="#torch.nn.MaxUnpool2d" title="torch.nn.MaxUnpool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.MaxUnpool2d</span></code></a> later</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span>, where</p>
<div class="math">
\[H_{out} = \left\lfloor\frac{H_{in} + 2 * \text{padding[0]} - \text{dilation[0]}
      \times (\text{kernel\_size[0]} - 1) - 1}{\text{stride[0]}} + 1\right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor\frac{W_{in} + 2 * \text{padding[1]} - \text{dilation[1]}
      \times (\text{kernel\_size[1]} - 1) - 1}{\text{stride[1]}} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool2d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of non-square window</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool2d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="maxpool3d">
<h3><span class="hidden-section">MaxPool3d</span><a class="headerlink" href="#maxpool3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MaxPool3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MaxPool3d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>return_indices=False</em>, <em>ceil_mode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#MaxPool3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MaxPool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D max pooling over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size <span class="math">\((N, C, D, H, W)\)</span>,
output <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> <span class="math">\((kD, kH, kW)\)</span>
can be precisely described as:</p>
<div class="math">
\[\begin{aligned}
    \text{out}(N_i, C_j, d, h, w) ={} & \max_{k=0, \ldots, kD-1} \max_{m=0, \ldots, kH-1} \max_{n=0, \ldots, kW-1} \\
                                      & \text{input}(N_i, C_j, \text{stride[0]} \times d + k,
                                                     \text{stride[1]} \times h + m, \text{stride[2]} \times w + n)
\end{aligned}

\]</div>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> is non-zero, then the input is implicitly zero-padded on both sides
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points. <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> controls the spacing between the kernel points.
It is harder to describe, but this <a class="reference external" href="https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md">link</a> has a nice visualization of what <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> does.</p>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">dilation</span></code> can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the depth, height and width dimension</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of three ints – in which case, the first <cite>int</cite> is used for the depth dimension,
the second <cite>int</cite> for the height dimension and the third <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window to take a max over</p></li>
<li><p><strong>stride</strong> – the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero padding to be added on all three sides</p></li>
<li><p><strong>dilation</strong> – a parameter that controls the stride of elements in the window</p></li>
<li><p><strong>return_indices</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, will return the max indices along with the outputs.
Useful for <a class="reference internal" href="#torch.nn.MaxUnpool3d" title="torch.nn.MaxUnpool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.MaxUnpool3d</span></code></a> later</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span>, where</p>
<div class="math">
\[D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] - \text{dilation}[0] \times
  (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor

\]</div>
<div class="math">
\[H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] - \text{dilation}[1] \times
  (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] - \text{dilation}[2] \times
  (\text{kernel\_size}[2] - 1) - 1}{\text{stride}[2]} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool3d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of non-square window</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool3d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span><span class="mi">44</span><span class="p">,</span> <span class="mi">31</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="maxunpool1d">
<h3><span class="hidden-section">MaxUnpool1d</span><a class="headerlink" href="#maxunpool1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MaxUnpool1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MaxUnpool1d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#MaxUnpool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MaxUnpool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes a partial inverse of <a class="reference internal" href="#torch.nn.MaxPool1d" title="torch.nn.MaxPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool1d</span></code></a>.</p>
<p><a class="reference internal" href="#torch.nn.MaxPool1d" title="torch.nn.MaxPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool1d</span></code></a> is not fully invertible, since the non-maximal values are lost.</p>
<p><a class="reference internal" href="#torch.nn.MaxUnpool1d" title="torch.nn.MaxUnpool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxUnpool1d</span></code></a> takes in as input the output of <a class="reference internal" href="#torch.nn.MaxPool1d" title="torch.nn.MaxPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool1d</span></code></a>
including the indices of the maximal values and computes a partial inverse
in which all non-maximal values are set to zero.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.MaxPool1d" title="torch.nn.MaxPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool1d</span></code></a> can map several input sizes to the same output
sizes. Hence, the inversion process can get ambiguous.
To accommodate this, you can provide the needed output size
as an additional argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code> in the forward call.
See the Inputs and Example below.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the max pooling window.</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Stride of the max pooling window.
It is set to <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> by default.</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Padding that was added to the input</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Inputs:</dt><dd><ul class="simple">
<li><p><cite>input</cite>: the input Tensor to invert</p></li>
<li><p><cite>indices</cite>: the indices given out by <a class="reference internal" href="#torch.nn.MaxPool1d" title="torch.nn.MaxPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool1d</span></code></a></p></li>
<li><p><cite>output_size</cite> (optional): the targeted output size</p></li>
</ul>
</dd>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out})\)</span>, where</p>
<div class="math">
\[H_{out} = (H_{in} - 1) \times \text{stride}[0] - 2 \times \text{padding}[0] + \text{kernel\_size}[0]

\]</div>
<p>or as given by <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code> in the call operator</p>
</li>
</ul>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">pool</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool1d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">return_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxUnpool1d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[[</span><span class="mf">1.</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">8</span><span class="p">]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span> <span class="o">=</span> <span class="n">pool</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span><span class="p">)</span>
<span class="go">tensor([[[ 0.,  2.,  0.,  4.,  0.,  6.,  0., 8.]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Example showcasing the use of output_size</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[[</span><span class="mf">1.</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">9</span><span class="p">]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span> <span class="o">=</span> <span class="n">pool</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">tensor([[[ 0.,  2.,  0.,  4.,  0.,  6.,  0., 8.,  0.]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span><span class="p">)</span>
<span class="go">tensor([[[ 0.,  2.,  0.,  4.,  0.,  6.,  0., 8.]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="maxunpool2d">
<h3><span class="hidden-section">MaxUnpool2d</span><a class="headerlink" href="#maxunpool2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MaxUnpool2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MaxUnpool2d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#MaxUnpool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MaxUnpool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes a partial inverse of <a class="reference internal" href="#torch.nn.MaxPool2d" title="torch.nn.MaxPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool2d</span></code></a>.</p>
<p><a class="reference internal" href="#torch.nn.MaxPool2d" title="torch.nn.MaxPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool2d</span></code></a> is not fully invertible, since the non-maximal values are lost.</p>
<p><a class="reference internal" href="#torch.nn.MaxUnpool2d" title="torch.nn.MaxUnpool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxUnpool2d</span></code></a> takes in as input the output of <a class="reference internal" href="#torch.nn.MaxPool2d" title="torch.nn.MaxPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool2d</span></code></a>
including the indices of the maximal values and computes a partial inverse
in which all non-maximal values are set to zero.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.MaxPool2d" title="torch.nn.MaxPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool2d</span></code></a> can map several input sizes to the same output
sizes. Hence, the inversion process can get ambiguous.
To accommodate this, you can provide the needed output size
as an additional argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code> in the forward call.
See the Inputs and Example below.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the max pooling window.</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Stride of the max pooling window.
It is set to <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> by default.</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Padding that was added to the input</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Inputs:</dt><dd><ul class="simple">
<li><p><cite>input</cite>: the input Tensor to invert</p></li>
<li><p><cite>indices</cite>: the indices given out by <a class="reference internal" href="#torch.nn.MaxPool2d" title="torch.nn.MaxPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool2d</span></code></a></p></li>
<li><p><cite>output_size</cite> (optional): the targeted output size</p></li>
</ul>
</dd>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span>, where</p>
<div class="math">
\[H_{out} = (H_{in} - 1) \times \text{stride[0]} - 2 \times \text{padding[0]} + \text{kernel\_size[0]}

\]</div>
<div class="math">
\[W_{out} = (W_{in} - 1) \times \text{stride[1]} - 2 \times \text{padding[1]} + \text{kernel\_size[1]}

\]</div>
<p>or as given by <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code> in the call operator</p>
</li>
</ul>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">pool</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool2d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">return_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxUnpool2d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[[[</span> <span class="mf">1.</span><span class="p">,</span>  <span class="mi">2</span><span class="p">,</span>  <span class="mi">3</span><span class="p">,</span>  <span class="mi">4</span><span class="p">],</span>
<span class="go">                            [ 5,  6,  7,  8],</span>
<span class="go">                            [ 9, 10, 11, 12],</span>
<span class="go">                            [13, 14, 15, 16]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span> <span class="o">=</span> <span class="n">pool</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span><span class="p">)</span>
<span class="go">tensor([[[[  0.,   0.,   0.,   0.],</span>
<span class="go">          [  0.,   6.,   0.,   8.],</span>
<span class="go">          [  0.,   0.,   0.,   0.],</span>
<span class="go">          [  0.,  14.,   0.,  16.]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># specify a different output size than input size</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">Size</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">]))</span>
<span class="go">tensor([[[[  0.,   0.,   0.,   0.,   0.],</span>
<span class="go">          [  6.,   0.,   8.,   0.,   0.],</span>
<span class="go">          [  0.,   0.,   0.,  14.,   0.],</span>
<span class="go">          [ 16.,   0.,   0.,   0.,   0.],</span>
<span class="go">          [  0.,   0.,   0.,   0.,   0.]]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="maxunpool3d">
<h3><span class="hidden-section">MaxUnpool3d</span><a class="headerlink" href="#maxunpool3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MaxUnpool3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MaxUnpool3d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#MaxUnpool3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MaxUnpool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes a partial inverse of <a class="reference internal" href="#torch.nn.MaxPool3d" title="torch.nn.MaxPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool3d</span></code></a>.</p>
<p><a class="reference internal" href="#torch.nn.MaxPool3d" title="torch.nn.MaxPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool3d</span></code></a> is not fully invertible, since the non-maximal values are lost.
<a class="reference internal" href="#torch.nn.MaxUnpool3d" title="torch.nn.MaxUnpool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxUnpool3d</span></code></a> takes in as input the output of <a class="reference internal" href="#torch.nn.MaxPool3d" title="torch.nn.MaxPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool3d</span></code></a>
including the indices of the maximal values and computes a partial inverse
in which all non-maximal values are set to zero.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.MaxPool3d" title="torch.nn.MaxPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool3d</span></code></a> can map several input sizes to the same output
sizes. Hence, the inversion process can get ambiguous.
To accommodate this, you can provide the needed output size
as an additional argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code> in the forward call.
See the Inputs section below.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Size of the max pooling window.</p></li>
<li><p><strong>stride</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Stride of the max pooling window.
It is set to <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> by default.</p></li>
<li><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – Padding that was added to the input</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Inputs:</dt><dd><ul class="simple">
<li><p><cite>input</cite>: the input Tensor to invert</p></li>
<li><p><cite>indices</cite>: the indices given out by <a class="reference internal" href="#torch.nn.MaxPool3d" title="torch.nn.MaxPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool3d</span></code></a></p></li>
<li><p><cite>output_size</cite> (optional): the targeted output size</p></li>
</ul>
</dd>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span>, where</p>
<div class="math">
\[D_{out} = (D_{in} - 1) \times \text{stride[0]} - 2 \times \text{padding[0]} + \text{kernel\_size[0]}

\]</div>
<div class="math">
\[H_{out} = (H_{in} - 1) \times \text{stride[1]} - 2 \times \text{padding[1]} + \text{kernel\_size[1]}

\]</div>
<div class="math">
\[W_{out} = (W_{in} - 1) \times \text{stride[2]} - 2 \times \text{padding[2]} + \text{kernel\_size[2]}

\]</div>
<p>or as given by <code class="xref py py-attr docutils literal notranslate"><span class="pre">output_size</span></code> in the call operator</p>
</li>
</ul>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pool</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool3d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">return_indices</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpool</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxUnpool3d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span> <span class="o">=</span> <span class="n">pool</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">51</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">15</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpooled_output</span> <span class="o">=</span> <span class="n">unpool</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">indices</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">unpooled_output</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([20, 16, 51, 33, 15])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="avgpool1d">
<h3><span class="hidden-section">AvgPool1d</span><a class="headerlink" href="#avgpool1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AvgPool1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AvgPool1d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>ceil_mode=False</em>, <em>count_include_pad=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AvgPool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AvgPool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D average pooling over an input signal composed of several
input planes.</p>
<p>In the simplest case, the output value of the layer with input size <span class="math">\((N, C, L)\)</span>,
output <span class="math">\((N, C, L_{out})\)</span> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> <span class="math">\(k\)</span>
can be precisely described as:</p>
<div class="math">
\[\text{out}(N_i, C_j, l) = \frac{1}{k} \sum_{m=0}^{k-1}
                       \text{input}(N_i, C_j, \text{stride} \times l + m)\]</div>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> is non-zero, then the input is implicitly zero-padded on both sides
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points.</p>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> can each be
an <code class="docutils literal notranslate"><span class="pre">int</span></code> or a one-element tuple.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window</p></li>
<li><p><strong>stride</strong> – the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero padding to be added on both sides</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
<li><p><strong>count_include_pad</strong> – when True, will include the zero-padding in the averaging calculation</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, L_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, L_{out})\)</span>, where</p>
<div class="math">
\[L_{out} = \left\lfloor \frac{L_{in} +
2 \times \text{padding} - \text{kernel\_size}}{\text{stride}} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool with window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AvgPool1d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[[</span><span class="mf">1.</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">,</span><span class="mi">6</span><span class="p">,</span><span class="mi">7</span><span class="p">]]]))</span>
<span class="go">tensor([[[ 2.,  4.,  6.]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="avgpool2d">
<h3><span class="hidden-section">AvgPool2d</span><a class="headerlink" href="#avgpool2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AvgPool2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AvgPool2d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>ceil_mode=False</em>, <em>count_include_pad=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AvgPool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AvgPool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D average pooling over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size <span class="math">\((N, C, H, W)\)</span>,
output <span class="math">\((N, C, H_{out}, W_{out})\)</span> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> <span class="math">\((kH, kW)\)</span>
can be precisely described as:</p>
<div class="math">
\[out(N_i, C_j, h, w)  = \frac{1}{kH * kW} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1}
                       input(N_i, C_j, stride[0] \times h + m, stride[1] \times w + n)\]</div>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> is non-zero, then the input is implicitly zero-padded on both sides
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points.</p>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the height and width dimension</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of two ints – in which case, the first <cite>int</cite> is used for the height dimension,
and the second <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window</p></li>
<li><p><strong>stride</strong> – the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero padding to be added on both sides</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
<li><p><strong>count_include_pad</strong> – when True, will include the zero-padding in the averaging calculation</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span>, where</p>
<div class="math">
\[H_{out} = \left\lfloor\frac{H_{in}  + 2 \times \text{padding}[0] -
  \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor\frac{W_{in}  + 2 \times \text{padding}[1] -
  \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AvgPool2d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of non-square window</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AvgPool2d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="avgpool3d">
<h3><span class="hidden-section">AvgPool3d</span><a class="headerlink" href="#avgpool3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AvgPool3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AvgPool3d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>ceil_mode=False</em>, <em>count_include_pad=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AvgPool3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AvgPool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D average pooling over an input signal composed of several input
planes.</p>
<p>In the simplest case, the output value of the layer with input size <span class="math">\((N, C, D, H, W)\)</span>,
output <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code> <span class="math">\((kD, kH, kW)\)</span>
can be precisely described as:</p>
<div class="math">
\[\begin{aligned}
    \text{out}(N_i, C_j, d, h, w) ={} & \sum_{k=0}^{kD-1} \sum_{m=0}^{kH-1} \sum_{n=0}^{kW-1} \\
                                      & \frac{\text{input}(N_i, C_j, \text{stride}[0] \times d + k,
                                              \text{stride}[1] \times h + m, \text{stride}[2] \times w + n)}
                                             {kD \times kH \times kW}
\end{aligned}

\]</div>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> is non-zero, then the input is implicitly zero-padded on all three sides
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding</span></code> number of points.</p>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the depth, height and width dimension</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of three ints – in which case, the first <cite>int</cite> is used for the depth dimension,
the second <cite>int</cite> for the height dimension and the third <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window</p></li>
<li><p><strong>stride</strong> – the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero padding to be added on all three sides</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
<li><p><strong>count_include_pad</strong> – when True, will include the zero-padding in the averaging calculation</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span>, where</p>
<div class="math">
\[D_{out} = \left\lfloor\frac{D_{in} + 2 \times \text{padding}[0] -
      \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor

\]</div>
<div class="math">
\[H_{out} = \left\lfloor\frac{H_{in} + 2 \times \text{padding}[1] -
      \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor\frac{W_{in} + 2 \times \text{padding}[2] -
      \text{kernel\_size}[2]}{\text{stride}[2]} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AvgPool3d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of non-square window</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AvgPool3d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span><span class="mi">44</span><span class="p">,</span> <span class="mi">31</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="fractionalmaxpool2d">
<h3><span class="hidden-section">FractionalMaxPool2d</span><a class="headerlink" href="#fractionalmaxpool2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.FractionalMaxPool2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">FractionalMaxPool2d</code><span class="sig-paren">(</span><em>kernel_size</em>, <em>output_size=None</em>, <em>output_ratio=None</em>, <em>return_indices=False</em>, <em>_random_samples=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#FractionalMaxPool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.FractionalMaxPool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D fractional max pooling over an input signal composed of several input planes.</p>
<p>Fractional MaxPooling is described in detail in the paper <a class="reference external" href="http://arxiv.org/abs/1412.6071">Fractional MaxPooling</a> by Ben Graham</p>
<p>The max-pooling operation is applied in <span class="math">\(kH \times kW\)</span> regions by a stochastic
step size determined by the target output size.
The number of output features is equal to the number of input planes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window to take a max over.
Can be a single number k (for a square kernel of k x k) or a tuple <cite>(kh, kw)</cite></p></li>
<li><p><strong>output_size</strong> – the target output size of the image of the form <cite>oH x oW</cite>.
Can be a tuple <cite>(oH, oW)</cite> or a single number oH for a square image <cite>oH x oH</cite></p></li>
<li><p><strong>output_ratio</strong> – If one wants to have an output size as a ratio of the input size, this option can be given.
This has to be a number or tuple in the range (0, 1)</p></li>
<li><p><strong>return_indices</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, will return the indices along with the outputs.
Useful to pass to <code class="xref py py-meth docutils literal notranslate"><span class="pre">nn.MaxUnpool2d()</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window of size=3, and target output size 13x12</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">FractionalMaxPool2d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">output_size</span><span class="o">=</span><span class="p">(</span><span class="mi">13</span><span class="p">,</span> <span class="mi">12</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window and target output size being half of input image size</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">FractionalMaxPool2d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">output_ratio</span><span class="o">=</span><span class="p">(</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="lppool1d">
<h3><span class="hidden-section">LPPool1d</span><a class="headerlink" href="#lppool1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LPPool1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LPPool1d</code><span class="sig-paren">(</span><em>norm_type</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>ceil_mode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#LPPool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LPPool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D power-average pooling over an input signal composed of several input
planes.</p>
<p>On each window, the function computed is:</p>
<div class="math">
\[f(X) = \sqrt[p]{\sum_{x \in X} x^{p}}

\]</div>
<ul class="simple">
<li><p>At p = <span class="math">\(\infty\)</span>, one gets Max Pooling</p></li>
<li><p>At p = 1, one gets Sum Pooling (which is proportional to Average Pooling)</p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If the sum to the power of <cite>p</cite> is zero, the gradient of this function is
not defined. This implementation will set the gradient to zero in this case.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – a single int, the size of the window</p></li>
<li><p><strong>stride</strong> – a single int, the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, L_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, L_{out})\)</span>, where</p>
<div class="math">
\[L_{out} = \left\lfloor\frac{L_{in} +
2 \times \text{padding} - \text{kernel\_size}}{\text{stride}} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
<dt>Examples::</dt><dd><div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># power-2 pool of window of length 3, with stride 2.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LPPool1d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="lppool2d">
<h3><span class="hidden-section">LPPool2d</span><a class="headerlink" href="#lppool2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LPPool2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LPPool2d</code><span class="sig-paren">(</span><em>norm_type</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>ceil_mode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#LPPool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LPPool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D power-average pooling over an input signal composed of several input
planes.</p>
<p>On each window, the function computed is:</p>
<div class="math">
\[f(X) = \sqrt[p]{\sum_{x \in X} x^{p}}

\]</div>
<ul class="simple">
<li><p>At p = <span class="math">\(\infty\)</span>, one gets Max Pooling</p></li>
<li><p>At p = 1, one gets Sum Pooling (which is proportional to average pooling)</p></li>
</ul>
<p>The parameters <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">stride</span></code> can either be:</p>
<blockquote>
<div><ul class="simple">
<li><p>a single <code class="docutils literal notranslate"><span class="pre">int</span></code> – in which case the same value is used for the height and width dimension</p></li>
<li><p>a <code class="docutils literal notranslate"><span class="pre">tuple</span></code> of two ints – in which case, the first <cite>int</cite> is used for the height dimension,
and the second <cite>int</cite> for the width dimension</p></li>
</ul>
</div></blockquote>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If the sum to the power of <cite>p</cite> is zero, the gradient of this function is
not defined. This implementation will set the gradient to zero in this case.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>kernel_size</strong> – the size of the window</p></li>
<li><p><strong>stride</strong> – the stride of the window. Default value is <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the output shape</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span>, where</p>
<div class="math">
\[H_{out} = \left\lfloor\frac{H_{in}  + 2 \times \text{padding}[0] - \text{dilation}[0] \times
      (\text{kernel\_size}[0] - 1) - 1}{\text{stride}[0]} + 1\right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor\frac{W_{in}  + 2 \times \text{padding}[1] - \text{dilation}[1] \times
      (\text{kernel\_size}[1] - 1) - 1}{\text{stride}[1]} + 1\right\rfloor

\]</div>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># power-2 pool of square window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LPPool2d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of non-square window of power 1.2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LPPool2d</span><span class="p">(</span><span class="mf">1.2</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">stride</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="adaptivemaxpool1d">
<h3><span class="hidden-section">AdaptiveMaxPool1d</span><a class="headerlink" href="#adaptivemaxpool1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AdaptiveMaxPool1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AdaptiveMaxPool1d</code><span class="sig-paren">(</span><em>output_size</em>, <em>return_indices=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AdaptiveMaxPool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveMaxPool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D adaptive max pooling over an input signal composed of several input planes.</p>
<p>The output size is H, for any input size.
The number of output features is equal to the number of input planes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output_size</strong> – the target output size H</p></li>
<li><p><strong>return_indices</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, will return the indices along with the outputs.
Useful to pass to nn.MaxUnpool1d. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 5</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool1d</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="adaptivemaxpool2d">
<h3><span class="hidden-section">AdaptiveMaxPool2d</span><a class="headerlink" href="#adaptivemaxpool2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AdaptiveMaxPool2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AdaptiveMaxPool2d</code><span class="sig-paren">(</span><em>output_size</em>, <em>return_indices=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AdaptiveMaxPool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveMaxPool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D adaptive max pooling over an input signal composed of several input planes.</p>
<p>The output is of size H x W, for any input size.
The number of output features is equal to the number of input planes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output_size</strong> – the target output size of the image of the form H x W.
Can be a tuple (H, W) or a single H for a square image H x H.
H and W can be either a <code class="docutils literal notranslate"><span class="pre">int</span></code>, or <code class="docutils literal notranslate"><span class="pre">None</span></code> which means the size will
be the same as that of the input.</p></li>
<li><p><strong>return_indices</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, will return the indices along with the outputs.
Useful to pass to nn.MaxUnpool2d. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 5x7</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool2d</span><span class="p">((</span><span class="mi">5</span><span class="p">,</span><span class="mi">7</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 7x7 (square)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool2d</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 10x7</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool2d</span><span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="mi">7</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="adaptivemaxpool3d">
<h3><span class="hidden-section">AdaptiveMaxPool3d</span><a class="headerlink" href="#adaptivemaxpool3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AdaptiveMaxPool3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AdaptiveMaxPool3d</code><span class="sig-paren">(</span><em>output_size</em>, <em>return_indices=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AdaptiveMaxPool3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveMaxPool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D adaptive max pooling over an input signal composed of several input planes.</p>
<p>The output is of size D x H x W, for any input size.
The number of output features is equal to the number of input planes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output_size</strong> – the target output size of the image of the form D x H x W.
Can be a tuple (D, H, W) or a single D for a cube D x D x D.
D, H and W can be either a <code class="docutils literal notranslate"><span class="pre">int</span></code>, or <code class="docutils literal notranslate"><span class="pre">None</span></code> which means the size will
be the same as that of the input.</p></li>
<li><p><strong>return_indices</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, will return the indices along with the outputs.
Useful to pass to nn.MaxUnpool3d. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 5x7x9</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool3d</span><span class="p">((</span><span class="mi">5</span><span class="p">,</span><span class="mi">7</span><span class="p">,</span><span class="mi">9</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 7x7x7 (cube)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool3d</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 7x9x8</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool3d</span><span class="p">((</span><span class="mi">7</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="adaptiveavgpool1d">
<h3><span class="hidden-section">AdaptiveAvgPool1d</span><a class="headerlink" href="#adaptiveavgpool1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AdaptiveAvgPool1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AdaptiveAvgPool1d</code><span class="sig-paren">(</span><em>output_size</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AdaptiveAvgPool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveAvgPool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D adaptive average pooling over an input signal composed of several input planes.</p>
<p>The output size is H, for any input size.
The number of output features is equal to the number of input planes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>output_size</strong> – the target output size H</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 5</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveAvgPool1d</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="adaptiveavgpool2d">
<h3><span class="hidden-section">AdaptiveAvgPool2d</span><a class="headerlink" href="#adaptiveavgpool2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AdaptiveAvgPool2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AdaptiveAvgPool2d</code><span class="sig-paren">(</span><em>output_size</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AdaptiveAvgPool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveAvgPool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D adaptive average pooling over an input signal composed of several input planes.</p>
<p>The output is of size H x W, for any input size.
The number of output features is equal to the number of input planes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>output_size</strong> – the target output size of the image of the form H x W.
Can be a tuple (H, W) or a single H for a square image H x H.
H and W can be either a <code class="docutils literal notranslate"><span class="pre">int</span></code>, or <code class="docutils literal notranslate"><span class="pre">None</span></code> which means the size will
be the same as that of the input.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 5x7</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveAvgPool2d</span><span class="p">((</span><span class="mi">5</span><span class="p">,</span><span class="mi">7</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 7x7 (square)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveAvgPool2d</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 10x7</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool2d</span><span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="mi">7</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="adaptiveavgpool3d">
<h3><span class="hidden-section">AdaptiveAvgPool3d</span><a class="headerlink" href="#adaptiveavgpool3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AdaptiveAvgPool3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AdaptiveAvgPool3d</code><span class="sig-paren">(</span><em>output_size</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pooling.html#AdaptiveAvgPool3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveAvgPool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D adaptive average pooling over an input signal composed of several input planes.</p>
<p>The output is of size D x H x W, for any input size.
The number of output features is equal to the number of input planes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>output_size</strong> – the target output size of the form D x H x W.
Can be a tuple (D, H, W) or a single number D for a cube D x D x D.
D, H and W can be either a <code class="docutils literal notranslate"><span class="pre">int</span></code>, or <code class="docutils literal notranslate"><span class="pre">None</span></code> which means the size will
be the same as that of the input.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 5x7x9</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveAvgPool3d</span><span class="p">((</span><span class="mi">5</span><span class="p">,</span><span class="mi">7</span><span class="p">,</span><span class="mi">9</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 7x7x7 (cube)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveAvgPool3d</span><span class="p">(</span><span class="mi">7</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># target output size of 7x9x8</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AdaptiveMaxPool3d</span><span class="p">((</span><span class="mi">7</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">64</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">8</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="padding-layers">
<h2>Padding layers<a class="headerlink" href="#padding-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="reflectionpad1d">
<h3><span class="hidden-section">ReflectionPad1d</span><a class="headerlink" href="#reflectionpad1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ReflectionPad1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ReflectionPad1d</code><span class="sig-paren">(</span><em>padding</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ReflectionPad1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ReflectionPad1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor using the reflection of the input boundary.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 2-<cite>tuple</cite>, uses
(<span class="math">\(\text{padding\_left}\)</span>, <span class="math">\(\text{padding\_right}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, W_{out})\)</span> where</p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReflectionPad1d</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[0., 1., 2., 3.],</span>
<span class="go">         [4., 5., 6., 7.]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[2., 1., 0., 1., 2., 3., 2., 1.],</span>
<span class="go">         [6., 5., 4., 5., 6., 7., 6., 5.]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReflectionPad1d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[3., 2., 1., 0., 1., 2., 3., 2.],</span>
<span class="go">         [7., 6., 5., 4., 5., 6., 7., 6.]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="reflectionpad2d">
<h3><span class="hidden-section">ReflectionPad2d</span><a class="headerlink" href="#reflectionpad2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ReflectionPad2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ReflectionPad2d</code><span class="sig-paren">(</span><em>padding</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ReflectionPad2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ReflectionPad2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor using the reflection of the input boundary.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 4-<cite>tuple</cite>, uses (<span class="math">\(\text{padding\_left}\)</span>,
<span class="math">\(\text{padding\_right}\)</span>, <span class="math">\(\text{padding\_top}\)</span>, <span class="math">\(\text{padding\_bottom}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span> where</p>
<p><span class="math">\(H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}\)</span></p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReflectionPad2d</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">9</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[[0., 1., 2.],</span>
<span class="go">          [3., 4., 5.],</span>
<span class="go">          [6., 7., 8.]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[8., 7., 6., 7., 8., 7., 6.],</span>
<span class="go">          [5., 4., 3., 4., 5., 4., 3.],</span>
<span class="go">          [2., 1., 0., 1., 2., 1., 0.],</span>
<span class="go">          [5., 4., 3., 4., 5., 4., 3.],</span>
<span class="go">          [8., 7., 6., 7., 8., 7., 6.],</span>
<span class="go">          [5., 4., 3., 4., 5., 4., 3.],</span>
<span class="go">          [2., 1., 0., 1., 2., 1., 0.]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReflectionPad2d</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[7., 6., 7., 8., 7.],</span>
<span class="go">          [4., 3., 4., 5., 4.],</span>
<span class="go">          [1., 0., 1., 2., 1.],</span>
<span class="go">          [4., 3., 4., 5., 4.],</span>
<span class="go">          [7., 6., 7., 8., 7.]]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="replicationpad1d">
<h3><span class="hidden-section">ReplicationPad1d</span><a class="headerlink" href="#replicationpad1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ReplicationPad1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ReplicationPad1d</code><span class="sig-paren">(</span><em>padding</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ReplicationPad1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ReplicationPad1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor using replication of the input boundary.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 2-<cite>tuple</cite>, uses
(<span class="math">\(\text{padding\_left}\)</span>, <span class="math">\(\text{padding\_right}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, W_{out})\)</span> where</p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReplicationPad1d</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[0., 1., 2., 3.],</span>
<span class="go">         [4., 5., 6., 7.]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[0., 0., 0., 1., 2., 3., 3., 3.],</span>
<span class="go">         [4., 4., 4., 5., 6., 7., 7., 7.]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReplicationPad1d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[0., 0., 0., 0., 1., 2., 3., 3.],</span>
<span class="go">         [4., 4., 4., 4., 5., 6., 7., 7.]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="replicationpad2d">
<h3><span class="hidden-section">ReplicationPad2d</span><a class="headerlink" href="#replicationpad2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ReplicationPad2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ReplicationPad2d</code><span class="sig-paren">(</span><em>padding</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ReplicationPad2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ReplicationPad2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor using replication of the input boundary.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 4-<cite>tuple</cite>, uses (<span class="math">\(\text{padding\_left}\)</span>,
<span class="math">\(\text{padding\_right}\)</span>, <span class="math">\(\text{padding\_top}\)</span>, <span class="math">\(\text{padding\_bottom}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span> where</p>
<p><span class="math">\(H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}\)</span></p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReplicationPad2d</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">9</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[[0., 1., 2.],</span>
<span class="go">          [3., 4., 5.],</span>
<span class="go">          [6., 7., 8.]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[0., 0., 0., 1., 2., 2., 2.],</span>
<span class="go">          [0., 0., 0., 1., 2., 2., 2.],</span>
<span class="go">          [0., 0., 0., 1., 2., 2., 2.],</span>
<span class="go">          [3., 3., 3., 4., 5., 5., 5.],</span>
<span class="go">          [6., 6., 6., 7., 8., 8., 8.],</span>
<span class="go">          [6., 6., 6., 7., 8., 8., 8.],</span>
<span class="go">          [6., 6., 6., 7., 8., 8., 8.]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReplicationPad2d</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[0., 0., 1., 2., 2.],</span>
<span class="go">          [0., 0., 1., 2., 2.],</span>
<span class="go">          [0., 0., 1., 2., 2.],</span>
<span class="go">          [3., 3., 4., 5., 5.],</span>
<span class="go">          [6., 6., 7., 8., 8.]]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="replicationpad3d">
<h3><span class="hidden-section">ReplicationPad3d</span><a class="headerlink" href="#replicationpad3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ReplicationPad3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ReplicationPad3d</code><span class="sig-paren">(</span><em>padding</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ReplicationPad3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ReplicationPad3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor using replication of the input boundary.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 6-<cite>tuple</cite>, uses
(<span class="math">\(\text{padding\_left}\)</span>, <span class="math">\(\text{padding\_right}\)</span>,
<span class="math">\(\text{padding\_top}\)</span>, <span class="math">\(\text{padding\_bottom}\)</span>,
<span class="math">\(\text{padding\_front}\)</span>, <span class="math">\(\text{padding\_back}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span> where</p>
<p><span class="math">\(D_{out} = D_{in} + \text{padding\_front} + \text{padding\_back}\)</span></p>
<p><span class="math">\(H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}\)</span></p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReplicationPad3d</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">320</span><span class="p">,</span> <span class="mi">480</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReplicationPad3d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="zeropad2d">
<h3><span class="hidden-section">ZeroPad2d</span><a class="headerlink" href="#zeropad2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ZeroPad2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ZeroPad2d</code><span class="sig-paren">(</span><em>padding</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ZeroPad2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ZeroPad2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor boundaries with zero.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 4-<cite>tuple</cite>, uses (<span class="math">\(\text{padding\_left}\)</span>,
<span class="math">\(\text{padding\_right}\)</span>, <span class="math">\(\text{padding\_top}\)</span>, <span class="math">\(\text{padding\_bottom}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span> where</p>
<p><span class="math">\(H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}\)</span></p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ZeroPad2d</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[[-0.1678, -0.4418,  1.9466],</span>
<span class="go">          [ 0.9604, -0.4219, -0.5241],</span>
<span class="go">          [-0.9162, -0.5436, -0.6446]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000, -0.1678, -0.4418,  1.9466,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000,  0.9604, -0.4219, -0.5241,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000, -0.9162, -0.5436, -0.6446,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ZeroPad2d</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000],</span>
<span class="go">          [ 0.0000, -0.1678, -0.4418,  1.9466,  0.0000],</span>
<span class="go">          [ 0.0000,  0.9604, -0.4219, -0.5241,  0.0000],</span>
<span class="go">          [ 0.0000, -0.9162, -0.5436, -0.6446,  0.0000]]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="constantpad1d">
<h3><span class="hidden-section">ConstantPad1d</span><a class="headerlink" href="#constantpad1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ConstantPad1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ConstantPad1d</code><span class="sig-paren">(</span><em>padding</em>, <em>value</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ConstantPad1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ConstantPad1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor boundaries with a constant value.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in both boundaries. If a 2-<cite>tuple</cite>, uses
(<span class="math">\(\text{padding\_left}\)</span>, <span class="math">\(\text{padding\_right}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, W_{out})\)</span> where</p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConstantPad1d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mf">3.5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[-1.0491, -0.7152, -0.0749,  0.8530],</span>
<span class="go">         [-1.3287,  1.8966,  0.1466, -0.2771]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[ 3.5000,  3.5000, -1.0491, -0.7152, -0.0749,  0.8530,  3.5000,</span>
<span class="go">           3.5000],</span>
<span class="go">         [ 3.5000,  3.5000, -1.3287,  1.8966,  0.1466, -0.2771,  3.5000,</span>
<span class="go">           3.5000]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConstantPad1d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mf">3.5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[ 1.6616,  1.4523, -1.1255],</span>
<span class="go">         [-3.6372,  0.1182, -1.8652]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[ 3.5000,  3.5000,  1.6616,  1.4523, -1.1255,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000, -3.6372,  0.1182, -1.8652,  3.5000,  3.5000]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConstantPad1d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="mf">3.5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[ 3.5000,  3.5000,  3.5000,  1.6616,  1.4523, -1.1255,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000, -3.6372,  0.1182, -1.8652,  3.5000]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="constantpad2d">
<h3><span class="hidden-section">ConstantPad2d</span><a class="headerlink" href="#constantpad2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ConstantPad2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ConstantPad2d</code><span class="sig-paren">(</span><em>padding</em>, <em>value</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ConstantPad2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ConstantPad2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor boundaries with a constant value.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 4-<cite>tuple</cite>, uses (<span class="math">\(\text{padding\_left}\)</span>,
<span class="math">\(\text{padding\_right}\)</span>, <span class="math">\(\text{padding\_top}\)</span>, <span class="math">\(\text{padding\_bottom}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span> where</p>
<p><span class="math">\(H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}\)</span></p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConstantPad2d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mf">3.5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[ 1.6585,  0.4320],</span>
<span class="go">         [-0.8701, -0.4649]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000,  1.6585,  0.4320,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000, -0.8701, -0.4649,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000,  3.5000]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConstantPad2d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="mf">3.5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000,  1.6585,  0.4320],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000, -0.8701, -0.4649],</span>
<span class="go">         [ 3.5000,  3.5000,  3.5000,  3.5000,  3.5000]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="constantpad3d">
<h3><span class="hidden-section">ConstantPad3d</span><a class="headerlink" href="#constantpad3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ConstantPad3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ConstantPad3d</code><span class="sig-paren">(</span><em>padding</em>, <em>value</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/padding.html#ConstantPad3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ConstantPad3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads the input tensor boundaries with a constant value.</p>
<p>For <cite>N</cite>-dimensional padding, use <a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.pad()</span></code></a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>padding</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – the size of the padding. If is <cite>int</cite>, uses the same
padding in all boundaries. If a 6-<cite>tuple</cite>, uses
(<span class="math">\(\text{padding\_left}\)</span>, <span class="math">\(\text{padding\_right}\)</span>,
<span class="math">\(\text{padding\_top}\)</span>, <span class="math">\(\text{padding\_bottom}\)</span>,
<span class="math">\(\text{padding\_front}\)</span>, <span class="math">\(\text{padding\_back}\)</span>)</p>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul>
<li><p>Input: <span class="math">\((N, C, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span> where</p>
<p><span class="math">\(D_{out} = D_{in} + \text{padding\_front} + \text{padding\_back}\)</span></p>
<p><span class="math">\(H_{out} = H_{in} + \text{padding\_top} + \text{padding\_bottom}\)</span></p>
<p><span class="math">\(W_{out} = W_{in} + \text{padding\_left} + \text{padding\_right}\)</span></p>
</li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConstantPad3d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mf">3.5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># using different paddings for different sides</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ConstantPad3d</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="mf">3.5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="non-linear-activations-weighted-sum-nonlinearity">
<h2>Non-linear activations (weighted sum, nonlinearity)<a class="headerlink" href="#non-linear-activations-weighted-sum-nonlinearity" title="Permalink to this headline">¶</a></h2>
<div class="section" id="elu">
<h3><span class="hidden-section">ELU</span><a class="headerlink" href="#elu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ELU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ELU</code><span class="sig-paren">(</span><em>alpha=1.0</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#ELU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ELU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1))

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>alpha</strong> – the <span class="math">\(\alpha\)</span> value for the ELU formulation. Default: 1.0</p></li>
<li><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/ELU.png" src="_images/ELU.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ELU</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="hardshrink">
<h3><span class="hidden-section">Hardshrink</span><a class="headerlink" href="#hardshrink" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Hardshrink">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Hardshrink</code><span class="sig-paren">(</span><em>lambd=0.5</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Hardshrink"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Hardshrink" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the hard shrinkage function element-wise:</p>
<div class="math">
\[\text{HardShrink}(x) =
\begin{cases}
x, & \text{ if } x > \lambda \\
x, & \text{ if } x < -\lambda \\
0, & \text{ otherwise }
\end{cases}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>lambd</strong> – the <span class="math">\(\lambda\)</span> value for the Hardshrink formulation. Default: 0.5</p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Hardshrink.png" src="_images/Hardshrink.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Hardshrink</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="hardtanh">
<h3><span class="hidden-section">Hardtanh</span><a class="headerlink" href="#hardtanh" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Hardtanh">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Hardtanh</code><span class="sig-paren">(</span><em>min_val=-1.0</em>, <em>max_val=1.0</em>, <em>inplace=False</em>, <em>min_value=None</em>, <em>max_value=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Hardtanh"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Hardtanh" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the HardTanh function element-wise</p>
<p>HardTanh is defined as:</p>
<div class="math">
\[\text{HardTanh}(x) = \begin{cases}
    1 & \text{ if } x > 1 \\
    -1 & \text{ if } x < -1 \\
    x & \text{ otherwise } \\
\end{cases}

\]</div>
<p>The range of the linear region <span class="math">\([-1, 1]\)</span> can be adjusted using
<code class="xref py py-attr docutils literal notranslate"><span class="pre">min_val</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_val</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>min_val</strong> – minimum value of the linear region range. Default: -1</p></li>
<li><p><strong>max_val</strong> – maximum value of the linear region range. Default: 1</p></li>
<li><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<p>Keyword arguments <code class="xref py py-attr docutils literal notranslate"><span class="pre">min_value</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_value</span></code>
have been deprecated in favor of <code class="xref py py-attr docutils literal notranslate"><span class="pre">min_val</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_val</span></code>.</p>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Hardtanh.png" src="_images/Hardtanh.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Hardtanh</span><span class="p">(</span><span class="o">-</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="leakyrelu">
<h3><span class="hidden-section">LeakyReLU</span><a class="headerlink" href="#leakyrelu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LeakyReLU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LeakyReLU</code><span class="sig-paren">(</span><em>negative_slope=0.01</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#LeakyReLU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LeakyReLU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)

\]</div>
<p>or</p>
<div class="math">
\[\text{LeakyRELU}(x) =
\begin{cases}
x, & \text{ if } x \geq 0 \\
\text{negative\_slope} \times x, & \text{ otherwise }
\end{cases}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>negative_slope</strong> – Controls the angle of the negative slope. Default: 1e-2</p></li>
<li><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/LeakyReLU.png" src="_images/LeakyReLU.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LeakyReLU</span><span class="p">(</span><span class="mf">0.1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="logsigmoid">
<h3><span class="hidden-section">LogSigmoid</span><a class="headerlink" href="#logsigmoid" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LogSigmoid">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LogSigmoid</code><a class="reference internal" href="_modules/torch/nn/modules/activation.html#LogSigmoid"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LogSigmoid" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{LogSigmoid}(x) = \log\left(\frac{ 1 }{ 1 + \exp(-x)}\right)

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/LogSigmoid.png" src="_images/LogSigmoid.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LogSigmoid</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="multiheadattention">
<h3><span class="hidden-section">MultiheadAttention</span><a class="headerlink" href="#multiheadattention" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MultiheadAttention">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MultiheadAttention</code><span class="sig-paren">(</span><em>embed_dim</em>, <em>num_heads</em>, <em>dropout=0.0</em>, <em>bias=True</em>, <em>add_bias_kv=False</em>, <em>add_zero_attn=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#MultiheadAttention"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MultiheadAttention" title="Permalink to this definition">¶</a></dt>
<dd><p>Allows the model to jointly attend to information
from different representation subspaces.
See reference: Attention Is All You Need</p>
<div class="math">
\[\text{MultiHead}(Q, K, V) = \text{Concat}(head_1,\dots,head_h)W^O
\text{where} head_i = \text{Attention}(QW_i^Q, KW_i^K, VW_i^V)

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>embed_dim</strong> – total dimension of the model</p></li>
<li><p><strong>num_heads</strong> – parallel attention layers, or heads</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">multihead_attn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MultiheadAttention</span><span class="p">(</span><span class="n">embed_dim</span><span class="p">,</span> <span class="n">num_heads</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">attn_output</span><span class="p">,</span> <span class="n">attn_output_weights</span> <span class="o">=</span> <span class="n">multihead_attn</span><span class="p">(</span><span class="n">query</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.nn.MultiheadAttention.forward">
<code class="descname">forward</code><span class="sig-paren">(</span><em>query</em>, <em>key</em>, <em>value</em>, <em>key_padding_mask=None</em>, <em>incremental_state=None</em>, <em>need_weights=True</em>, <em>static_kv=False</em>, <em>attn_mask=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#MultiheadAttention.forward"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MultiheadAttention.forward" title="Permalink to this definition">¶</a></dt>
<dd><dl class="simple">
<dt>Inputs of forward function</dt><dd><p>query: [target length, batch size, embed dim]
key: [sequence length, batch size, embed dim]
value: [sequence length, batch size, embed dim]
key_padding_mask: if True, mask padding based on batch size
incremental_state: if provided, previous time steps are cashed
need_weights: output attn_output_weights
static_kv: key and value are static</p>
</dd>
<dt>Outputs of forward function</dt><dd><p>attn_output: [target length, batch size, embed dim]
attn_output_weights: [batch size, target length, sequence length]</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="prelu">
<h3><span class="hidden-section">PReLU</span><a class="headerlink" href="#prelu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.PReLU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">PReLU</code><span class="sig-paren">(</span><em>num_parameters=1</em>, <em>init=0.25</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#PReLU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.PReLU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{PReLU}(x) = \max(0,x) + a * \min(0,x)

\]</div>
<p>or</p>
<div class="math">
\[\text{PReLU}(x) =
\begin{cases}
x, & \text{ if } x \geq 0 \\
ax, & \text{ otherwise }
\end{cases}

\]</div>
<p>Here <span class="math">\(a\)</span> is a learnable parameter. When called without arguments, <cite>nn.PReLU()</cite> uses a single
parameter <span class="math">\(a\)</span> across all input channels. If called with <cite>nn.PReLU(nChannels)</cite>,
a separate <span class="math">\(a\)</span> is used for each input channel.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>weight decay should not be used when learning <span class="math">\(a\)</span> for good performance.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Channel dim is the 2nd dim of input. When input has dims &lt; 2, then there is
no channel dim and the number of channels = 1.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_parameters</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – number of <span class="math">\(a\)</span> to learn.
Although it takes an int as input, there is only two values are legitimate:
1, or the number of channels at input. Default: 1</p></li>
<li><p><strong>init</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – the initial value of <span class="math">\(a\)</span>. Default: 0.25</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><p><strong>~PReLU.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of shape (<code class="xref py py-attr docutils literal notranslate"><span class="pre">num_parameters</span></code>).</p>
</dd>
</dl>
<img alt="_images/PReLU.png" src="_images/PReLU.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">PReLU</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="relu">
<h3><span class="hidden-section">ReLU</span><a class="headerlink" href="#relu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ReLU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ReLU</code><span class="sig-paren">(</span><em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#ReLU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ReLU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the rectified linear unit function element-wise:</p>
<p><span class="math">\(\text{ReLU}(x)= \max(0, x)\)</span></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/ReLU.png" src="_images/ReLU.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>  <span class="o">&gt;&gt;&gt;</span> <span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()</span>
  <span class="o">&gt;&gt;&gt;</span> <span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
  <span class="o">&gt;&gt;&gt;</span> <span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>


<span class="n">An</span> <span class="n">implementation</span> <span class="n">of</span> <span class="n">CReLU</span> <span class="o">-</span> <span class="n">https</span><span class="p">:</span><span class="o">//</span><span class="n">arxiv</span><span class="o">.</span><span class="n">org</span><span class="o">/</span><span class="nb">abs</span><span class="o">/</span><span class="mf">1603.05201</span>

  <span class="o">&gt;&gt;&gt;</span> <span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">()</span>
  <span class="o">&gt;&gt;&gt;</span> <span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
  <span class="o">&gt;&gt;&gt;</span> <span class="n">output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">((</span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">),</span><span class="n">m</span><span class="p">(</span><span class="o">-</span><span class="nb">input</span><span class="p">)))</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="relu6">
<h3><span class="hidden-section">ReLU6</span><a class="headerlink" href="#relu6" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.ReLU6">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">ReLU6</code><span class="sig-paren">(</span><em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#ReLU6"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.ReLU6" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{ReLU6}(x) = \min(\max(0,x), 6)

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/ReLU6.png" src="_images/ReLU6.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">ReLU6</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="rrelu">
<h3><span class="hidden-section">RReLU</span><a class="headerlink" href="#rrelu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.RReLU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">RReLU</code><span class="sig-paren">(</span><em>lower=0.125</em>, <em>upper=0.3333333333333333</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#RReLU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.RReLU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the randomized leaky rectified liner unit function, element-wise,
as described in the paper:</p>
<p><a class="reference external" href="https://arxiv.org/abs/1505.00853">Empirical Evaluation of Rectified Activations in Convolutional Network</a>.</p>
<p>The function is defined as:</p>
<div class="math">
\[\text{RReLU}(x) =
\begin{cases}
    x & \text{if } x \geq 0 \\
    ax & \text{ otherwise }
\end{cases}

\]</div>
<p>where <span class="math">\(a\)</span> is randomly sampled from uniform distribution
<span class="math">\(\mathcal{U}(\text{lower}, \text{upper})\)</span>.</p>
<blockquote>
<div><p>See: <a class="reference external" href="https://arxiv.org/pdf/1505.00853.pdf">https://arxiv.org/pdf/1505.00853.pdf</a></p>
</div></blockquote>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>lower</strong> – lower bound of the uniform distribution. Default: <span class="math">\(\frac{1}{8}\)</span></p></li>
<li><p><strong>upper</strong> – upper bound of the uniform distribution. Default: <span class="math">\(\frac{1}{3}\)</span></p></li>
<li><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">RReLU</span><span class="p">(</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="selu">
<h3><span class="hidden-section">SELU</span><a class="headerlink" href="#selu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.SELU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">SELU</code><span class="sig-paren">(</span><em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#SELU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.SELU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applied element-wise, as:</p>
<div class="math">
\[\text{SELU}(x) = \text{scale} * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))

\]</div>
<p>with <span class="math">\(\alpha = 1.6732632423543772848170429916717\)</span> and
<span class="math">\(\text{scale} = 1.0507009873554804934193349852946\)</span>.</p>
<p>More details can be found in the paper <a class="reference external" href="https://arxiv.org/abs/1706.02515">Self-Normalizing Neural Networks</a> .</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>inplace</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/SELU.png" src="_images/SELU.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">SELU</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="celu">
<h3><span class="hidden-section">CELU</span><a class="headerlink" href="#celu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.CELU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">CELU</code><span class="sig-paren">(</span><em>alpha=1.0</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#CELU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.CELU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))

\]</div>
<p>More details can be found in the paper <a class="reference external" href="https://arxiv.org/abs/1704.07483">Continuously Differentiable Exponential Linear Units</a> .</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>alpha</strong> – the <span class="math">\(\alpha\)</span> value for the CELU formulation. Default: 1.0</p></li>
<li><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/CELU.png" src="_images/CELU.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CELU</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="sigmoid">
<h3><span class="hidden-section">Sigmoid</span><a class="headerlink" href="#sigmoid" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Sigmoid">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Sigmoid</code><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Sigmoid"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Sigmoid" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Sigmoid.png" src="_images/Sigmoid.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sigmoid</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="softplus">
<h3><span class="hidden-section">Softplus</span><a class="headerlink" href="#softplus" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Softplus">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Softplus</code><span class="sig-paren">(</span><em>beta=1</em>, <em>threshold=20</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Softplus"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Softplus" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))

\]</div>
<p>SoftPlus is a smooth approximation to the ReLU function and can be used
to constrain the output of a machine to always be positive.</p>
<p>For numerical stability the implementation reverts to the linear function
for inputs above a certain value.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>beta</strong> – the <span class="math">\(\beta\)</span> value for the Softplus formulation. Default: 1</p></li>
<li><p><strong>threshold</strong> – values above this revert to a linear function. Default: 20</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Softplus.png" src="_images/Softplus.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Softplus</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="softshrink">
<h3><span class="hidden-section">Softshrink</span><a class="headerlink" href="#softshrink" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Softshrink">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Softshrink</code><span class="sig-paren">(</span><em>lambd=0.5</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Softshrink"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Softshrink" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the soft shrinkage function elementwise:</p>
<div class="math">
\[\text{SoftShrinkage}(x) =
\begin{cases}
x - \lambda, & \text{ if } x > \lambda \\
x + \lambda, & \text{ if } x < -\lambda \\
0, & \text{ otherwise }
\end{cases}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>lambd</strong> – the <span class="math">\(\lambda\)</span> value for the Softshrink formulation. Default: 0.5</p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Softshrink.png" src="_images/Softshrink.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Softshrink</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="softsign">
<h3><span class="hidden-section">Softsign</span><a class="headerlink" href="#softsign" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Softsign">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Softsign</code><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Softsign"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Softsign" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{SoftSign}(x) = \frac{x}{ 1 + |x|}

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Softsign.png" src="_images/Softsign.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Softsign</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="tanh">
<h3><span class="hidden-section">Tanh</span><a class="headerlink" href="#tanh" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Tanh">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Tanh</code><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Tanh"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Tanh" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{Tanh}(x) = \tanh(x) = \frac{e^x - e^{-x}} {e^x + e^{-x}}

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Tanh.png" src="_images/Tanh.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Tanh</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="tanhshrink">
<h3><span class="hidden-section">Tanhshrink</span><a class="headerlink" href="#tanhshrink" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Tanhshrink">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Tanhshrink</code><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Tanhshrink"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Tanhshrink" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function:</p>
<div class="math">
\[\text{Tanhshrink}(x) = x - \text{Tanh}(x)

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<img alt="_images/Tanhshrink.png" src="_images/Tanhshrink.png" />
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Tanhshrink</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="threshold">
<h3><span class="hidden-section">Threshold</span><a class="headerlink" href="#threshold" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Threshold">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Threshold</code><span class="sig-paren">(</span><em>threshold</em>, <em>value</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Threshold"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Threshold" title="Permalink to this definition">¶</a></dt>
<dd><p>Thresholds each element of the input Tensor.</p>
<p>Threshold is defined as:</p>
<div class="math">
\[y =
\begin{cases}
x, &\text{ if } x > \text{threshold} \\
\text{value}, &\text{ otherwise }
\end{cases}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>threshold</strong> – The value to threshold at</p></li>
<li><p><strong>value</strong> – The value to replace with</p></li>
<li><p><strong>inplace</strong> – can optionally do the operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Threshold</span><span class="p">(</span><span class="mf">0.1</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="non-linear-activations-other">
<h2>Non-linear activations (other)<a class="headerlink" href="#non-linear-activations-other" title="Permalink to this headline">¶</a></h2>
<div class="section" id="softmin">
<h3><span class="hidden-section">Softmin</span><a class="headerlink" href="#softmin" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Softmin">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Softmin</code><span class="sig-paren">(</span><em>dim=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Softmin"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Softmin" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the Softmin function to an n-dimensional input Tensor
rescaling them so that the elements of the n-dimensional output Tensor
lie in the range <cite>[0, 1]</cite> and sum to 1.</p>
<p>Softmin is defined as:</p>
<div class="math">
\[\text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)}

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((*)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – A dimension along which Softmin will be computed (so every slice
along dim will sum to 1).</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>a Tensor of the same dimension and shape as the input, with
values in the range [0, 1]</p>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Softmin</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="softmax">
<h3><span class="hidden-section">Softmax</span><a class="headerlink" href="#softmax" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Softmax">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Softmax</code><span class="sig-paren">(</span><em>dim=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Softmax"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Softmax" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the Softmax function to an n-dimensional input Tensor
rescaling them so that the elements of the n-dimensional output Tensor
lie in the range [0,1] and sum to 1.</p>
<p>Softmax is defined as:</p>
<div class="math">
\[\text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((*)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>a Tensor of the same dimension and shape as the input with
values in the range [0, 1]</p>
</dd>
<dt class="field-even">Parameters</dt>
<dd class="field-even"><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – A dimension along which Softmax will be computed (so every slice
along dim will sum to 1).</p>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This module doesn’t work directly with NLLLoss,
which expects the Log to be computed between the Softmax and itself.
Use <cite>LogSoftmax</cite> instead (it’s faster and has better numerical properties).</p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Softmax</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="softmax2d">
<h3><span class="hidden-section">Softmax2d</span><a class="headerlink" href="#softmax2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Softmax2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Softmax2d</code><a class="reference internal" href="_modules/torch/nn/modules/activation.html#Softmax2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Softmax2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies SoftMax over features to each spatial location.</p>
<p>When given an image of <code class="docutils literal notranslate"><span class="pre">Channels</span> <span class="pre">x</span> <span class="pre">Height</span> <span class="pre">x</span> <span class="pre">Width</span></code>, it will
apply <cite>Softmax</cite> to each location <span class="math">\((Channels, h_i, w_j)\)</span></p>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, H, W)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H, W)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>a Tensor of the same dimension and shape as the input with
values in the range [0, 1]</p>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Softmax2d</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># you softmax over the 2nd dimension</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">12</span><span class="p">,</span> <span class="mi">13</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="logsoftmax">
<h3><span class="hidden-section">LogSoftmax</span><a class="headerlink" href="#logsoftmax" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LogSoftmax">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LogSoftmax</code><span class="sig-paren">(</span><em>dim=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/activation.html#LogSoftmax"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LogSoftmax" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the <span class="math">\(\log(\text{Softmax}(x))\)</span> function to an n-dimensional
input Tensor. The LogSoftmax formulation can be simplified as:</p>
<div class="math">
\[\text{LogSoftmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right)

\]</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span> where <cite>*</cite> means, any number of additional
dimensions</p></li>
<li><p>Output: <span class="math">\((*)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – A dimension along which LogSoftmax will be computed.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>a Tensor of the same dimension and shape as the input with
values in the range [-inf, 0)</p>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LogSoftmax</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="adaptivelogsoftmaxwithloss">
<h3><span class="hidden-section">AdaptiveLogSoftmaxWithLoss</span><a class="headerlink" href="#adaptivelogsoftmaxwithloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AdaptiveLogSoftmaxWithLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AdaptiveLogSoftmaxWithLoss</code><span class="sig-paren">(</span><em>in_features</em>, <em>n_classes</em>, <em>cutoffs</em>, <em>div_value=4.0</em>, <em>head_bias=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/adaptive.html#AdaptiveLogSoftmaxWithLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveLogSoftmaxWithLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Efficient softmax approximation as described in
<a class="reference external" href="https://arxiv.org/abs/1609.04309">Efficient softmax approximation for GPUs</a> by Edouard Grave, Armand Joulin,
Moustapha Cissé, David Grangier, and Hervé Jégou.</p>
<p>Adaptive softmax is an approximate strategy for training models with large
output spaces. It is most effective when the label distribution is highly
imbalanced, for example in natural language modelling, where the word
frequency distribution approximately follows the <a class="reference external" href="https://en.wikipedia.org/wiki/Zipf%27s_law">Zipf’s law</a>.</p>
<p>Adaptive softmax partitions the labels into several clusters, according to
their frequency. These clusters may contain different number of targets
each.
Additionally, clusters containing less frequent labels assign lower
dimensional embeddings to those labels, which speeds up the computation.
For each minibatch, only clusters for which at least one target is
present are evaluated.</p>
<p>The idea is that the clusters which are accessed frequently
(like the first one, containing most frequent labels), should also be cheap
to compute – that is, contain a small number of assigned labels.</p>
<p>We highly recommend taking a look at the original paper for more details.</p>
<ul class="simple">
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">cutoffs</span></code> should be an ordered Sequence of integers sorted
in the increasing order.
It controls number of clusters and the partitioning of targets into
clusters. For example setting <code class="docutils literal notranslate"><span class="pre">cutoffs</span> <span class="pre">=</span> <span class="pre">[10,</span> <span class="pre">100,</span> <span class="pre">1000]</span></code>
means that first <cite>10</cite> targets will be assigned
to the ‘head’ of the adaptive softmax, targets <cite>11, 12, …, 100</cite> will be
assigned to the first cluster, and targets <cite>101, 102, …, 1000</cite> will be
assigned to the second cluster, while targets
<cite>1001, 1002, …, n_classes - 1</cite> will be assigned
to the last, third cluster.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">div_value</span></code> is used to compute the size of each additional cluster,
which is given as
<span class="math">\(\left\lfloor\frac{in\_features}{div\_value^{idx}}\right\rfloor\)</span>,
where <span class="math">\(idx\)</span> is the cluster index (with clusters
for less frequent words having larger indices,
and indices starting from <span class="math">\(1\)</span>).</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">head_bias</span></code> if set to True, adds a bias term to the ‘head’ of the
adaptive softmax. See paper for details. Set to False in the official
implementation.</p></li>
</ul>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Labels passed as inputs to this module should be sorted accoridng to
their frequency. This means that the most frequent label should be
represented by the index <cite>0</cite>, and the least frequent
label should be represented by the index <cite>n_classes - 1</cite>.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This module returns a <code class="docutils literal notranslate"><span class="pre">NamedTuple</span></code> with <code class="docutils literal notranslate"><span class="pre">output</span></code>
and <code class="docutils literal notranslate"><span class="pre">loss</span></code> fields. See further documentation for details.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>To compute log-probabilities for all classes, the <code class="docutils literal notranslate"><span class="pre">log_prob</span></code>
method can be used.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_features</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of features in the input tensor</p></li>
<li><p><strong>n_classes</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of classes in the dataset</p></li>
<li><p><strong>cutoffs</strong> (<em>Sequence</em>) – Cutoffs used to assign targets to their buckets</p></li>
<li><p><strong>div_value</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – value used as an exponent to compute sizes
of the clusters. Default: 4.0</p></li>
<li><p><strong>head_bias</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, adds a bias term to the ‘head’ of the
adaptive softmax. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p><ul class="simple">
<li><p><strong>output</strong> is a Tensor of size <code class="docutils literal notranslate"><span class="pre">N</span></code> containing computed target
log probabilities for each example</p></li>
<li><p><strong>loss</strong> is a Scalar representing the computed negative
log likelihood loss</p></li>
</ul>
</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p><code class="docutils literal notranslate"><span class="pre">NamedTuple</span></code> with <code class="docutils literal notranslate"><span class="pre">output</span></code> and <code class="docutils literal notranslate"><span class="pre">loss</span></code> fields</p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>input: <span class="math">\((N, in\_features)\)</span></p></li>
<li><p>target: <span class="math">\((N)\)</span> where each value satisfies <span class="math">\(0 &lt;= target[i] &lt;= n\_classes\)</span></p></li>
<li><p>output1: <span class="math">\((N)\)</span></p></li>
<li><p>output2: <code class="docutils literal notranslate"><span class="pre">Scalar</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.nn.AdaptiveLogSoftmaxWithLoss.log_prob">
<code class="descname">log_prob</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/adaptive.html#AdaptiveLogSoftmaxWithLoss.log_prob"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveLogSoftmaxWithLoss.log_prob" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes log probabilities for all <span class="math">\(n\_classes\)</span></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – a minibatch of examples</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>log-probabilities of for each class <span class="math">\(c\)</span>
in range <span class="math">\(0 &lt;= c &lt;= n\_classes\)</span>, where <span class="math">\(n\_classes\)</span> is a
parameter passed to <code class="docutils literal notranslate"><span class="pre">AdaptiveLogSoftmaxWithLoss</span></code> constructor.</p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, in\_features)\)</span></p></li>
<li><p>Output: <span class="math">\((N, n\_classes)\)</span></p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.nn.AdaptiveLogSoftmaxWithLoss.predict">
<code class="descname">predict</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/adaptive.html#AdaptiveLogSoftmaxWithLoss.predict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AdaptiveLogSoftmaxWithLoss.predict" title="Permalink to this definition">¶</a></dt>
<dd><p>This is equivalent to <cite>self.log_pob(input).argmax(dim=1)</cite>,
but is more efficient in some cases.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – a minibatch of examples</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>a class with the highest probability for each example</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>output (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor">Tensor</a>)</p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, in\_features)\)</span></p></li>
<li><p>Output: <span class="math">\((N)\)</span></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
</div>
<div class="section" id="normalization-layers">
<h2>Normalization layers<a class="headerlink" href="#normalization-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="batchnorm1d">
<h3><span class="hidden-section">BatchNorm1d</span><a class="headerlink" href="#batchnorm1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.BatchNorm1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">BatchNorm1d</code><span class="sig-paren">(</span><em>num_features</em>, <em>eps=1e-05</em>, <em>momentum=0.1</em>, <em>affine=True</em>, <em>track_running_stats=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/batchnorm.html#BatchNorm1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.BatchNorm1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Batch Normalization over a 2D or 3D input (a mini-batch of 1D
inputs with optional additional channel dimension) as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1502.03167">Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta\]</div>
<p>The mean and standard-deviation are calculated per-dimension over
the mini-batches and <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable parameter vectors
of size <cite>C</cite> (where <cite>C</cite> is the input size). By default, the elements of <span class="math">\(\gamma\)</span> are sampled
from <span class="math">\(\mathcal{U}(0, 1)\)</span> and the elements of <span class="math">\(\beta\)</span> are set to 0.</p>
<p>Also by default, during training this layer keeps running estimates of its
computed mean and variance, which are then used for normalization during
evaluation. The running estimates are kept with a default <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code>
of 0.1.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">track_running_stats</span></code> is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, this layer then does not
keep running estimates, and batch statistics are instead used during
evaluation time as well.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> argument is different from one used in optimizer
classes and the conventional notion of momentum. Mathematically, the
update rule for running statistics here is
<span class="math">\(\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t\)</span>,
where <span class="math">\(\hat{x}\)</span> is the estimated statistic and <span class="math">\(x_t\)</span> is the
new observed value.</p>
</div>
<p>Because the Batch Normalization is done over the <cite>C</cite> dimension, computing statistics
on <cite>(N, L)</cite> slices, it’s common terminology to call this Temporal Batch Normalization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_features</strong> – <span class="math">\(C\)</span> from an expected input of size
<span class="math">\((N, C, L)\)</span> or <span class="math">\(L\)</span> from input of size <span class="math">\((N, L)\)</span></p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability.
Default: 1e-5</p></li>
<li><p><strong>momentum</strong> – the value used for the running_mean and running_var
computation. Can be set to <code class="docutils literal notranslate"><span class="pre">None</span></code> for cumulative moving average
(i.e. simple average). Default: 0.1</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module has
learnable affine parameters. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>track_running_stats</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this
module tracks the running mean and variance, and when set to <code class="docutils literal notranslate"><span class="pre">False</span></code>,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C)\)</span> or <span class="math">\((N, C, L)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C)\)</span> or <span class="math">\((N, C, L)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm1d</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm1d</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="batchnorm2d">
<h3><span class="hidden-section">BatchNorm2d</span><a class="headerlink" href="#batchnorm2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.BatchNorm2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">BatchNorm2d</code><span class="sig-paren">(</span><em>num_features</em>, <em>eps=1e-05</em>, <em>momentum=0.1</em>, <em>affine=True</em>, <em>track_running_stats=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/batchnorm.html#BatchNorm2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.BatchNorm2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Batch Normalization over a 4D input (a mini-batch of 2D inputs
with additional channel dimension) as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1502.03167">Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta\]</div>
<p>The mean and standard-deviation are calculated per-dimension over
the mini-batches and <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable parameter vectors
of size <cite>C</cite> (where <cite>C</cite> is the input size). By default, the elements of <span class="math">\(\gamma\)</span> are sampled
from <span class="math">\(\mathcal{U}(0, 1)\)</span> and the elements of <span class="math">\(\beta\)</span> are set to 0.</p>
<p>Also by default, during training this layer keeps running estimates of its
computed mean and variance, which are then used for normalization during
evaluation. The running estimates are kept with a default <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code>
of 0.1.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">track_running_stats</span></code> is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, this layer then does not
keep running estimates, and batch statistics are instead used during
evaluation time as well.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> argument is different from one used in optimizer
classes and the conventional notion of momentum. Mathematically, the
update rule for running statistics here is
<span class="math">\(\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t\)</span>,
where <span class="math">\(\hat{x}\)</span> is the estimated statistic and <span class="math">\(x_t\)</span> is the
new observed value.</p>
</div>
<p>Because the Batch Normalization is done over the <cite>C</cite> dimension, computing statistics
on <cite>(N, H, W)</cite> slices, it’s common terminology to call this Spatial Batch Normalization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_features</strong> – <span class="math">\(C\)</span> from an expected input of size
<span class="math">\((N, C, H, W)\)</span></p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability.
Default: 1e-5</p></li>
<li><p><strong>momentum</strong> – the value used for the running_mean and running_var
computation. Can be set to <code class="docutils literal notranslate"><span class="pre">None</span></code> for cumulative moving average
(i.e. simple average). Default: 0.1</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module has
learnable affine parameters. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>track_running_stats</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this
module tracks the running mean and variance, and when set to <code class="docutils literal notranslate"><span class="pre">False</span></code>,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, H, W)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H, W)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm2d</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm2d</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">35</span><span class="p">,</span> <span class="mi">45</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="batchnorm3d">
<h3><span class="hidden-section">BatchNorm3d</span><a class="headerlink" href="#batchnorm3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.BatchNorm3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">BatchNorm3d</code><span class="sig-paren">(</span><em>num_features</em>, <em>eps=1e-05</em>, <em>momentum=0.1</em>, <em>affine=True</em>, <em>track_running_stats=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/batchnorm.html#BatchNorm3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.BatchNorm3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Batch Normalization over a 5D input (a mini-batch of 3D inputs
with additional channel dimension) as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1502.03167">Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta\]</div>
<p>The mean and standard-deviation are calculated per-dimension over
the mini-batches and <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable parameter vectors
of size <cite>C</cite> (where <cite>C</cite> is the input size). By default, the elements of <span class="math">\(\gamma\)</span> are sampled
from <span class="math">\(\mathcal{U}(0, 1)\)</span> and the elements of <span class="math">\(\beta\)</span> are set to 0.</p>
<p>Also by default, during training this layer keeps running estimates of its
computed mean and variance, which are then used for normalization during
evaluation. The running estimates are kept with a default <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code>
of 0.1.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">track_running_stats</span></code> is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, this layer then does not
keep running estimates, and batch statistics are instead used during
evaluation time as well.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> argument is different from one used in optimizer
classes and the conventional notion of momentum. Mathematically, the
update rule for running statistics here is
<span class="math">\(\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momentum} \times x_t\)</span>,
where <span class="math">\(\hat{x}\)</span> is the estimated statistic and <span class="math">\(x_t\)</span> is the
new observed value.</p>
</div>
<p>Because the Batch Normalization is done over the <cite>C</cite> dimension, computing statistics
on <cite>(N, D, H, W)</cite> slices, it’s common terminology to call this Volumetric Batch Normalization
or Spatio-temporal Batch Normalization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_features</strong> – <span class="math">\(C\)</span> from an expected input of size
<span class="math">\((N, C, D, H, W)\)</span></p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability.
Default: 1e-5</p></li>
<li><p><strong>momentum</strong> – the value used for the running_mean and running_var
computation. Can be set to <code class="docutils literal notranslate"><span class="pre">None</span></code> for cumulative moving average
(i.e. simple average). Default: 0.1</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module has
learnable affine parameters. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>track_running_stats</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this
module tracks the running mean and variance, and when set to <code class="docutils literal notranslate"><span class="pre">False</span></code>,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, D, H, W)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D, H, W)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm3d</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm3d</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">35</span><span class="p">,</span> <span class="mi">45</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="groupnorm">
<h3><span class="hidden-section">GroupNorm</span><a class="headerlink" href="#groupnorm" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.GroupNorm">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">GroupNorm</code><span class="sig-paren">(</span><em>num_groups</em>, <em>num_channels</em>, <em>eps=1e-05</em>, <em>affine=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/normalization.html#GroupNorm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.GroupNorm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Group Normalization over a mini-batch of inputs as described in
the paper <a class="reference external" href="https://arxiv.org/abs/1803.08494">Group Normalization</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta

\]</div>
<p>The input channels are separated into <code class="xref py py-attr docutils literal notranslate"><span class="pre">num_groups</span></code> groups, each containing
<code class="docutils literal notranslate"><span class="pre">num_channels</span> <span class="pre">/</span> <span class="pre">num_groups</span></code> channels. The mean and standard-deviation are calculated
separately over the each group. <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable
per-channel affine transform parameter vectors of size <code class="xref py py-attr docutils literal notranslate"><span class="pre">num_channels</span></code> if
<code class="xref py py-attr docutils literal notranslate"><span class="pre">affine</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
<p>This layer uses statistics computed from input data in both training and
evaluation modes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_groups</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – number of groups to separate the channels into</p></li>
<li><p><strong>num_channels</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – number of channels expected in input</p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability. Default: 1e-5</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module
has learnable per-channel affine parameters initialized to ones (for weights)
and zeros (for biases). Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, *)\)</span> where <span class="math">\(C=\text{num\_channels}\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, *)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Separate 6 channels into 3 groups</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GroupNorm</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Separate 6 channels into 6 groups (equivalent with InstanceNorm)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GroupNorm</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Put all 6 channels into a single group (equivalent with LayerNorm)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GroupNorm</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Activating the module</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="syncbatchnorm">
<h3><span class="hidden-section">SyncBatchNorm</span><a class="headerlink" href="#syncbatchnorm" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.SyncBatchNorm">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">SyncBatchNorm</code><span class="sig-paren">(</span><em>num_features</em>, <em>eps=1e-05</em>, <em>momentum=0.1</em>, <em>affine=True</em>, <em>track_running_stats=True</em>, <em>process_group=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/batchnorm.html#SyncBatchNorm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.SyncBatchNorm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Batch Normalization over a N-Dimensional input (a mini-batch of [N-2]D inputs
with additional channel dimension) as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1502.03167">Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta\]</div>
<p>The mean and standard-deviation are calculated per-dimension over all
mini-batches of the same process groups. <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span>
are learnable parameter vectors of size <cite>C</cite> (where <cite>C</cite> is the input size).
By default, the elements of <span class="math">\(\gamma\)</span> are sampled from
<span class="math">\(\mathcal{U}(0, 1)\)</span> and the elements of <span class="math">\(\beta\)</span> are set to 0.</p>
<p>Also by default, during training this layer keeps running estimates of its
computed mean and variance, which are then used for normalization during
evaluation. The running estimates are kept with a default <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code>
of 0.1.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">track_running_stats</span></code> is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, this layer then does not
keep running estimates, and batch statistics are instead used during
evaluation time as well.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> argument is different from one used in optimizer
classes and the conventional notion of momentum. Mathematically, the
update rule for running statistics here is
<span class="math">\(\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momemtum} \times x_t\)</span>,
where <span class="math">\(\hat{x}\)</span> is the estimated statistic and <span class="math">\(x_t\)</span> is the
new observed value.</p>
</div>
<p>Because the Batch Normalization is done over the <cite>C</cite> dimension, computing statistics
on <cite>(N, +)</cite> slices, it’s common terminology to call this Volumetric Batch Normalization
or Spatio-temporal Batch Normalization.</p>
<p>Currently SyncBatchNorm only supports DistributedDataParallel with single GPU per process. Use
torch.nn.SyncBatchNorm.convert_sync_batchnorm() to convert BatchNorm layer to SyncBatchNorm before wrapping
Network with DDP.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_features</strong> – <span class="math">\(C\)</span> from an expected input of size
<span class="math">\((N, C, +)\)</span></p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability.
Default: 1e-5</p></li>
<li><p><strong>momentum</strong> – the value used for the running_mean and running_var
computation. Can be set to <code class="docutils literal notranslate"><span class="pre">None</span></code> for cumulative moving average
(i.e. simple average). Default: 0.1</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module has
learnable affine parameters. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>track_running_stats</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this
module tracks the running mean and variance, and when set to <code class="docutils literal notranslate"><span class="pre">False</span></code>,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>process_group</strong> – synchronization of stats happen within each process group
individually. Default behavior is synchronization across the whole
world</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, +)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, +)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">SyncBatchNorm</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># creating process group (optional)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># process_ids is a list of int identifying rank ids.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">process_group</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="n">new_group</span><span class="p">(</span><span class="n">process_ids</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm3d</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">process_group</span><span class="o">=</span><span class="n">process_group</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">35</span><span class="p">,</span> <span class="mi">45</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># network is nn.BatchNorm layer</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">sync_bn_network</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">convert_sync_batchnorm</span><span class="p">(</span><span class="n">network</span><span class="p">,</span> <span class="n">process_group</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># only single gpu per process is currently supported</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ddp_sync_bn_network</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">parallel</span><span class="o">.</span><span class="n">DistributedDataParallel</span><span class="p">(</span>
<span class="gp">&gt;&gt;&gt; </span>                        <span class="n">sync_bn_network</span><span class="p">,</span>
<span class="gp">&gt;&gt;&gt; </span>                        <span class="n">device_ids</span><span class="o">=</span><span class="p">[</span><span class="n">args</span><span class="o">.</span><span class="n">local_rank</span><span class="p">],</span>
<span class="gp">&gt;&gt;&gt; </span>                        <span class="n">output_device</span><span class="o">=</span><span class="n">args</span><span class="o">.</span><span class="n">local_rank</span><span class="p">)</span>
</pre></div>
</div>
<dl class="classmethod">
<dt id="torch.nn.SyncBatchNorm.convert_sync_batchnorm">
<em class="property">classmethod </em><code class="descname">convert_sync_batchnorm</code><span class="sig-paren">(</span><em>module</em>, <em>process_group=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/batchnorm.html#SyncBatchNorm.convert_sync_batchnorm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.SyncBatchNorm.convert_sync_batchnorm" title="Permalink to this definition">¶</a></dt>
<dd><p>Helper function to convert <cite>torch.nn.BatchNormND</cite> layer in the model to
<cite>torch.nn.SyncBatchNorm</cite> layer.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>nn.Module</em></a>) – containing module</p></li>
<li><p><strong>process_group</strong> (<em>optional</em>) – process group to scope synchronization,</p></li>
</ul>
</dd>
</dl>
<p>default is the whole world</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>The original module with the converted <cite>torch.nn.SyncBatchNorm</cite> layer</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Network with nn.BatchNorm layer</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">module</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
<span class="gp">&gt;&gt;&gt; </span>           <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">),</span>
<span class="gp">&gt;&gt;&gt; </span>           <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">BatchNorm1d</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>         <span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># creating process group (optional)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># process_ids is a list of int identifying rank ids.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">process_group</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="n">new_group</span><span class="p">(</span><span class="n">process_ids</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">sync_bn_module</span> <span class="o">=</span> <span class="n">convert_sync_batchnorm</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">process_group</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="instancenorm1d">
<h3><span class="hidden-section">InstanceNorm1d</span><a class="headerlink" href="#instancenorm1d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.InstanceNorm1d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">InstanceNorm1d</code><span class="sig-paren">(</span><em>num_features</em>, <em>eps=1e-05</em>, <em>momentum=0.1</em>, <em>affine=False</em>, <em>track_running_stats=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/instancenorm.html#InstanceNorm1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.InstanceNorm1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Instance Normalization over a 3D input (a mini-batch of 1D
inputs with optional additional channel dimension) as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1607.08022">Instance Normalization: The Missing Ingredient for Fast Stylization</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta\]</div>
<p>The mean and standard-deviation are calculated per-dimension separately
for each object in a mini-batch. <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable parameter vectors
of size <cite>C</cite> (where <cite>C</cite> is the input size) if <code class="xref py py-attr docutils literal notranslate"><span class="pre">affine</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
<p>By default, this layer uses instance statistics computed from input data in
both training and evaluation modes.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">track_running_stats</span></code> is set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, during training this
layer keeps running estimates of its computed mean and variance, which are
then used for normalization during evaluation. The running estimates are
kept with a default <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> of 0.1.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> argument is different from one used in optimizer
classes and the conventional notion of momentum. Mathematically, the
update rule for running statistics here is
<span class="math">\(\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momemtum} \times x_t\)</span>,
where <span class="math">\(\hat{x}\)</span> is the estimated statistic and <span class="math">\(x_t\)</span> is the
new observed value.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.InstanceNorm1d" title="torch.nn.InstanceNorm1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm1d</span></code></a> and <a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> are very similar, but
have some subtle differences. <a class="reference internal" href="#torch.nn.InstanceNorm1d" title="torch.nn.InstanceNorm1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm1d</span></code></a> is applied
on each channel of channeled data like multidimensional time series, but
<a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> is usually applied on entire sample and often in NLP
tasks. Additionaly, <a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> applies elementwise affine
transform, while <a class="reference internal" href="#torch.nn.InstanceNorm1d" title="torch.nn.InstanceNorm1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm1d</span></code></a> usually don’t apply affine
transform.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_features</strong> – <span class="math">\(C\)</span> from an expected input of size
<span class="math">\((N, C, L)\)</span> or <span class="math">\(L\)</span> from input of size <span class="math">\((N, L)\)</span></p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability. Default: 1e-5</p></li>
<li><p><strong>momentum</strong> – the value used for the running_mean and running_var computation. Default: 0.1</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module has
learnable affine parameters, initialized the same way as done for batch normalization.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>track_running_stats</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this
module tracks the running mean and variance, and when set to <code class="docutils literal notranslate"><span class="pre">False</span></code>,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, L)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, L)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">InstanceNorm1d</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">InstanceNorm1d</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">40</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="instancenorm2d">
<h3><span class="hidden-section">InstanceNorm2d</span><a class="headerlink" href="#instancenorm2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.InstanceNorm2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">InstanceNorm2d</code><span class="sig-paren">(</span><em>num_features</em>, <em>eps=1e-05</em>, <em>momentum=0.1</em>, <em>affine=False</em>, <em>track_running_stats=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/instancenorm.html#InstanceNorm2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.InstanceNorm2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Instance Normalization over a 4D input (a mini-batch of 2D inputs
with additional channel dimension) as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1607.08022">Instance Normalization: The Missing Ingredient for Fast Stylization</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta\]</div>
<p>The mean and standard-deviation are calculated per-dimension separately
for each object in a mini-batch. <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable parameter vectors
of size <cite>C</cite> (where <cite>C</cite> is the input size) if <code class="xref py py-attr docutils literal notranslate"><span class="pre">affine</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
<p>By default, this layer uses instance statistics computed from input data in
both training and evaluation modes.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">track_running_stats</span></code> is set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, during training this
layer keeps running estimates of its computed mean and variance, which are
then used for normalization during evaluation. The running estimates are
kept with a default <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> of 0.1.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> argument is different from one used in optimizer
classes and the conventional notion of momentum. Mathematically, the
update rule for running statistics here is
<span class="math">\(\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momemtum} \times x_t\)</span>,
where <span class="math">\(\hat{x}\)</span> is the estimated statistic and <span class="math">\(x_t\)</span> is the
new observed value.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.InstanceNorm2d" title="torch.nn.InstanceNorm2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm2d</span></code></a> and <a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> are very similar, but
have some subtle differences. <a class="reference internal" href="#torch.nn.InstanceNorm2d" title="torch.nn.InstanceNorm2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm2d</span></code></a> is applied
on each channel of channeled data like RGB images, but
<a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> is usually applied on entire sample and often in NLP
tasks. Additionaly, <a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> applies elementwise affine
transform, while <a class="reference internal" href="#torch.nn.InstanceNorm2d" title="torch.nn.InstanceNorm2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm2d</span></code></a> usually don’t apply affine
transform.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_features</strong> – <span class="math">\(C\)</span> from an expected input of size
<span class="math">\((N, C, H, W)\)</span></p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability. Default: 1e-5</p></li>
<li><p><strong>momentum</strong> – the value used for the running_mean and running_var computation. Default: 0.1</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module has
learnable affine parameters, initialized the same way as done for batch normalization.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>track_running_stats</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this
module tracks the running mean and variance, and when set to <code class="docutils literal notranslate"><span class="pre">False</span></code>,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, H, W)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H, W)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">InstanceNorm2d</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">InstanceNorm2d</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">35</span><span class="p">,</span> <span class="mi">45</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="instancenorm3d">
<h3><span class="hidden-section">InstanceNorm3d</span><a class="headerlink" href="#instancenorm3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.InstanceNorm3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">InstanceNorm3d</code><span class="sig-paren">(</span><em>num_features</em>, <em>eps=1e-05</em>, <em>momentum=0.1</em>, <em>affine=False</em>, <em>track_running_stats=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/instancenorm.html#InstanceNorm3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.InstanceNorm3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Instance Normalization over a 5D input (a mini-batch of 3D inputs
with additional channel dimension) as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1607.08022">Instance Normalization: The Missing Ingredient for Fast Stylization</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta\]</div>
<p>The mean and standard-deviation are calculated per-dimension separately
for each object in a mini-batch. <span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable parameter vectors
of size C (where C is the input size) if <code class="xref py py-attr docutils literal notranslate"><span class="pre">affine</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
<p>By default, this layer uses instance statistics computed from input data in
both training and evaluation modes.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">track_running_stats</span></code> is set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, during training this
layer keeps running estimates of its computed mean and variance, which are
then used for normalization during evaluation. The running estimates are
kept with a default <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> of 0.1.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This <code class="xref py py-attr docutils literal notranslate"><span class="pre">momentum</span></code> argument is different from one used in optimizer
classes and the conventional notion of momentum. Mathematically, the
update rule for running statistics here is
<span class="math">\(\hat{x}_\text{new} = (1 - \text{momentum}) \times \hat{x} + \text{momemtum} \times x_t\)</span>,
where <span class="math">\(\hat{x}\)</span> is the estimated statistic and <span class="math">\(x_t\)</span> is the
new observed value.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><a class="reference internal" href="#torch.nn.InstanceNorm3d" title="torch.nn.InstanceNorm3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm3d</span></code></a> and <a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> are very similar, but
have some subtle differences. <a class="reference internal" href="#torch.nn.InstanceNorm3d" title="torch.nn.InstanceNorm3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm3d</span></code></a> is applied
on each channel of channeled data like 3D models with RGB color, but
<a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> is usually applied on entire sample and often in NLP
tasks. Additionaly, <a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> applies elementwise affine
transform, while <a class="reference internal" href="#torch.nn.InstanceNorm3d" title="torch.nn.InstanceNorm3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm3d</span></code></a> usually don’t apply affine
transform.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_features</strong> – <span class="math">\(C\)</span> from an expected input of size
<span class="math">\((N, C, D, H, W)\)</span></p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability. Default: 1e-5</p></li>
<li><p><strong>momentum</strong> – the value used for the running_mean and running_var computation. Default: 0.1</p></li>
<li><p><strong>affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module has
learnable affine parameters, initialized the same way as done for batch normalization.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>track_running_stats</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this
module tracks the running mean and variance, and when set to <code class="docutils literal notranslate"><span class="pre">False</span></code>,
this module does not track such statistics and always uses batch
statistics in both training and eval modes. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, D, H, W)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D, H, W)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">InstanceNorm3d</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">InstanceNorm3d</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="n">affine</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">35</span><span class="p">,</span> <span class="mi">45</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="layernorm">
<h3><span class="hidden-section">LayerNorm</span><a class="headerlink" href="#layernorm" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LayerNorm">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LayerNorm</code><span class="sig-paren">(</span><em>normalized_shape</em>, <em>eps=1e-05</em>, <em>elementwise_affine=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/normalization.html#LayerNorm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LayerNorm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Layer Normalization over a mini-batch of inputs as described in
the paper <a class="reference external" href="https://arxiv.org/abs/1607.06450">Layer Normalization</a> .</p>
<div class="math">
\[y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta

\]</div>
<p>The mean and standard-deviation are calculated separately over the last
certain number dimensions which have to be of the shape specified by
<code class="xref py py-attr docutils literal notranslate"><span class="pre">normalized_shape</span></code>.
<span class="math">\(\gamma\)</span> and <span class="math">\(\beta\)</span> are learnable affine transform parameters of
<code class="xref py py-attr docutils literal notranslate"><span class="pre">normalized_shape</span></code> if <code class="xref py py-attr docutils literal notranslate"><span class="pre">elementwise_affine</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Unlike Batch Normalization and Instance Normalization, which applies
scalar scale and bias for each entire channel/plane with the
<code class="xref py py-attr docutils literal notranslate"><span class="pre">affine</span></code> option, Layer Normalization applies per-element scale and
bias with <code class="xref py py-attr docutils literal notranslate"><span class="pre">elementwise_affine</span></code>.</p>
</div>
<p>This layer uses statistics computed from input data in both training and
evaluation modes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>normalized_shape</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a><em> or </em><em>torch.Size</em>) – <p>input shape from an expected input
of size</p>
<div class="math">
\[[* \times \text{normalized\_shape}[0] \times \text{normalized\_shape}[1]
    \times \ldots \times \text{normalized\_shape}[-1]]

\]</div>
<p>If a single integer is used, it is treated as a singleton list, and this module will
normalize over the last dimension which is expected to be of that specific size.</p>
</p></li>
<li><p><strong>eps</strong> – a value added to the denominator for numerical stability. Default: 1e-5</p></li>
<li><p><strong>elementwise_affine</strong> – a boolean value that when set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, this module
has learnable per-element affine parameters initialized to ones (for weights)
and zeros (for biases). Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span></p></li>
<li><p>Output: <span class="math">\((N, *)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># With Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LayerNorm</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">()[</span><span class="mi">1</span><span class="p">:])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Without Learnable Parameters</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LayerNorm</span><span class="p">(</span><span class="nb">input</span><span class="o">.</span><span class="n">size</span><span class="p">()[</span><span class="mi">1</span><span class="p">:],</span> <span class="n">elementwise_affine</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Normalize over last two dimensions</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LayerNorm</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Normalize over last dimension of size 10</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LayerNorm</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Activating the module</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="localresponsenorm">
<h3><span class="hidden-section">LocalResponseNorm</span><a class="headerlink" href="#localresponsenorm" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LocalResponseNorm">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LocalResponseNorm</code><span class="sig-paren">(</span><em>size</em>, <em>alpha=0.0001</em>, <em>beta=0.75</em>, <em>k=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/normalization.html#LocalResponseNorm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LocalResponseNorm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies local response normalization over an input signal composed
of several input planes, where channels occupy the second dimension.
Applies normalization across channels.</p>
<div class="math">
\[b_{c} = a_{c}\left(k + \frac{\alpha}{n}
\sum_{c'=\max(0, c-n/2)}^{\min(N-1,c+n/2)}a_{c'}^2\right)^{-\beta}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size</strong> – amount of neighbouring channels used for normalization</p></li>
<li><p><strong>alpha</strong> – multiplicative factor. Default: 0.0001</p></li>
<li><p><strong>beta</strong> – exponent. Default: 0.75</p></li>
<li><p><strong>k</strong> – additive factor. Default: 1</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, *)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, *)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">lrn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LocalResponseNorm</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">signal_2d</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">32</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">24</span><span class="p">,</span> <span class="mi">24</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">signal_4d</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">7</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output_2d</span> <span class="o">=</span> <span class="n">lrn</span><span class="p">(</span><span class="n">signal_2d</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output_4d</span> <span class="o">=</span> <span class="n">lrn</span><span class="p">(</span><span class="n">signal_4d</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="recurrent-layers">
<h2>Recurrent layers<a class="headerlink" href="#recurrent-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="rnn">
<h3><span class="hidden-section">RNN</span><a class="headerlink" href="#rnn" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.RNN">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">RNN</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/rnn.html#RNN"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.RNN" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a multi-layer Elman RNN with <span class="math">\(tanh\)</span> or <span class="math">\(ReLU\)</span> non-linearity to an
input sequence.</p>
<p>For each element in the input sequence, each layer computes the following
function:</p>
<div class="math">
\[h_t = \text{tanh}(W_{ih} x_t + b_{ih} + W_{hh} h_{(t-1)} + b_{hh})

\]</div>
<p>where <span class="math">\(h_t\)</span> is the hidden state at time <cite>t</cite>, <span class="math">\(x_t\)</span> is
the input at time <cite>t</cite>, and <span class="math">\(h_{(t-1)}\)</span> is the hidden state of the
previous layer at time <cite>t-1</cite> or the initial hidden state at time <cite>0</cite>.
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">nonlinearity</span></code> is <code class="docutils literal notranslate"><span class="pre">'relu'</span></code>, then <cite>ReLU</cite> is used instead of <cite>tanh</cite>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input_size</strong> – The number of expected features in the input <cite>x</cite></p></li>
<li><p><strong>hidden_size</strong> – The number of features in the hidden state <cite>h</cite></p></li>
<li><p><strong>num_layers</strong> – Number of recurrent layers. E.g., setting <code class="docutils literal notranslate"><span class="pre">num_layers=2</span></code>
would mean stacking two RNNs together to form a <cite>stacked RNN</cite>,
with the second RNN taking in outputs of the first RNN and
computing the final results. Default: 1</p></li>
<li><p><strong>nonlinearity</strong> – The non-linearity to use. Can be either <code class="docutils literal notranslate"><span class="pre">'tanh'</span></code> or <code class="docutils literal notranslate"><span class="pre">'relu'</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'tanh'</span></code></p></li>
<li><p><strong>bias</strong> – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, then the layer does not use bias weights <cite>b_ih</cite> and <cite>b_hh</cite>.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>batch_first</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, then the input and output tensors are provided
as <cite>(batch, seq, feature)</cite>. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
<li><p><strong>dropout</strong> – If non-zero, introduces a <cite>Dropout</cite> layer on the outputs of each
RNN layer except the last layer, with dropout probability equal to
<code class="xref py py-attr docutils literal notranslate"><span class="pre">dropout</span></code>. Default: 0</p></li>
<li><p><strong>bidirectional</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, becomes a bidirectional RNN. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Inputs: input, h_0</dt><dd><ul class="simple">
<li><p><strong>input</strong> of shape <cite>(seq_len, batch, input_size)</cite>: tensor containing the features
of the input sequence. The input can also be a packed variable length
sequence. See <a class="reference internal" href="#torch.nn.utils.rnn.pack_padded_sequence" title="torch.nn.utils.rnn.pack_padded_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.pack_padded_sequence()</span></code></a>
or <a class="reference internal" href="#torch.nn.utils.rnn.pack_sequence" title="torch.nn.utils.rnn.pack_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.pack_sequence()</span></code></a>
for details.</p></li>
<li><p><strong>h_0</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the initial hidden state for each element in the batch.
Defaults to zero if not provided. If the RNN is bidirectional,
num_directions should be 2, else it should be 1.</p></li>
</ul>
</dd>
<dt>Outputs: output, h_n</dt><dd><ul>
<li><p><strong>output</strong> of shape <cite>(seq_len, batch, num_directions * hidden_size)</cite>: tensor
containing the output features (<cite>h_t</cite>) from the last layer of the RNN,
for each <cite>t</cite>.  If a <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.PackedSequence</span></code></a> has
been given as the input, the output will also be a packed sequence.</p>
<p>For the unpacked case, the directions can be separated
using <code class="docutils literal notranslate"><span class="pre">output.view(seq_len,</span> <span class="pre">batch,</span> <span class="pre">num_directions,</span> <span class="pre">hidden_size)</span></code>,
with forward and backward being direction <cite>0</cite> and <cite>1</cite> respectively.
Similarly, the directions can be separated in the packed case.</p>
</li>
<li><p><strong>h_n</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the hidden state for <cite>t = seq_len</cite>.</p>
<p>Like <em>output</em>, the layers can be separated using
<code class="docutils literal notranslate"><span class="pre">h_n.view(num_layers,</span> <span class="pre">num_directions,</span> <span class="pre">batch,</span> <span class="pre">hidden_size)</span></code>.</p>
</li>
</ul>
</dd>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input1: <span class="math">\((L, N, H_{in})\)</span> tensor containing input features where
<span class="math">\(H_{in}=\text{input\_size}\)</span> and <cite>L</cite> represents a sequence length.</p></li>
<li><p>Input2: <span class="math">\((S, N, H_{out})\)</span> tensor
containing the initial hidden state for each element in the batch.
<span class="math">\(H_{out}=\text{hidden\_size}\)</span>
Defaults to zero if not provided. where <span class="math">\(S=\text{num\_layers} * \text{num\_directions}\)</span>
If the RNN is bidirectional, num_directions should be 2, else it should be 1.</p></li>
<li><p>Output1: <span class="math">\((L, N, H_{all})\)</span> where <span class="math">\(H_all=\text{num\_directions} * \text{hidden\_size}\)</span></p></li>
<li><p>Output2: <span class="math">\((S, N, H_{out})\)</span> tensor containing the next hidden state
for each element in the batch</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~RNN.weight_ih_l[k]</strong> – the learnable input-hidden weights of the k-th layer,
of shape <cite>(hidden_size, input_size)</cite> for <cite>k = 0</cite>. Otherwise, the shape is
<cite>(hidden_size, num_directions * hidden_size)</cite></p></li>
<li><p><strong>~RNN.weight_hh_l[k]</strong> – the learnable hidden-hidden weights of the k-th layer,
of shape <cite>(hidden_size, hidden_size)</cite></p></li>
<li><p><strong>~RNN.bias_ih_l[k]</strong> – the learnable input-hidden bias of the k-th layer,
of shape <cite>(hidden_size)</cite></p></li>
<li><p><strong>~RNN.bias_hh_l[k]</strong> – the learnable hidden-hidden bias of the k-th layer,
of shape <cite>(hidden_size)</cite></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>All the weights and biases are initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>
where <span class="math">\(k = \frac{1}{\text{hidden\_size}}\)</span></p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If the following conditions are satisfied:
1) cudnn is enabled,
2) input data is on the GPU
3) input data has dtype <code class="docutils literal notranslate"><span class="pre">torch.float16</span></code>
4) V100 GPU is used,
5) input data is not in <code class="docutils literal notranslate"><span class="pre">PackedSequence</span></code> format
persistent algorithm can be selected to improve performance.</p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">RNN</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">h0</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="p">,</span> <span class="n">hn</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">h0</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="lstm">
<h3><span class="hidden-section">LSTM</span><a class="headerlink" href="#lstm" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LSTM">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LSTM</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/rnn.html#LSTM"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LSTM" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a multi-layer long short-term memory (LSTM) RNN to an input
sequence.</p>
<p>For each element in the input sequence, each layer computes the following
function:</p>
<div class="math">
\[\begin{array}{ll} \\
    i_t = \sigma(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\
    f_t = \sigma(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\
    g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{(t-1)} + b_{hg}) \\
    o_t = \sigma(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\
    c_t = f_t * c_{(t-1)} + i_t * g_t \\
    h_t = o_t * \tanh(c_t) \\
\end{array}

\]</div>
<p>where <span class="math">\(h_t\)</span> is the hidden state at time <cite>t</cite>, <span class="math">\(c_t\)</span> is the cell
state at time <cite>t</cite>, <span class="math">\(x_t\)</span> is the input at time <cite>t</cite>, <span class="math">\(h_{(t-1)}\)</span>
is the hidden state of the layer at time <cite>t-1</cite> or the initial hidden
state at time <cite>0</cite>, and <span class="math">\(i_t\)</span>, <span class="math">\(f_t\)</span>, <span class="math">\(g_t\)</span>,
<span class="math">\(o_t\)</span> are the input, forget, cell, and output gates, respectively.
<span class="math">\(\sigma\)</span> is the sigmoid function, and <span class="math">\(*\)</span> is the Hadamard product.</p>
<p>In a multilayer LSTM, the input <span class="math">\(x^{(l)}_t\)</span> of the <span class="math">\(l\)</span> -th layer
(<span class="math">\(l &gt;= 2\)</span>) is the hidden state <span class="math">\(h^{(l-1)}_t\)</span> of the previous layer multiplied by
dropout <span class="math">\(\delta^{(l-1)}_t\)</span> where each <span class="math">\(\delta^{(l-1)}_t\)</span> is a Bernoulli random
variable which is <span class="math">\(0\)</span> with probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">dropout</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input_size</strong> – The number of expected features in the input <cite>x</cite></p></li>
<li><p><strong>hidden_size</strong> – The number of features in the hidden state <cite>h</cite></p></li>
<li><p><strong>num_layers</strong> – Number of recurrent layers. E.g., setting <code class="docutils literal notranslate"><span class="pre">num_layers=2</span></code>
would mean stacking two LSTMs together to form a <cite>stacked LSTM</cite>,
with the second LSTM taking in outputs of the first LSTM and
computing the final results. Default: 1</p></li>
<li><p><strong>bias</strong> – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, then the layer does not use bias weights <cite>b_ih</cite> and <cite>b_hh</cite>.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>batch_first</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, then the input and output tensors are provided
as (batch, seq, feature). Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
<li><p><strong>dropout</strong> – If non-zero, introduces a <cite>Dropout</cite> layer on the outputs of each
LSTM layer except the last layer, with dropout probability equal to
<code class="xref py py-attr docutils literal notranslate"><span class="pre">dropout</span></code>. Default: 0</p></li>
<li><p><strong>bidirectional</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, becomes a bidirectional LSTM. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Inputs: input, (h_0, c_0)</dt><dd><ul>
<li><p><strong>input</strong> of shape <cite>(seq_len, batch, input_size)</cite>: tensor containing the features
of the input sequence.
The input can also be a packed variable length sequence.
See <a class="reference internal" href="#torch.nn.utils.rnn.pack_padded_sequence" title="torch.nn.utils.rnn.pack_padded_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.pack_padded_sequence()</span></code></a> or
<a class="reference internal" href="#torch.nn.utils.rnn.pack_sequence" title="torch.nn.utils.rnn.pack_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.pack_sequence()</span></code></a> for details.</p></li>
<li><p><strong>h_0</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the initial hidden state for each element in the batch.
If the LSTM is bidirectional, num_directions should be 2, else it should be 1.</p></li>
<li><p><strong>c_0</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the initial cell state for each element in the batch.</p>
<p>If <cite>(h_0, c_0)</cite> is not provided, both <strong>h_0</strong> and <strong>c_0</strong> default to zero.</p>
</li>
</ul>
</dd>
<dt>Outputs: output, (h_n, c_n)</dt><dd><ul>
<li><p><strong>output</strong> of shape <cite>(seq_len, batch, num_directions * hidden_size)</cite>: tensor
containing the output features <cite>(h_t)</cite> from the last layer of the LSTM,
for each <cite>t</cite>. If a <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.PackedSequence</span></code></a> has been
given as the input, the output will also be a packed sequence.</p>
<p>For the unpacked case, the directions can be separated
using <code class="docutils literal notranslate"><span class="pre">output.view(seq_len,</span> <span class="pre">batch,</span> <span class="pre">num_directions,</span> <span class="pre">hidden_size)</span></code>,
with forward and backward being direction <cite>0</cite> and <cite>1</cite> respectively.
Similarly, the directions can be separated in the packed case.</p>
</li>
<li><p><strong>h_n</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the hidden state for <cite>t = seq_len</cite>.</p>
<p>Like <em>output</em>, the layers can be separated using
<code class="docutils literal notranslate"><span class="pre">h_n.view(num_layers,</span> <span class="pre">num_directions,</span> <span class="pre">batch,</span> <span class="pre">hidden_size)</span></code> and similarly for <em>c_n</em>.</p>
</li>
<li><p><strong>c_n</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the cell state for <cite>t = seq_len</cite>.</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~LSTM.weight_ih_l[k]</strong> – the learnable input-hidden weights of the <span class="math">\(\text{k}^{th}\)</span> layer
<cite>(W_ii|W_if|W_ig|W_io)</cite>, of shape <cite>(4*hidden_size, input_size)</cite> for <cite>k = 0</cite>.
Otherwise, the shape is <cite>(4*hidden_size, num_directions * hidden_size)</cite></p></li>
<li><p><strong>~LSTM.weight_hh_l[k]</strong> – the learnable hidden-hidden weights of the <span class="math">\(\text{k}^{th}\)</span> layer
<cite>(W_hi|W_hf|W_hg|W_ho)</cite>, of shape <cite>(4*hidden_size, hidden_size)</cite></p></li>
<li><p><strong>~LSTM.bias_ih_l[k]</strong> – the learnable input-hidden bias of the <span class="math">\(\text{k}^{th}\)</span> layer
<cite>(b_ii|b_if|b_ig|b_io)</cite>, of shape <cite>(4*hidden_size)</cite></p></li>
<li><p><strong>~LSTM.bias_hh_l[k]</strong> – the learnable hidden-hidden bias of the <span class="math">\(\text{k}^{th}\)</span> layer
<cite>(b_hi|b_hf|b_hg|b_ho)</cite>, of shape <cite>(4*hidden_size)</cite></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>All the weights and biases are initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>
where <span class="math">\(k = \frac{1}{\text{hidden\_size}}\)</span></p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If the following conditions are satisfied:
1) cudnn is enabled,
2) input data is on the GPU
3) input data has dtype <code class="docutils literal notranslate"><span class="pre">torch.float16</span></code>
4) V100 GPU is used,
5) input data is not in <code class="docutils literal notranslate"><span class="pre">PackedSequence</span></code> format
persistent algorithm can be selected to improve performance.</p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LSTM</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">h0</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">c0</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="p">,</span> <span class="p">(</span><span class="n">hn</span><span class="p">,</span> <span class="n">cn</span><span class="p">)</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="p">(</span><span class="n">h0</span><span class="p">,</span> <span class="n">c0</span><span class="p">))</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="gru">
<h3><span class="hidden-section">GRU</span><a class="headerlink" href="#gru" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.GRU">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">GRU</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/rnn.html#GRU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.GRU" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a multi-layer gated recurrent unit (GRU) RNN to an input sequence.</p>
<p>For each element in the input sequence, each layer computes the following
function:</p>
<div class="math">
\[\begin{array}{ll}
    r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\
    z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\
    n_t = \tanh(W_{in} x_t + b_{in} + r_t * (W_{hn} h_{(t-1)}+ b_{hn})) \\
    h_t = (1 - z_t) * n_t + z_t * h_{(t-1)}
\end{array}

\]</div>
<p>where <span class="math">\(h_t\)</span> is the hidden state at time <cite>t</cite>, <span class="math">\(x_t\)</span> is the input
at time <cite>t</cite>, <span class="math">\(h_{(t-1)}\)</span> is the hidden state of the layer
at time <cite>t-1</cite> or the initial hidden state at time <cite>0</cite>, and <span class="math">\(r_t\)</span>,
<span class="math">\(z_t\)</span>, <span class="math">\(n_t\)</span> are the reset, update, and new gates, respectively.
<span class="math">\(\sigma\)</span> is the sigmoid function, and <span class="math">\(*\)</span> is the Hadamard product.</p>
<p>In a multilayer GRU, the input <span class="math">\(x^{(l)}_t\)</span> of the <span class="math">\(l\)</span> -th layer
(<span class="math">\(l &gt;= 2\)</span>) is the hidden state <span class="math">\(h^{(l-1)}_t\)</span> of the previous layer multiplied by
dropout <span class="math">\(\delta^{(l-1)}_t\)</span> where each <span class="math">\(\delta^{(l-1)}_t\)</span> is a Bernoulli random
variable which is <span class="math">\(0\)</span> with probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">dropout</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input_size</strong> – The number of expected features in the input <cite>x</cite></p></li>
<li><p><strong>hidden_size</strong> – The number of features in the hidden state <cite>h</cite></p></li>
<li><p><strong>num_layers</strong> – Number of recurrent layers. E.g., setting <code class="docutils literal notranslate"><span class="pre">num_layers=2</span></code>
would mean stacking two GRUs together to form a <cite>stacked GRU</cite>,
with the second GRU taking in outputs of the first GRU and
computing the final results. Default: 1</p></li>
<li><p><strong>bias</strong> – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, then the layer does not use bias weights <cite>b_ih</cite> and <cite>b_hh</cite>.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>batch_first</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, then the input and output tensors are provided
as (batch, seq, feature). Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
<li><p><strong>dropout</strong> – If non-zero, introduces a <cite>Dropout</cite> layer on the outputs of each
GRU layer except the last layer, with dropout probability equal to
<code class="xref py py-attr docutils literal notranslate"><span class="pre">dropout</span></code>. Default: 0</p></li>
<li><p><strong>bidirectional</strong> – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, becomes a bidirectional GRU. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Inputs: input, h_0</dt><dd><ul class="simple">
<li><p><strong>input</strong> of shape <cite>(seq_len, batch, input_size)</cite>: tensor containing the features
of the input sequence. The input can also be a packed variable length
sequence. See <a class="reference internal" href="#torch.nn.utils.rnn.pack_padded_sequence" title="torch.nn.utils.rnn.pack_padded_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.pack_padded_sequence()</span></code></a>
for details.</p></li>
<li><p><strong>h_0</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the initial hidden state for each element in the batch.
Defaults to zero if not provided. If the RNN is bidirectional,
num_directions should be 2, else it should be 1.</p></li>
</ul>
</dd>
<dt>Outputs: output, h_n</dt><dd><ul>
<li><p><strong>output</strong> of shape <cite>(seq_len, batch, num_directions * hidden_size)</cite>: tensor
containing the output features h_t from the last layer of the GRU,
for each <cite>t</cite>. If a <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.utils.rnn.PackedSequence</span></code></a> has been
given as the input, the output will also be a packed sequence.
For the unpacked case, the directions can be separated
using <code class="docutils literal notranslate"><span class="pre">output.view(seq_len,</span> <span class="pre">batch,</span> <span class="pre">num_directions,</span> <span class="pre">hidden_size)</span></code>,
with forward and backward being direction <cite>0</cite> and <cite>1</cite> respectively.</p>
<p>Similarly, the directions can be separated in the packed case.</p>
</li>
<li><p><strong>h_n</strong> of shape <cite>(num_layers * num_directions, batch, hidden_size)</cite>: tensor
containing the hidden state for <cite>t = seq_len</cite></p>
<p>Like <em>output</em>, the layers can be separated using
<code class="docutils literal notranslate"><span class="pre">h_n.view(num_layers,</span> <span class="pre">num_directions,</span> <span class="pre">batch,</span> <span class="pre">hidden_size)</span></code>.</p>
</li>
</ul>
</dd>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input1: <span class="math">\((L, N, H_{in})\)</span> tensor containing input features where
<span class="math">\(H_{in}=\text{input\_size}\)</span> and <cite>L</cite> represents a sequence length.</p></li>
<li><p>Input2: <span class="math">\((S, N, H_{out})\)</span> tensor
containing the initial hidden state for each element in the batch.
<span class="math">\(H_{out}=\text{hidden\_size}\)</span>
Defaults to zero if not provided. where <span class="math">\(S=\text{num\_layers} * \text{num\_directions}\)</span>
If the RNN is bidirectional, num_directions should be 2, else it should be 1.</p></li>
<li><p>Output1: <span class="math">\((L, N, H_{all})\)</span> where <span class="math">\(H_all=\text{num\_directions} * \text{hidden\_size}\)</span></p></li>
<li><p>Output2: <span class="math">\((S, N, H_{out})\)</span> tensor containing the next hidden state
for each element in the batch</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~GRU.weight_ih_l[k]</strong> – the learnable input-hidden weights of the <span class="math">\(\text{k}^{th}\)</span> layer
(W_ir|W_iz|W_in), of shape <cite>(3*hidden_size, input_size)</cite> for <cite>k = 0</cite>.
Otherwise, the shape is <cite>(3*hidden_size, num_directions * hidden_size)</cite></p></li>
<li><p><strong>~GRU.weight_hh_l[k]</strong> – the learnable hidden-hidden weights of the <span class="math">\(\text{k}^{th}\)</span> layer
(W_hr|W_hz|W_hn), of shape <cite>(3*hidden_size, hidden_size)</cite></p></li>
<li><p><strong>~GRU.bias_ih_l[k]</strong> – the learnable input-hidden bias of the <span class="math">\(\text{k}^{th}\)</span> layer
(b_ir|b_iz|b_in), of shape <cite>(3*hidden_size)</cite></p></li>
<li><p><strong>~GRU.bias_hh_l[k]</strong> – the learnable hidden-hidden bias of the <span class="math">\(\text{k}^{th}\)</span> layer
(b_hr|b_hz|b_hn), of shape <cite>(3*hidden_size)</cite></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>All the weights and biases are initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>
where <span class="math">\(k = \frac{1}{\text{hidden\_size}}\)</span></p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If the following conditions are satisfied:
1) cudnn is enabled,
2) input data is on the GPU
3) input data has dtype <code class="docutils literal notranslate"><span class="pre">torch.float16</span></code>
4) V100 GPU is used,
5) input data is not in <code class="docutils literal notranslate"><span class="pre">PackedSequence</span></code> format
persistent algorithm can be selected to improve performance.</p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRU</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">h0</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="p">,</span> <span class="n">hn</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">h0</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="rnncell">
<h3><span class="hidden-section">RNNCell</span><a class="headerlink" href="#rnncell" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.RNNCell">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">RNNCell</code><span class="sig-paren">(</span><em>input_size</em>, <em>hidden_size</em>, <em>bias=True</em>, <em>nonlinearity='tanh'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/rnn.html#RNNCell"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.RNNCell" title="Permalink to this definition">¶</a></dt>
<dd><p>An Elman RNN cell with tanh or ReLU non-linearity.</p>
<div class="math">
\[h' = \tanh(W_{ih} x + b_{ih}  +  W_{hh} h + b_{hh})\]</div>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">nonlinearity</span></code> is <cite>‘relu’</cite>, then ReLU is used in place of tanh.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input_size</strong> – The number of expected features in the input <cite>x</cite></p></li>
<li><p><strong>hidden_size</strong> – The number of features in the hidden state <cite>h</cite></p></li>
<li><p><strong>bias</strong> – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, then the layer does not use bias weights <cite>b_ih</cite> and <cite>b_hh</cite>.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>nonlinearity</strong> – The non-linearity to use. Can be either <code class="docutils literal notranslate"><span class="pre">'tanh'</span></code> or <code class="docutils literal notranslate"><span class="pre">'relu'</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'tanh'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Inputs: input, hidden</dt><dd><ul class="simple">
<li><p><strong>input</strong> of shape <cite>(batch, input_size)</cite>: tensor containing input features</p></li>
<li><p><strong>hidden</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the initial hidden
state for each element in the batch.
Defaults to zero if not provided.</p></li>
</ul>
</dd>
<dt>Outputs: h’</dt><dd><ul class="simple">
<li><p><strong>h’</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the next hidden state
for each element in the batch</p></li>
</ul>
</dd>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input1: <span class="math">\((N, H_{in})\)</span> tensor containing input features where
<span class="math">\(H_{in}\)</span> = <cite>input_size</cite></p></li>
<li><p>Input2: <span class="math">\((N, H_{out})\)</span> tensor containing the initial hidden
state for each element in the batch where <span class="math">\(H_{out}\)</span> = <cite>hidden_size</cite>
Defaults to zero if not provided.</p></li>
<li><p>Output: <span class="math">\((N, H_{out})\)</span> tensor containing the next hidden state
for each element in the batch</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~RNNCell.weight_ih</strong> – the learnable input-hidden weights, of shape
<cite>(hidden_size, input_size)</cite></p></li>
<li><p><strong>~RNNCell.weight_hh</strong> – the learnable hidden-hidden weights, of shape
<cite>(hidden_size, hidden_size)</cite></p></li>
<li><p><strong>~RNNCell.bias_ih</strong> – the learnable input-hidden bias, of shape <cite>(hidden_size)</cite></p></li>
<li><p><strong>~RNNCell.bias_hh</strong> – the learnable hidden-hidden bias, of shape <cite>(hidden_size)</cite></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>All the weights and biases are initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>
where <span class="math">\(k = \frac{1}{\text{hidden\_size}}\)</span></p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">RNNCell</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hx</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="p">[]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">6</span><span class="p">):</span>
<span class="go">        hx = rnn(input[i], hx)</span>
<span class="go">        output.append(hx)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="lstmcell">
<h3><span class="hidden-section">LSTMCell</span><a class="headerlink" href="#lstmcell" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.LSTMCell">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">LSTMCell</code><span class="sig-paren">(</span><em>input_size</em>, <em>hidden_size</em>, <em>bias=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/rnn.html#LSTMCell"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.LSTMCell" title="Permalink to this definition">¶</a></dt>
<dd><p>A long short-term memory (LSTM) cell.</p>
<div class="math">
\[\begin{array}{ll}
i = \sigma(W_{ii} x + b_{ii} + W_{hi} h + b_{hi}) \\
f = \sigma(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\
g = \tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\
o = \sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\
c' = f * c + i * g \\
h' = o * \tanh(c') \\
\end{array}\]</div>
<p>where <span class="math">\(\sigma\)</span> is the sigmoid function, and <span class="math">\(*\)</span> is the Hadamard product.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input_size</strong> – The number of expected features in the input <cite>x</cite></p></li>
<li><p><strong>hidden_size</strong> – The number of features in the hidden state <cite>h</cite></p></li>
<li><p><strong>bias</strong> – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, then the layer does not use bias weights <cite>b_ih</cite> and
<cite>b_hh</cite>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Inputs: input, (h_0, c_0)</dt><dd><ul>
<li><p><strong>input</strong> of shape <cite>(batch, input_size)</cite>: tensor containing input features</p></li>
<li><p><strong>h_0</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the initial hidden
state for each element in the batch.</p></li>
<li><p><strong>c_0</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the initial cell state
for each element in the batch.</p>
<p>If <cite>(h_0, c_0)</cite> is not provided, both <strong>h_0</strong> and <strong>c_0</strong> default to zero.</p>
</li>
</ul>
</dd>
<dt>Outputs: (h_1, c_1)</dt><dd><ul class="simple">
<li><p><strong>h_1</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the next hidden state
for each element in the batch</p></li>
<li><p><strong>c_1</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the next cell state
for each element in the batch</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~LSTMCell.weight_ih</strong> – the learnable input-hidden weights, of shape
<cite>(4*hidden_size, input_size)</cite></p></li>
<li><p><strong>~LSTMCell.weight_hh</strong> – the learnable hidden-hidden weights, of shape
<cite>(4*hidden_size, hidden_size)</cite></p></li>
<li><p><strong>~LSTMCell.bias_ih</strong> – the learnable input-hidden bias, of shape <cite>(4*hidden_size)</cite></p></li>
<li><p><strong>~LSTMCell.bias_hh</strong> – the learnable hidden-hidden bias, of shape <cite>(4*hidden_size)</cite></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>All the weights and biases are initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>
where <span class="math">\(k = \frac{1}{\text{hidden\_size}}\)</span></p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LSTMCell</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hx</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cx</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="p">[]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">6</span><span class="p">):</span>
<span class="go">        hx, cx = rnn(input[i], (hx, cx))</span>
<span class="go">        output.append(hx)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="grucell">
<h3><span class="hidden-section">GRUCell</span><a class="headerlink" href="#grucell" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.GRUCell">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">GRUCell</code><span class="sig-paren">(</span><em>input_size</em>, <em>hidden_size</em>, <em>bias=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/rnn.html#GRUCell"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.GRUCell" title="Permalink to this definition">¶</a></dt>
<dd><p>A gated recurrent unit (GRU) cell</p>
<div class="math">
\[\begin{array}{ll}
r = \sigma(W_{ir} x + b_{ir} + W_{hr} h + b_{hr}) \\
z = \sigma(W_{iz} x + b_{iz} + W_{hz} h + b_{hz}) \\
n = \tanh(W_{in} x + b_{in} + r * (W_{hn} h + b_{hn})) \\
h' = (1 - z) * n + z * h
\end{array}\]</div>
<p>where <span class="math">\(\sigma\)</span> is the sigmoid function, and <span class="math">\(*\)</span> is the Hadamard product.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input_size</strong> – The number of expected features in the input <cite>x</cite></p></li>
<li><p><strong>hidden_size</strong> – The number of features in the hidden state <cite>h</cite></p></li>
<li><p><strong>bias</strong> – If <code class="docutils literal notranslate"><span class="pre">False</span></code>, then the layer does not use bias weights <cite>b_ih</cite> and
<cite>b_hh</cite>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Inputs: input, hidden</dt><dd><ul class="simple">
<li><p><strong>input</strong> of shape <cite>(batch, input_size)</cite>: tensor containing input features</p></li>
<li><p><strong>hidden</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the initial hidden
state for each element in the batch.
Defaults to zero if not provided.</p></li>
</ul>
</dd>
<dt>Outputs: h’</dt><dd><ul class="simple">
<li><p><strong>h’</strong> of shape <cite>(batch, hidden_size)</cite>: tensor containing the next hidden state
for each element in the batch</p></li>
</ul>
</dd>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input1: <span class="math">\((N, H_{in})\)</span> tensor containing input features where
<span class="math">\(H_{in}\)</span> = <cite>input_size</cite></p></li>
<li><p>Input2: <span class="math">\((N, H_{out})\)</span> tensor containing the initial hidden
state for each element in the batch where <span class="math">\(H_{out}\)</span> = <cite>hidden_size</cite>
Defaults to zero if not provided.</p></li>
<li><p>Output: <span class="math">\((N, H_{out})\)</span> tensor containing the next hidden state
for each element in the batch</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~GRUCell.weight_ih</strong> – the learnable input-hidden weights, of shape
<cite>(3*hidden_size, input_size)</cite></p></li>
<li><p><strong>~GRUCell.weight_hh</strong> – the learnable hidden-hidden weights, of shape
<cite>(3*hidden_size, hidden_size)</cite></p></li>
<li><p><strong>~GRUCell.bias_ih</strong> – the learnable input-hidden bias, of shape <cite>(3*hidden_size)</cite></p></li>
<li><p><strong>~GRUCell.bias_hh</strong> – the learnable hidden-hidden bias, of shape <cite>(3*hidden_size)</cite></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>All the weights and biases are initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>
where <span class="math">\(k = \frac{1}{\text{hidden\_size}}\)</span></p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">rnn</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">GRUCell</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hx</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="p">[]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">6</span><span class="p">):</span>
<span class="go">        hx = rnn(input[i], hx)</span>
<span class="go">        output.append(hx)</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="linear-layers">
<h2>Linear layers<a class="headerlink" href="#linear-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="identity">
<h3><span class="hidden-section">Identity</span><a class="headerlink" href="#identity" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Identity">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Identity</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/linear.html#Identity"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Identity" title="Permalink to this definition">¶</a></dt>
<dd><p>A placeholder identity operator that is argument-insensitive.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>args</strong> – any argument (unused)</p></li>
<li><p><strong>kwargs</strong> – any keyword argument (unused)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Identity</span><span class="p">(</span><span class="mi">54</span><span class="p">,</span> <span class="n">unused_argument1</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">unused_argument2</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([128, 20])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="linear">
<h3><span class="hidden-section">Linear</span><a class="headerlink" href="#linear" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Linear">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Linear</code><span class="sig-paren">(</span><em>in_features</em>, <em>out_features</em>, <em>bias=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/linear.html#Linear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Linear" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a linear transformation to the incoming data: <span class="math">\(y = xA^T + b\)</span></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in_features</strong> – size of each input sample</p></li>
<li><p><strong>out_features</strong> – size of each output sample</p></li>
<li><p><strong>bias</strong> – If set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the layer will not learn an additive bias.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *, H_{in})\)</span> where <span class="math">\(*\)</span> means any number of
additional dimensions and <span class="math">\(H_{in} = \text{in\_features}\)</span></p></li>
<li><p>Output: <span class="math">\((N, *, H_{out})\)</span> where all but the last dimension
are the same shape as the input and <span class="math">\(H_{out} = \text{out\_features}\)</span>.</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~Linear.weight</strong> – the learnable weights of the module of shape
<span class="math">\((\text{out\_features}, \text{in\_features})\)</span>. The values are
initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>, where
<span class="math">\(k = \frac{1}{\text{in\_features}}\)</span></p></li>
<li><p><strong>~Linear.bias</strong> – the learnable bias of the module of shape <span class="math">\((\text{out\_features})\)</span>.
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, the values are initialized from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span> where
<span class="math">\(k = \frac{1}{\text{in\_features}}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([128, 30])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="bilinear">
<h3><span class="hidden-section">Bilinear</span><a class="headerlink" href="#bilinear" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Bilinear">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Bilinear</code><span class="sig-paren">(</span><em>in1_features</em>, <em>in2_features</em>, <em>out_features</em>, <em>bias=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/linear.html#Bilinear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Bilinear" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a bilinear transformation to the incoming data:
<span class="math">\(y = x_1 A x_2 + b\)</span></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>in1_features</strong> – size of each first input sample</p></li>
<li><p><strong>in2_features</strong> – size of each second input sample</p></li>
<li><p><strong>out_features</strong> – size of each output sample</p></li>
<li><p><strong>bias</strong> – If set to False, the layer will not learn an additive bias.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input1: <span class="math">\((N, *, H_{in1})\)</span> where <span class="math">\(H_{in1}=\text{in1\_features}\)</span> and
<span class="math">\(*\)</span> means any number of additional dimensions. All but the last dimension
of the inputs should be the same.</p></li>
<li><p>Input2: <span class="math">\((N, *, H_{in2})\)</span> where <span class="math">\(H_{in2}=\text{in2\_features}\)</span>.</p></li>
<li><p>Output: <span class="math">\((N, *, H_{out})\)</span> where <span class="math">\(H_{out}=\text{out\_features}\)</span>
and all but the last dimension are the same shape as the input.</p></li>
</ul>
</dd>
</dl>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~Bilinear.weight</strong> – the learnable weights of the module of shape
<span class="math">\((\text{out\_features}, \text{in1\_features}, \text{in2\_features})\)</span>.
The values are initialized from <span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>, where
<span class="math">\(k = \frac{1}{\text{in1\_features}}\)</span></p></li>
<li><p><strong>~Bilinear.bias</strong> – the learnable bias of the module of shape <span class="math">\((\text{out\_features})\)</span>.
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">bias</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, the values are initialized from
<span class="math">\(\mathcal{U}(-\sqrt{k}, \sqrt{k})\)</span>, where
<span class="math">\(k = \frac{1}{\text{in1\_features}}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Bilinear</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">30</span><span class="p">,</span> <span class="mi">40</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">128</span><span class="p">,</span> <span class="mi">30</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="n">input1</span><span class="p">,</span> <span class="n">input2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([128, 40])</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="dropout-layers">
<h2>Dropout layers<a class="headerlink" href="#dropout-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="dropout">
<h3><span class="hidden-section">Dropout</span><a class="headerlink" href="#dropout" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Dropout">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Dropout</code><span class="sig-paren">(</span><em>p=0.5</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/dropout.html#Dropout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Dropout" title="Permalink to this definition">¶</a></dt>
<dd><p>During training, randomly zeroes some of the elements of the input
tensor with probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">p</span></code> using samples from a Bernoulli
distribution. Each channel will be zeroed out independently on every forward
call.</p>
<p>This has proven to be an effective technique for regularization and
preventing the co-adaptation of neurons as described in the paper
<a class="reference external" href="https://arxiv.org/abs/1207.0580">Improving neural networks by preventing co-adaptation of feature
detectors</a> .</p>
<p>Furthermore, the outputs are scaled by a factor of <span class="math">\(\frac{1}{1-p}\)</span> during
training. This means that during evaluation the module simply computes an
identity function.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> – probability of an element to be zeroed. Default: 0.5</p></li>
<li><p><strong>inplace</strong> – If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, will do this operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span>. Input can be of any shape</p></li>
<li><p>Output: <span class="math">\((*)\)</span>. Output is of the same shape as input</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="dropout2d">
<h3><span class="hidden-section">Dropout2d</span><a class="headerlink" href="#dropout2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Dropout2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Dropout2d</code><span class="sig-paren">(</span><em>p=0.5</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/dropout.html#Dropout2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Dropout2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Randomly zero out entire channels (a channel is a 2D feature map,
e.g., the <span class="math">\(j\)</span>-th channel of the <span class="math">\(i\)</span>-th sample in the
batched input is a 2D tensor <span class="math">\(\text{input}[i, j]\)</span>).
Each channel will be zeroed out independently on every forward call with
probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">p</span></code> using samples from a Bernoulli distribution.</p>
<p>Usually the input comes from <code class="xref py py-class docutils literal notranslate"><span class="pre">nn.Conv2d</span></code> modules.</p>
<p>As described in the paper
<a class="reference external" href="http://arxiv.org/abs/1411.4280">Efficient Object Localization Using Convolutional Networks</a> ,
if adjacent pixels within feature maps are strongly correlated
(as is normally the case in early convolution layers) then i.i.d. dropout
will not regularize the activations and will otherwise just result
in an effective learning rate decrease.</p>
<p>In this case, <code class="xref py py-func docutils literal notranslate"><span class="pre">nn.Dropout2d()</span></code> will help promote independence between
feature maps and should be used instead.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – probability of an element to be zero-ed.</p></li>
<li><p><strong>inplace</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, will do this operation
in-place</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, H, W)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H, W)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Dropout2d</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="dropout3d">
<h3><span class="hidden-section">Dropout3d</span><a class="headerlink" href="#dropout3d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Dropout3d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Dropout3d</code><span class="sig-paren">(</span><em>p=0.5</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/dropout.html#Dropout3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Dropout3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Randomly zero out entire channels (a channel is a 3D feature map,
e.g., the <span class="math">\(j\)</span>-th channel of the <span class="math">\(i\)</span>-th sample in the
batched input is a 3D tensor <span class="math">\(\text{input}[i, j]\)</span>).
Each channel will be zeroed out independently on every forward call with
probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">p</span></code> using samples from a Bernoulli distribution.</p>
<p>Usually the input comes from <code class="xref py py-class docutils literal notranslate"><span class="pre">nn.Conv3d</span></code> modules.</p>
<p>As described in the paper
<a class="reference external" href="http://arxiv.org/abs/1411.4280">Efficient Object Localization Using Convolutional Networks</a> ,
if adjacent pixels within feature maps are strongly correlated
(as is normally the case in early convolution layers) then i.i.d. dropout
will not regularize the activations and will otherwise just result
in an effective learning rate decrease.</p>
<p>In this case, <code class="xref py py-func docutils literal notranslate"><span class="pre">nn.Dropout3d()</span></code> will help promote independence between
feature maps and should be used instead.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – probability of an element to be zeroed.</p></li>
<li><p><strong>inplace</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, will do this operation
in-place</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, D, H, W)\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, D, H, W)\)</span> (same shape as input)</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Dropout3d</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="alphadropout">
<h3><span class="hidden-section">AlphaDropout</span><a class="headerlink" href="#alphadropout" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.AlphaDropout">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">AlphaDropout</code><span class="sig-paren">(</span><em>p=0.5</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/dropout.html#AlphaDropout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.AlphaDropout" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Alpha Dropout over the input.</p>
<p>Alpha Dropout is a type of Dropout that maintains the self-normalizing
property.
For an input with zero mean and unit standard deviation, the output of
Alpha Dropout maintains the original mean and standard deviation of the
input.
Alpha Dropout goes hand-in-hand with SELU activation function, which ensures
that the outputs have zero mean and unit standard deviation.</p>
<p>During training, it randomly masks some of the elements of the input
tensor with probability <em>p</em> using samples from a bernoulli distribution.
The elements to masked are randomized on every forward call, and scaled
and shifted to maintain zero mean and unit standard deviation.</p>
<p>During evaluation the module simply computes an identity function.</p>
<p>More details can be found in the paper <a class="reference external" href="https://arxiv.org/abs/1706.02515">Self-Normalizing Neural Networks</a> .</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – probability of an element to be dropped. Default: 0.5</p></li>
<li><p><strong>inplace</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, will do this operation
in-place</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span>. Input can be of any shape</p></li>
<li><p>Output: <span class="math">\((*)\)</span>. Output is of the same shape as input</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">AlphaDropout</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="sparse-layers">
<h2>Sparse layers<a class="headerlink" href="#sparse-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="embedding">
<h3><span class="hidden-section">Embedding</span><a class="headerlink" href="#embedding" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Embedding">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Embedding</code><span class="sig-paren">(</span><em>num_embeddings</em>, <em>embedding_dim</em>, <em>padding_idx=None</em>, <em>max_norm=None</em>, <em>norm_type=2.0</em>, <em>scale_grad_by_freq=False</em>, <em>sparse=False</em>, <em>_weight=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/sparse.html#Embedding"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Embedding" title="Permalink to this definition">¶</a></dt>
<dd><p>A simple lookup table that stores embeddings of a fixed dictionary and size.</p>
<p>This module is often used to store word embeddings and retrieve them using indices.
The input to the module is a list of indices, and the output is the corresponding
word embeddings.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_embeddings</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – size of the dictionary of embeddings</p></li>
<li><p><strong>embedding_dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – the size of each embedding vector</p></li>
<li><p><strong>padding_idx</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – If given, pads the output with the embedding vector at <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding_idx</span></code>
(initialized to zeros) whenever it encounters the index.</p></li>
<li><p><strong>max_norm</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – If given, each embedding vector with norm larger than <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>
is renormalized to have norm <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>.</p></li>
<li><p><strong>norm_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – The p of the p-norm to compute for the <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code> option. Default <code class="docutils literal notranslate"><span class="pre">2</span></code>.</p></li>
<li><p><strong>scale_grad_by_freq</strong> (<em>boolean</em><em>, </em><em>optional</em>) – If given, this will scale gradients by the inverse of frequency of
the words in the mini-batch. Default <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>sparse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, gradient w.r.t. <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> matrix will be a sparse tensor.
See Notes for more details regarding sparse gradients.</p></li>
</ul>
</dd>
<dt class="field-even">Variables</dt>
<dd class="field-even"><p><strong>~Embedding.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape (num_embeddings, embedding_dim)
initialized from <span class="math">\(\mathcal{N}(0, 1)\)</span></p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span>, LongTensor of arbitrary shape containing the indices to extract</p></li>
<li><p>Output: <span class="math">\((*, H)\)</span>, where <cite>*</cite> is the input shape and <span class="math">\(H=\text{embedding\_dim}\)</span></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Keep in mind that only a limited number of optimizers support
sparse gradients: currently it’s <code class="xref py py-class docutils literal notranslate"><span class="pre">optim.SGD</span></code> (<cite>CUDA</cite> and <cite>CPU</cite>),
<code class="xref py py-class docutils literal notranslate"><span class="pre">optim.SparseAdam</span></code> (<cite>CUDA</cite> and <cite>CPU</cite>) and <code class="xref py py-class docutils literal notranslate"><span class="pre">optim.Adagrad</span></code> (<cite>CPU</cite>)</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>With <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding_idx</span></code> set, the embedding vector at
<code class="xref py py-attr docutils literal notranslate"><span class="pre">padding_idx</span></code> is initialized to all zeros. However, note that this
vector can be modified afterwards, e.g., using a customized
initialization method, and thus changing the vector used to pad the
output. The gradient for this vector from <a class="reference internal" href="#torch.nn.Embedding" title="torch.nn.Embedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">Embedding</span></code></a>
is always zero.</p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># an Embedding module containing 10 tensors of size 3</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># a batch of 2 samples of 4 indices each</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">LongTensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">],[</span><span class="mi">4</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">9</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[-0.0251, -1.6902,  0.7172],</span>
<span class="go">         [-0.6431,  0.0748,  0.6969],</span>
<span class="go">         [ 1.4970,  1.3448, -0.9685],</span>
<span class="go">         [-0.3677, -2.7265, -0.1685]],</span>

<span class="go">        [[ 1.4970,  1.3448, -0.9685],</span>
<span class="go">         [ 0.4362, -0.4004,  0.9400],</span>
<span class="go">         [-0.6431,  0.0748,  0.6969],</span>
<span class="go">         [ 0.9124, -2.3616,  1.1151]]])</span>


<span class="gp">&gt;&gt;&gt; </span><span class="c1"># example with padding_idx</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">LongTensor</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mi">5</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[ 0.0000,  0.0000,  0.0000],</span>
<span class="go">         [ 0.1535, -2.0309,  0.9315],</span>
<span class="go">         [ 0.0000,  0.0000,  0.0000],</span>
<span class="go">         [-0.1655,  0.9897,  0.0635]]])</span>
</pre></div>
</div>
<dl class="classmethod">
<dt id="torch.nn.Embedding.from_pretrained">
<em class="property">classmethod </em><code class="descname">from_pretrained</code><span class="sig-paren">(</span><em>embeddings</em>, <em>freeze=True</em>, <em>padding_idx=None</em>, <em>max_norm=None</em>, <em>norm_type=2.0</em>, <em>scale_grad_by_freq=False</em>, <em>sparse=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/sparse.html#Embedding.from_pretrained"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Embedding.from_pretrained" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates Embedding instance from given 2-dimensional FloatTensor.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>embeddings</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – FloatTensor containing weights for the Embedding.
First dimension is being passed to Embedding as <code class="docutils literal notranslate"><span class="pre">num_embeddings</span></code>, second as <code class="docutils literal notranslate"><span class="pre">embedding_dim</span></code>.</p></li>
<li><p><strong>freeze</strong> (<em>boolean</em><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, the tensor does not get updated in the learning process.
Equivalent to <code class="docutils literal notranslate"><span class="pre">embedding.weight.requires_grad</span> <span class="pre">=</span> <span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>padding_idx</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – See module initialization documentation.</p></li>
<li><p><strong>max_norm</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – See module initialization documentation.</p></li>
<li><p><strong>norm_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – See module initialization documentation. Default <code class="docutils literal notranslate"><span class="pre">2</span></code>.</p></li>
<li><p><strong>scale_grad_by_freq</strong> (<em>boolean</em><em>, </em><em>optional</em>) – See module initialization documentation. Default <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>sparse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – See module initialization documentation.</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># FloatTensor containing pretrained weights</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">FloatTensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mf">2.3</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mf">5.1</span><span class="p">,</span> <span class="mf">6.3</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="n">weight</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Get embeddings for index 1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">LongTensor</span><span class="p">([</span><span class="mi">1</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[ 4.0000,  5.1000,  6.3000]])</span>
</pre></div>
</div>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="embeddingbag">
<h3><span class="hidden-section">EmbeddingBag</span><a class="headerlink" href="#embeddingbag" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.EmbeddingBag">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">EmbeddingBag</code><span class="sig-paren">(</span><em>num_embeddings</em>, <em>embedding_dim</em>, <em>max_norm=None</em>, <em>norm_type=2.0</em>, <em>scale_grad_by_freq=False</em>, <em>mode='mean'</em>, <em>sparse=False</em>, <em>_weight=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/sparse.html#EmbeddingBag"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.EmbeddingBag" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes sums or means of ‘bags’ of embeddings, without instantiating the
intermediate embeddings.</p>
<p>For bags of constant length and no <code class="xref py py-attr docutils literal notranslate"><span class="pre">per_sample_weights</span></code>, this class</p>
<blockquote>
<div><ul class="simple">
<li><p>with <code class="docutils literal notranslate"><span class="pre">mode=&quot;sum&quot;</span></code> is equivalent to <a class="reference internal" href="#torch.nn.Embedding" title="torch.nn.Embedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">Embedding</span></code></a> followed by <code class="docutils literal notranslate"><span class="pre">torch.sum(dim=0)</span></code>,</p></li>
<li><p>with <code class="docutils literal notranslate"><span class="pre">mode=&quot;mean&quot;</span></code> is equivalent to <a class="reference internal" href="#torch.nn.Embedding" title="torch.nn.Embedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">Embedding</span></code></a> followed by <code class="docutils literal notranslate"><span class="pre">torch.mean(dim=0)</span></code>,</p></li>
<li><p>with <code class="docutils literal notranslate"><span class="pre">mode=&quot;max&quot;</span></code> is equivalent to <a class="reference internal" href="#torch.nn.Embedding" title="torch.nn.Embedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">Embedding</span></code></a> followed by <code class="docutils literal notranslate"><span class="pre">torch.max(dim=0)</span></code>.</p></li>
</ul>
</div></blockquote>
<p>However, <a class="reference internal" href="#torch.nn.EmbeddingBag" title="torch.nn.EmbeddingBag"><code class="xref py py-class docutils literal notranslate"><span class="pre">EmbeddingBag</span></code></a> is much more time and memory efficient than using a chain of these
operations.</p>
<p>EmbeddingBag also supports per-sample weights as an argument to the forward
pass. This scales the output of the Embedding before performing a weighted
reduction as specified by <code class="docutils literal notranslate"><span class="pre">mode</span></code>. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">per_sample_weights`</span></code> is passed, the
only supported <code class="docutils literal notranslate"><span class="pre">mode</span></code> is <code class="docutils literal notranslate"><span class="pre">&quot;sum&quot;</span></code>, which computes a weighted sum according to
<code class="xref py py-attr docutils literal notranslate"><span class="pre">per_sample_weights</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_embeddings</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – size of the dictionary of embeddings</p></li>
<li><p><strong>embedding_dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – the size of each embedding vector</p></li>
<li><p><strong>max_norm</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – If given, each embedding vector with norm larger than <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>
is renormalized to have norm <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>.</p></li>
<li><p><strong>norm_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – The p of the p-norm to compute for the <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code> option. Default <code class="docutils literal notranslate"><span class="pre">2</span></code>.</p></li>
<li><p><strong>scale_grad_by_freq</strong> (<em>boolean</em><em>, </em><em>optional</em>) – if given, this will scale gradients by the inverse of frequency of
the words in the mini-batch. Default <code class="docutils literal notranslate"><span class="pre">False</span></code>.
Note: this option is not supported when <code class="docutils literal notranslate"><span class="pre">mode=&quot;max&quot;</span></code>.</p></li>
<li><p><strong>mode</strong> (<em>string</em><em>, </em><em>optional</em>) – <code class="docutils literal notranslate"><span class="pre">&quot;sum&quot;</span></code>, <code class="docutils literal notranslate"><span class="pre">&quot;mean&quot;</span></code> or <code class="docutils literal notranslate"><span class="pre">&quot;max&quot;</span></code>. Specifies the way to reduce the bag.
<code class="docutils literal notranslate"><span class="pre">&quot;sum&quot;</span></code> computes the weighted sum, taking <code class="xref py py-attr docutils literal notranslate"><span class="pre">per_sample_weights</span></code>
into consideration. <code class="docutils literal notranslate"><span class="pre">&quot;mean&quot;</span></code> computes the average of the values
in the bag, <code class="docutils literal notranslate"><span class="pre">&quot;max&quot;</span></code> computes the max value over each bag.
Default: <code class="docutils literal notranslate"><span class="pre">&quot;mean&quot;</span></code></p></li>
<li><p><strong>sparse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, gradient w.r.t. <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> matrix will be a sparse tensor. See
Notes for more details regarding sparse gradients. Note: this option is not
supported when <code class="docutils literal notranslate"><span class="pre">mode=&quot;max&quot;</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Variables</dt>
<dd class="field-even"><p><strong>~EmbeddingBag.weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the learnable weights of the module of shape <cite>(num_embeddings, embedding_dim)</cite>
initialized from <span class="math">\(\mathcal{N}(0, 1)\)</span>.</p>
</dd>
</dl>
<dl>
<dt>Inputs: <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> (LongTensor), <code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> (LongTensor, optional), and</dt><dd><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">per_index_weights</span></code> (Tensor, optional)</p>
<ul>
<li><p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> is 2D of shape <cite>(B, N)</cite>,</p>
<p>it will be treated as <code class="docutils literal notranslate"><span class="pre">B</span></code> bags (sequences) each of fixed length <code class="docutils literal notranslate"><span class="pre">N</span></code>, and
this will return <code class="docutils literal notranslate"><span class="pre">B</span></code> values aggregated in a way depending on the <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code>.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> is ignored and required to be <code class="docutils literal notranslate"><span class="pre">None</span></code> in this case.</p>
</li>
<li><p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> is 1D of shape <cite>(N)</cite>,</p>
<p>it will be treated as a concatenation of multiple bags (sequences).
<code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> is required to be a 1D tensor containing the
starting index positions of each bag in <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code>. Therefore,
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> of shape <cite>(B)</cite>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> will be viewed as
having <code class="docutils literal notranslate"><span class="pre">B</span></code> bags. Empty bags (i.e., having 0-length) will have
returned vectors filled by zeros.</p>
</li>
</ul>
<dl class="simple">
<dt>per_sample_weights (Tensor, optional): a tensor of float / double weights, or None</dt><dd><p>to indicate all weights should be taken to be <code class="docutils literal notranslate"><span class="pre">1</span></code>. If specified, <code class="xref py py-attr docutils literal notranslate"><span class="pre">per_sample_weights</span></code>
must have exactly the same shape as input and is treated as having the same
<code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code>, if those are not <code class="docutils literal notranslate"><span class="pre">None</span></code>. Only supported for <code class="docutils literal notranslate"><span class="pre">mode='sum'</span></code>.</p>
</dd>
</dl>
</dd>
</dl>
<p>Output shape: <cite>(B, embedding_dim)</cite></p>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># an Embedding module containing 10 tensors of size 3</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding_sum</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">EmbeddingBag</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;sum&#39;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># a batch of 2 samples of 4 indices each</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">LongTensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">9</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">offsets</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">LongTensor</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span><span class="mi">4</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding_sum</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">offsets</span><span class="p">)</span>
<span class="go">tensor([[-0.8861, -5.4350, -0.0523],</span>
<span class="go">        [ 1.1306, -2.5798, -1.0044]])</span>
</pre></div>
</div>
<dl class="classmethod">
<dt id="torch.nn.EmbeddingBag.from_pretrained">
<em class="property">classmethod </em><code class="descname">from_pretrained</code><span class="sig-paren">(</span><em>embeddings</em>, <em>freeze=True</em>, <em>max_norm=None</em>, <em>norm_type=2.0</em>, <em>scale_grad_by_freq=False</em>, <em>mode='mean'</em>, <em>sparse=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/sparse.html#EmbeddingBag.from_pretrained"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.EmbeddingBag.from_pretrained" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates EmbeddingBag instance from given 2-dimensional FloatTensor.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>embeddings</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – FloatTensor containing weights for the EmbeddingBag.
First dimension is being passed to EmbeddingBag as ‘num_embeddings’, second as ‘embedding_dim’.</p></li>
<li><p><strong>freeze</strong> (<em>boolean</em><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, the tensor does not get updated in the learning process.
Equivalent to <code class="docutils literal notranslate"><span class="pre">embeddingbag.weight.requires_grad</span> <span class="pre">=</span> <span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>max_norm</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – See module initialization documentation. Default: <code class="docutils literal notranslate"><span class="pre">None</span></code></p></li>
<li><p><strong>norm_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – See module initialization documentation. Default <code class="docutils literal notranslate"><span class="pre">2</span></code>.</p></li>
<li><p><strong>scale_grad_by_freq</strong> (<em>boolean</em><em>, </em><em>optional</em>) – See module initialization documentation. Default <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>mode</strong> (<em>string</em><em>, </em><em>optional</em>) – See module initialization documentation. Default: <code class="docutils literal notranslate"><span class="pre">&quot;mean&quot;</span></code></p></li>
<li><p><strong>sparse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – See module initialization documentation. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># FloatTensor containing pretrained weights</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">FloatTensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mf">2.3</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="mi">4</span><span class="p">,</span> <span class="mf">5.1</span><span class="p">,</span> <span class="mf">6.3</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embeddingbag</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">EmbeddingBag</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="n">weight</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Get embeddings for index 1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">LongTensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embeddingbag</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[ 2.5000,  3.7000,  4.6500]])</span>
</pre></div>
</div>
</dd></dl>

</dd></dl>

</div>
</div>
<div class="section" id="distance-functions">
<h2>Distance functions<a class="headerlink" href="#distance-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="cosinesimilarity">
<h3><span class="hidden-section">CosineSimilarity</span><a class="headerlink" href="#cosinesimilarity" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.CosineSimilarity">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">CosineSimilarity</code><span class="sig-paren">(</span><em>dim=1</em>, <em>eps=1e-08</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/distance.html#CosineSimilarity"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.CosineSimilarity" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns cosine similarity between <span class="math">\(x_1\)</span> and <span class="math">\(x_2\)</span>, computed along dim.</p>
<div class="math">
\[\text{similarity} = \dfrac{x_1 \cdot x_2}{\max(\Vert x_1 \Vert _2 \cdot \Vert x_2 \Vert _2, \epsilon)}.

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Dimension where cosine similarity is computed. Default: 1</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Small value to avoid division by zero.
Default: 1e-8</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input1: <span class="math">\((\ast_1, D, \ast_2)\)</span> where D is at position <cite>dim</cite></p></li>
<li><p>Input2: <span class="math">\((\ast_1, D, \ast_2)\)</span>, same shape as the Input1</p></li>
<li><p>Output: <span class="math">\((\ast_1, \ast_2)\)</span></p></li>
</ul>
</dd>
<dt>Examples::</dt><dd><div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">input1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cos</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CosineSimilarity</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">eps</span><span class="o">=</span><span class="mf">1e-6</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">cos</span><span class="p">(</span><span class="n">input1</span><span class="p">,</span> <span class="n">input2</span><span class="p">)</span>
</pre></div>
</div>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="pairwisedistance">
<h3><span class="hidden-section">PairwiseDistance</span><a class="headerlink" href="#pairwisedistance" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.PairwiseDistance">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">PairwiseDistance</code><span class="sig-paren">(</span><em>p=2.0</em>, <em>eps=1e-06</em>, <em>keepdim=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/distance.html#PairwiseDistance"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.PairwiseDistance" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes the batchwise pairwise distance between vectors <span class="math">\(v_1\)</span>, <span class="math">\(v_2\)</span> using the p-norm:</p>
<div class="math">
\[\Vert x \Vert _p = \left( \sum_{i=1}^n  \vert x_i \vert ^ p \right) ^ {1/p}.

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> (<em>real</em>) – the norm degree. Default: 2</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Small value to avoid division by zero.
Default: 1e-6</p></li>
<li><p><strong>keepdim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Determines whether or not to keep the vector dimension.
Default: False</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input1: <span class="math">\((N, D)\)</span> where <cite>D = vector dimension</cite></p></li>
<li><p>Input2: <span class="math">\((N, D)\)</span>, same shape as the Input1</p></li>
<li><p>Output: <span class="math">\((N)\)</span>. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">keepdim</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, then <span class="math">\((N, 1)\)</span>.</p></li>
</ul>
</dd>
<dt>Examples::</dt><dd><div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">pdist</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">PairwiseDistance</span><span class="p">(</span><span class="n">p</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">pdist</span><span class="p">(</span><span class="n">input1</span><span class="p">,</span> <span class="n">input2</span><span class="p">)</span>
</pre></div>
</div>
</dd>
</dl>
</dd></dl>

</div>
</div>
<div class="section" id="loss-functions">
<h2>Loss functions<a class="headerlink" href="#loss-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="l1loss">
<h3><span class="hidden-section">L1Loss</span><a class="headerlink" href="#l1loss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.L1Loss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">L1Loss</code><span class="sig-paren">(</span><em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#L1Loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.L1Loss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that measures the mean absolute error (MAE) between each element in
the input <span class="math">\(x\)</span> and target <span class="math">\(y\)</span>.</p>
<p>The unreduced (i.e. with <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> set to <code class="docutils literal notranslate"><span class="pre">'none'</span></code>) loss can be described as:</p>
<div class="math">
\[\ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
l_n = \left| x_n - y_n \right|,

\]</div>
<p>where <span class="math">\(N\)</span> is the batch size. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is not <code class="docutils literal notranslate"><span class="pre">'none'</span></code>
(default <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>), then:</p>
<div class="math">
\[\ell(x, y) =
\begin{cases}
    \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
    \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
\end{cases}

\]</div>
<p><span class="math">\(x\)</span> and <span class="math">\(y\)</span> are tensors of arbitrary shapes with a total
of <span class="math">\(n\)</span> elements each.</p>
<p>The sum operation still operates over all the elements, and divides by <span class="math">\(n\)</span>.</p>
<p>The division by <span class="math">\(n\)</span> can be avoided if one sets <code class="docutils literal notranslate"><span class="pre">reduction</span> <span class="pre">=</span> <span class="pre">'sum'</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <span class="math">\(*\)</span> means, any number of additional
dimensions</p></li>
<li><p>Target: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then
<span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">L1Loss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="mseloss">
<h3><span class="hidden-section">MSELoss</span><a class="headerlink" href="#mseloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MSELoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MSELoss</code><span class="sig-paren">(</span><em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#MSELoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MSELoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that measures the mean squared error (squared L2 norm) between
each element in the input <span class="math">\(x\)</span> and target <span class="math">\(y\)</span>.</p>
<p>The unreduced (i.e. with <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> set to <code class="docutils literal notranslate"><span class="pre">'none'</span></code>) loss can be described as:</p>
<div class="math">
\[\ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
l_n = \left( x_n - y_n \right)^2,

\]</div>
<p>where <span class="math">\(N\)</span> is the batch size. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is not <code class="docutils literal notranslate"><span class="pre">'none'</span></code>
(default <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>), then:</p>
<div class="math">
\[\ell(x, y) =
\begin{cases}
    \operatorname{mean}(L), &  \text{if reduction} = \text{'mean';}\\
    \operatorname{sum}(L),  &  \text{if reduction} = \text{'sum'.}
\end{cases}

\]</div>
<p><span class="math">\(x\)</span> and <span class="math">\(y\)</span> are tensors of arbitrary shapes with a total
of <span class="math">\(n\)</span> elements each.</p>
<p>The sum operation still operates over all the elements, and divides by <span class="math">\(n\)</span>.</p>
<p>The division by <span class="math">\(n\)</span> can be avoided if one sets <code class="docutils literal notranslate"><span class="pre">reduction</span> <span class="pre">=</span> <span class="pre">'sum'</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <span class="math">\(*\)</span> means, any number of additional
dimensions</p></li>
<li><p>Target: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MSELoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="crossentropyloss">
<h3><span class="hidden-section">CrossEntropyLoss</span><a class="headerlink" href="#crossentropyloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.CrossEntropyLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">CrossEntropyLoss</code><span class="sig-paren">(</span><em>weight=None</em>, <em>size_average=None</em>, <em>ignore_index=-100</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#CrossEntropyLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.CrossEntropyLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>This criterion combines <code class="xref py py-func docutils literal notranslate"><span class="pre">nn.LogSoftmax()</span></code> and <code class="xref py py-func docutils literal notranslate"><span class="pre">nn.NLLLoss()</span></code> in one single class.</p>
<p>It is useful when training a classification problem with <cite>C</cite> classes.
If provided, the optional argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> should be a 1D <cite>Tensor</cite>
assigning weight to each of the classes.
This is particularly useful when you have an unbalanced training set.</p>
<p>The <cite>input</cite> is expected to contain raw, unnormalized scores for each class.</p>
<p><cite>input</cite> has to be a Tensor of size either <span class="math">\((minibatch, C)\)</span> or
<span class="math">\((minibatch, C, d_1, d_2, ..., d_K)\)</span>
with <span class="math">\(K \geq 1\)</span> for the <cite>K</cite>-dimensional case (described later).</p>
<p>This criterion expects a class index in the range <span class="math">\([0, C-1]\)</span> as the
<cite>target</cite> for each value of a 1D tensor of size <cite>minibatch</cite>; if <cite>ignore_index</cite>
is specified, this criterion also accepts this class index (this index may not
necessarily be in the class range).</p>
<p>The loss can be described as:</p>
<div class="math">
\[\text{loss}(x, class) = -\log\left(\frac{\exp(x[class])}{\sum_j \exp(x[j])}\right)
               = -x[class] + \log\left(\sum_j \exp(x[j])\right)

\]</div>
<p>or in the case of the <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> argument being specified:</p>
<div class="math">
\[\text{loss}(x, class) = weight[class] \left(-x[class] + \log\left(\sum_j \exp(x[j])\right)\right)

\]</div>
<p>The losses are averaged across observations for each minibatch.</p>
<p>Can also be used for higher dimension inputs, such as 2D images, by providing
an input of size <span class="math">\((minibatch, C, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span>,
where <span class="math">\(K\)</span> is the number of dimensions, and a target of appropriate shape
(see below).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to each class.
If given, has to be a Tensor of size <cite>C</cite></p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>ignore_index</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Specifies a target value that is ignored
and does not contribute to the input gradient. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code> is
<code class="docutils literal notranslate"><span class="pre">True</span></code>, the loss is averaged over non-ignored targets.</p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C)\)</span> where <cite>C = number of classes</cite>, or
<span class="math">\((N, C, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span>
in the case of <cite>K</cite>-dimensional loss.</p></li>
<li><p>Target: <span class="math">\((N)\)</span> where each value is <span class="math">\(0 \leq \text{targets}[i] \leq C-1\)</span>, or
<span class="math">\((N, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span> in the case of
K-dimensional loss.</p></li>
<li><p>Output: scalar.
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then the same size as the target:
<span class="math">\((N)\)</span>, or
<span class="math">\((N, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span> in the case
of K-dimensional loss.</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CrossEntropyLoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="ctcloss">
<h3><span class="hidden-section">CTCLoss</span><a class="headerlink" href="#ctcloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.CTCLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">CTCLoss</code><span class="sig-paren">(</span><em>blank=0</em>, <em>reduction='mean'</em>, <em>zero_infinity=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#CTCLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.CTCLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>The Connectionist Temporal Classification loss.</p>
<p>Calculates loss between a continuous (unsegmented) time series and a target sequence. CTCLoss sums over the
probability of possible alignments of input to target, producing a loss value which is differentiable
with respect to each input node. The alignment of input to target is assumed to be “many-to-one”, which
limits the length of the target sequence such that it must be <span class="math">\(\leq\)</span> the input length.</p>
<dl>
<dt><strong>Args:</strong></dt><dd><p><strong>blank</strong> (int, optional): blank label. Default <span class="math">\(0\)</span>.
reduction (string, optional): Specifies the reduction to apply to the output:</p>
<blockquote>
<div><p><code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the output losses will be divided by the target lengths and
then the mean over the batch is taken. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p>
</div></blockquote>
<dl class="simple">
<dt><strong>zero_infinity</strong> (bool, optional):</dt><dd><p>Whether to zero infinite losses and the associated gradients.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>
Infinite losses mainly occur when the inputs are too short
to be aligned to the targets.</p>
</dd>
</dl>
</dd>
<dt><strong>Inputs:</strong></dt><dd><dl>
<dt><strong>log_probs</strong>: Tensor of size <span class="math">\((T, N, C)\)</span></dt><dd><div class="line-block">
<div class="line"><span class="math">\(T = \text{input length}\)</span></div>
<div class="line"><span class="math">\(N = \text{batch size}\)</span></div>
<div class="line"><span class="math">\(C = \text{number of classes (including blank)}\)</span></div>
</div>
<p>The logarithmized probabilities of the outputs
(e.g. obtained with <a class="reference internal" href="#torch.nn.functional.log_softmax" title="torch.nn.functional.log_softmax"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.log_softmax()</span></code></a>).</p>
</dd>
<dt><strong>targets</strong>: Tensor of size <span class="math">\((N, S)\)</span> or <span class="math">\((\text{sum(target_lengths)})\)</span></dt><dd><div class="line-block">
<div class="line"><span class="math">\(N = \text{batch size}\)</span></div>
<div class="line"><span class="math">\(S = \text{max target length, if shape is } (N, S)\)</span>.</div>
</div>
<div class="line-block">
<div class="line">Target sequences. Each element in the target sequence is a class index. Target index
cannot be blank (default=0).</div>
</div>
<div class="line-block">
<div class="line">In the <span class="math">\((N, S)\)</span> form, targets are padded to the length of the longest sequence, and stacked.</div>
<div class="line">In the <span class="math">\((\text{sum(target_lengths)})\)</span> form, the targets are assumed to be un-padded and concatenated
within 1 dimension.</div>
</div>
</dd>
<dt><strong>input_lengths</strong>: Tuple or tensor of size <span class="math">\((N)\)</span>.</dt><dd><p>Lengths of the inputs (must each be <span class="math">\(\leq T\)</span>).
Lengths are specified for each sequence to achieve masking under the
assumption that sequences are padded to equal lengths.</p>
</dd>
<dt><strong>target_lengths</strong>: Tuple or tensor of size  <span class="math">\((N)\)</span>.</dt><dd><div class="line-block">
<div class="line">Lengths of the targets. Lengths are specified for each sequence to achieve masking under the
assumption that sequences are padded to equal lengths.</div>
</div>
<div class="line-block">
<div class="line">If target shape is <span class="math">\((N,S)\)</span>, target_lengths are effectively the stop index
<span class="math">\(s_n\)</span> for each target sequence, such that <code class="docutils literal notranslate"><span class="pre">target_n</span> <span class="pre">=</span> <span class="pre">targets[n,0:s_n]</span></code> for
each target in a batch. Lengths must each be <span class="math">\(\leq S\)</span></div>
</div>
<div class="line-block">
<div class="line">If the targets are given as a 1d tensor that is the concatenation of individual targets,
the target_lengths must add up to the total length of the tensor.</div>
</div>
</dd>
</dl>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">T</span> <span class="o">=</span> <span class="mi">50</span>      <span class="c1"># Input sequence length</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">C</span> <span class="o">=</span> <span class="mi">20</span>      <span class="c1"># Number of classes (excluding blank)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">N</span> <span class="o">=</span> <span class="mi">16</span>      <span class="c1"># Batch size</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">S</span> <span class="o">=</span> <span class="mi">30</span>      <span class="c1"># Target sequence length of longest target in batch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">S_min</span> <span class="o">=</span> <span class="mi">10</span>  <span class="c1"># Minimum target length, for demonstration purposes</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Initialize random batch of input vectors, for *size = (T,N,C)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">T</span><span class="p">,</span> <span class="n">N</span><span class="p">,</span> <span class="n">C</span><span class="p">)</span><span class="o">.</span><span class="n">log_softmax</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">requires_grad_</span><span class="p">()</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Initialize random batch of targets (0 = blank, 1:C+1 = classes)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">low</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="n">C</span><span class="o">+</span><span class="mi">1</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="n">S</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input_lengths</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">N</span><span class="p">,),</span> <span class="n">fill_value</span><span class="o">=</span><span class="n">T</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target_lengths</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="n">low</span><span class="o">=</span><span class="n">S_min</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="n">S</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="p">(</span><span class="n">N</span><span class="p">,),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ctc_loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CTCLoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">ctc_loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">input_lengths</span><span class="p">,</span> <span class="n">target_lengths</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
<dl class="simple">
<dt>Reference:</dt><dd><p>A. Graves et al.: Connectionist Temporal Classification:
Labelling Unsegmented Sequence Data with Recurrent Neural Networks:
<a class="reference external" href="https://www.cs.toronto.edu/~graves/icml_2006.pdf">https://www.cs.toronto.edu/~graves/icml_2006.pdf</a></p>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In order to use CuDNN, the following must be satisfied: <code class="xref py py-attr docutils literal notranslate"><span class="pre">targets</span></code> must be
in concatenated format, all <code class="xref py py-attr docutils literal notranslate"><span class="pre">input_lengths</span></code> must be <cite>T</cite>.  <span class="math">\(blank=0\)</span>,
<code class="xref py py-attr docutils literal notranslate"><span class="pre">target_lengths</span></code> <span class="math">\(\leq 256\)</span>, the integer arguments must be of
dtype <code class="xref py py-attr docutils literal notranslate"><span class="pre">torch.int32</span></code>.</p>
<p>The regular implementation uses the (more common in PyTorch) <cite>torch.long</cite> dtype.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
</dd></dl>

</div>
<div class="section" id="nllloss">
<h3><span class="hidden-section">NLLLoss</span><a class="headerlink" href="#nllloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.NLLLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">NLLLoss</code><span class="sig-paren">(</span><em>weight=None</em>, <em>size_average=None</em>, <em>ignore_index=-100</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#NLLLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.NLLLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>The negative log likelihood loss. It is useful to train a classification
problem with <cite>C</cite> classes.</p>
<p>If provided, the optional argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> should be a 1D Tensor assigning
weight to each of the classes. This is particularly useful when you have an
unbalanced training set.</p>
<p>The <cite>input</cite> given through a forward call is expected to contain
log-probabilities of each class. <cite>input</cite> has to be a Tensor of size either
<span class="math">\((minibatch, C)\)</span> or <span class="math">\((minibatch, C, d_1, d_2, ..., d_K)\)</span>
with <span class="math">\(K \geq 1\)</span> for the <cite>K</cite>-dimensional case (described later).</p>
<p>Obtaining log-probabilities in a neural network is easily achieved by
adding a  <cite>LogSoftmax</cite>  layer in the last layer of your network.
You may use <cite>CrossEntropyLoss</cite> instead, if you prefer not to add an extra
layer.</p>
<p>The <cite>target</cite> that this loss expects should be a class index in the range <span class="math">\([0, C-1]\)</span>
where <cite>C = number of classes</cite>; if <cite>ignore_index</cite> is specified, this loss also accepts
this class index (this index may not necessarily be in the class range).</p>
<p>The unreduced (i.e. with <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> set to <code class="docutils literal notranslate"><span class="pre">'none'</span></code>) loss can be described as:</p>
<div class="math">
\[\ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
l_n = - w_{y_n} x_{n,y_n}, \quad
w_{c} = \text{weight}[c] \cdot \mathbb{1}\{c \not= \text{ignore\_index}\},

\]</div>
<p>where <span class="math">\(N\)</span> is the batch size. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is not <code class="docutils literal notranslate"><span class="pre">'none'</span></code>
(default <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>), then</p>
<div class="math">
\[\ell(x, y) = \begin{cases}
    \sum_{n=1}^N \frac{1}{\sum_{n=1}^N w_{y_n}} l_n, &
    \text{if reduction} = \text{'mean';}\\
    \sum_{n=1}^N l_n,  &
    \text{if reduction} = \text{'sum'.}
\end{cases}

\]</div>
<p>Can also be used for higher dimension inputs, such as 2D images, by providing
an input of size <span class="math">\((minibatch, C, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span>,
where <span class="math">\(K\)</span> is the number of dimensions, and a target of appropriate shape
(see below). In the case of images, it computes NLL loss per-pixel.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to each
class. If given, it has to be a Tensor of size <cite>C</cite>. Otherwise, it is
treated as if having all ones.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>ignore_index</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Specifies a target value that is ignored
and does not contribute to the input gradient. When
<code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, the loss is averaged over
non-ignored targets.</p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C)\)</span> where <cite>C = number of classes</cite>, or
<span class="math">\((N, C, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span>
in the case of <cite>K</cite>-dimensional loss.</p></li>
<li><p>Target: <span class="math">\((N)\)</span> where each value is <span class="math">\(0 \leq \text{targets}[i] \leq C-1\)</span>, or
<span class="math">\((N, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span> in the case of
K-dimensional loss.</p></li>
<li><p>Output: scalar.
If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then the same size as the target: <span class="math">\((N)\)</span>, or
<span class="math">\((N, d_1, d_2, ..., d_K)\)</span> with <span class="math">\(K \geq 1\)</span> in the case
of K-dimensional loss.</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LogSoftmax</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">NLLLoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># input is of size N x C = 3 x 5</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># each element in target has to have 0 &lt;= value &lt; C</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">),</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># 2D loss example (used, for example, with image inputs)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">N</span><span class="p">,</span> <span class="n">C</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">4</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">NLLLoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># input is of size N x C x height x width</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">data</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">conv</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="n">C</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">LogSoftmax</span><span class="p">(</span><span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># each element in target has to have 0 &lt;= value &lt; C</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">C</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="n">m</span><span class="p">(</span><span class="n">conv</span><span class="p">(</span><span class="n">data</span><span class="p">)),</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="poissonnllloss">
<h3><span class="hidden-section">PoissonNLLLoss</span><a class="headerlink" href="#poissonnllloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.PoissonNLLLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">PoissonNLLLoss</code><span class="sig-paren">(</span><em>log_input=True</em>, <em>full=False</em>, <em>size_average=None</em>, <em>eps=1e-08</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#PoissonNLLLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.PoissonNLLLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Negative log likelihood loss with Poisson distribution of target.</p>
<p>The loss can be described as:</p>
<div class="math">
\[\text{target} \sim \mathrm{Poisson}(\text{input})

\text{loss}(\text{input}, \text{target}) = \text{input} - \text{target} * \log(\text{input})
                            + \log(\text{target!})\]</div>
<p>The last term can be omitted or approximated with Stirling formula. The
approximation is used for target values more than 1. For targets less or
equal to 1 zeros are added to the loss.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>log_input</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code> the loss is computed as
<span class="math">\(\exp(\text{input}) - \text{target}*\text{input}\)</span>, if <code class="docutils literal notranslate"><span class="pre">False</span></code> the loss is
<span class="math">\(\text{input} - \text{target}*\log(\text{input}+\text{eps})\)</span>.</p></li>
<li><p><strong>full</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – <p>whether to compute full loss, i. e. to add the
Stirling approximation term</p>
<div class="math">
\[\text{target}*\log(\text{target}) - \text{target} + 0.5 * \log(2\pi\text{target}).

\]</div>
</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Small value to avoid evaluation of <span class="math">\(\log(0)\)</span> when
<code class="xref py py-attr docutils literal notranslate"><span class="pre">log_input</span> <span class="pre">=</span> <span class="pre">False</span></code>. Default: 1e-8</p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">PoissonNLLLoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">log_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="n">log_input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <span class="math">\(*\)</span> means, any number of additional
dimensions</p></li>
<li><p>Target: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar by default. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N, *)\)</span>,
the same shape as the input</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="kldivloss">
<h3><span class="hidden-section">KLDivLoss</span><a class="headerlink" href="#kldivloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.KLDivLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">KLDivLoss</code><span class="sig-paren">(</span><em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#KLDivLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.KLDivLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>The <a class="reference external" href="https://en.wikipedia.org/wiki/Kullback-Leibler_divergence">Kullback-Leibler divergence</a> Loss</p>
<p>KL divergence is a useful distance measure for continuous distributions
and is often useful when performing direct regression over the space of
(discretely sampled) continuous output distributions.</p>
<p>As with <a class="reference internal" href="#torch.nn.NLLLoss" title="torch.nn.NLLLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">NLLLoss</span></code></a>, the <cite>input</cite> given is expected to contain
<em>log-probabilities</em> and is not restricted to a 2D Tensor.
The targets are given as <em>probabilities</em> (i.e. without taking the logarithm).</p>
<p>This criterion expects a <cite>target</cite> <cite>Tensor</cite> of the same size as the
<cite>input</cite> <cite>Tensor</cite>.</p>
<p>The unreduced (i.e. with <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> set to <code class="docutils literal notranslate"><span class="pre">'none'</span></code>) loss can be described as:</p>
<div class="math">
\[l(x,y) = L = \{ l_1,\dots,l_N \}, \quad
l_n = y_n \cdot \left( \log y_n - x_n \right)

\]</div>
<p>where the index <span class="math">\(N\)</span> spans all dimensions of <code class="docutils literal notranslate"><span class="pre">input</span></code> and <span class="math">\(L\)</span> has the same
shape as <code class="docutils literal notranslate"><span class="pre">input</span></code>. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is not <code class="docutils literal notranslate"><span class="pre">'none'</span></code> (default <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>), then:</p>
<div class="math">
\[\ell(x, y) = \begin{cases}
    \operatorname{mean}(L), & \text{if reduction} = \text{'mean';} \\
    \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
\end{cases}

\]</div>
<p>In default <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> mode <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>, the losses are averaged for each minibatch over observations
<strong>as well as</strong> over dimensions. <code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code> mode gives the correct KL divergence where losses
are averaged over batch dimension only. <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> mode’s behavior will be changed to the same as
<code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code> in the next major release.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>.
<code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied.
<code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code>: the sum of the output will be divided by batchsize.
<code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed.
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the output will be divided by the number of elements in the output.
Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated,
and in the meantime, specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> = <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> doesn’t return the true kl divergence value, please use
<code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> = <code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code> which aligns with KL math definition.
In the next major release, <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> will be changed to be the same as <code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code>.</p>
</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <span class="math">\(*\)</span> means, any number of additional
dimensions</p></li>
<li><p>Target: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar by default. If :attr:<code class="docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N, *)\)</span>,
the same shape as the input</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="bceloss">
<h3><span class="hidden-section">BCELoss</span><a class="headerlink" href="#bceloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.BCELoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">BCELoss</code><span class="sig-paren">(</span><em>weight=None</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#BCELoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.BCELoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that measures the Binary Cross Entropy
between the target and the output:</p>
<p>The unreduced (i.e. with <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> set to <code class="docutils literal notranslate"><span class="pre">'none'</span></code>) loss can be described as:</p>
<div class="math">
\[\ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
l_n = - w_n \left[ y_n \cdot \log x_n + (1 - y_n) \cdot \log (1 - x_n) \right],

\]</div>
<p>where <span class="math">\(N\)</span> is the batch size. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is not <code class="docutils literal notranslate"><span class="pre">'none'</span></code>
(default <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>), then</p>
<div class="math">
\[\ell(x, y) = \begin{cases}
    \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
    \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
\end{cases}

\]</div>
<p>This is used for measuring the error of a reconstruction in for example
an auto-encoder. Note that the targets <span class="math">\(y\)</span> should be numbers
between 0 and 1.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to the loss
of each batch element. If given, has to be a Tensor of size <cite>nbatch</cite>.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <span class="math">\(*\)</span> means, any number of additional
dimensions</p></li>
<li><p>Target: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N, *)\)</span>, same
shape as input.</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sigmoid</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BCELoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">),</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="bcewithlogitsloss">
<h3><span class="hidden-section">BCEWithLogitsLoss</span><a class="headerlink" href="#bcewithlogitsloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.BCEWithLogitsLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">BCEWithLogitsLoss</code><span class="sig-paren">(</span><em>weight=None</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em>, <em>pos_weight=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#BCEWithLogitsLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.BCEWithLogitsLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>This loss combines a <cite>Sigmoid</cite> layer and the <cite>BCELoss</cite> in one single
class. This version is more numerically stable than using a plain <cite>Sigmoid</cite>
followed by a <cite>BCELoss</cite> as, by combining the operations into one layer,
we take advantage of the log-sum-exp trick for numerical stability.</p>
<p>The unreduced (i.e. with <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> set to <code class="docutils literal notranslate"><span class="pre">'none'</span></code>) loss can be described as:</p>
<div class="math">
\[\ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad
l_n = - w_n \left[ y_n \cdot \log \sigma(x_n)
+ (1 - y_n) \cdot \log (1 - \sigma(x_n)) \right],

\]</div>
<p>where <span class="math">\(N\)</span> is the batch size. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is not <code class="docutils literal notranslate"><span class="pre">'none'</span></code>
(default <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>), then</p>
<div class="math">
\[\ell(x, y) = \begin{cases}
    \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
    \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
\end{cases}

\]</div>
<p>This is used for measuring the error of a reconstruction in for example
an auto-encoder. Note that the targets <cite>t[i]</cite> should be numbers
between 0 and 1.</p>
<p>It’s possible to trade off recall and precision by adding weights to positive examples.
In the case of multi-label classification the loss can be described as:</p>
<div class="math">
\[\ell_c(x, y) = L_c = \{l_{1,c},\dots,l_{N,c}\}^\top, \quad
l_{n,c} = - w_{n,c} \left[ p_c y_{n,c} \cdot \log \sigma(x_{n,c})
+ (1 - y_{n,c}) \cdot \log (1 - \sigma(x_{n,c})) \right],

\]</div>
<p>where <span class="math">\(c\)</span> is the class number (<span class="math">\(c &gt; 1\)</span> for multi-label binary classification,
<span class="math">\(c = 1\)</span> for single-label binary classification),
<span class="math">\(n\)</span> is the number of the sample in the batch and
<span class="math">\(p_c\)</span> is the weight of the positive answer for the class <span class="math">\(c\)</span>.</p>
<p><span class="math">\(p_c &gt; 1\)</span> increases the recall, <span class="math">\(p_c &lt; 1\)</span> increases the precision.</p>
<p>For example, if a dataset contains 100 positive and 300 negative examples of a single class,
then <cite>pos_weight</cite> for the class should be equal to <span class="math">\(\frac{300}{100}=3\)</span>.
The loss would act as if the dataset contains <span class="math">\(3\times 100=300\)</span> positive examples.</p>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">64</span><span class="p">],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>  <span class="c1"># 64 classes, batch size = 10</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">([</span><span class="mi">10</span><span class="p">,</span> <span class="mi">64</span><span class="p">],</span> <span class="mf">0.999</span><span class="p">)</span>  <span class="c1"># A prediction (logit)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pos_weight</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">([</span><span class="mi">64</span><span class="p">])</span>  <span class="c1"># All weights are equal to 1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">criterion</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">BCEWithLogitsLoss</span><span class="p">(</span><span class="n">pos_weight</span><span class="o">=</span><span class="n">pos_weight</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">criterion</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>  <span class="c1"># -log(sigmoid(0.999))</span>
<span class="go">tensor(0.3135)</span>
</pre></div>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to the loss
of each batch element. If given, has to be a Tensor of size <cite>nbatch</cite>.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
<li><p><strong>pos_weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a weight of positive examples.
Must be a vector with length equal to the number of classes.</p></li>
</ul>
</dd>
</dl>
<dl>
<dt>Shape:</dt><dd><blockquote>
<div><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <span class="math">\(*\)</span> means, any number of additional dimensions</p></li>
<li><p>Target: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N, *)\)</span>, same
shape as input.</p></li>
</ul>
</div></blockquote>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">BCEWithLogitsLoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">loss</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="marginrankingloss">
<h3><span class="hidden-section">MarginRankingLoss</span><a class="headerlink" href="#marginrankingloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MarginRankingLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MarginRankingLoss</code><span class="sig-paren">(</span><em>margin=0.0</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#MarginRankingLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MarginRankingLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that measures the loss given
inputs <span class="math">\(x1\)</span>, <span class="math">\(x2\)</span>, two 1D mini-batch <cite>Tensors</cite>,
and a label 1D mini-batch tensor <span class="math">\(y\)</span> (containing 1 or -1).</p>
<p>If <span class="math">\(y = 1\)</span> then it assumed the first input should be ranked higher
(have a larger value) than the second input, and vice-versa for <span class="math">\(y = -1\)</span>.</p>
<p>The loss function for each sample in the mini-batch is:</p>
<div class="math">
\[\text{loss}(x, y) = \max(0, -y * (x1 - x2) + \text{margin})

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>margin</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Has a default value of <span class="math">\(0\)</span>.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, D)\)</span> where <cite>N</cite> is the batch size and <cite>D</cite> is the size of a sample.</p></li>
<li><p>Target: <span class="math">\((N)\)</span></p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N)\)</span>.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="hingeembeddingloss">
<h3><span class="hidden-section">HingeEmbeddingLoss</span><a class="headerlink" href="#hingeembeddingloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.HingeEmbeddingLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">HingeEmbeddingLoss</code><span class="sig-paren">(</span><em>margin=1.0</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#HingeEmbeddingLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.HingeEmbeddingLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Measures the loss given an input tensor <span class="math">\(x\)</span> and a labels tensor <span class="math">\(y\)</span>
(containing 1 or -1).
This is usually used for measuring whether two inputs are similar or
dissimilar, e.g. using the L1 pairwise distance as <span class="math">\(x\)</span>, and is typically
used for learning nonlinear embeddings or semi-supervised learning.</p>
<p>The loss function for <span class="math">\(n\)</span>-th sample in the mini-batch is</p>
<div class="math">
\[l_n = \begin{cases}
    x_n, & \text{if}\; y_n = 1,\\
    \max \{0, \Delta - x_n\}, & \text{if}\; y_n = -1,
\end{cases}

\]</div>
<p>and the total loss functions is</p>
<div class="math">
\[\ell(x, y) = \begin{cases}
    \operatorname{mean}(L), & \text{if reduction} = \text{'mean';}\\
    \operatorname{sum}(L),  & \text{if reduction} = \text{'sum'.}
\end{cases}

\]</div>
<p>where <span class="math">\(L = \{l_1,\dots,l_N\}^\top\)</span>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>margin</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Has a default value of <cite>1</cite>.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span> where <span class="math">\(*\)</span> means, any number of dimensions. The sum operation
operates over all the elements.</p></li>
<li><p>Target: <span class="math">\((*)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then same shape as the input</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="multilabelmarginloss">
<h3><span class="hidden-section">MultiLabelMarginLoss</span><a class="headerlink" href="#multilabelmarginloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MultiLabelMarginLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MultiLabelMarginLoss</code><span class="sig-paren">(</span><em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#MultiLabelMarginLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MultiLabelMarginLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that optimizes a multi-class multi-classification
hinge loss (margin-based loss) between input <span class="math">\(x\)</span> (a 2D mini-batch <cite>Tensor</cite>)
and output <span class="math">\(y\)</span> (which is a 2D <cite>Tensor</cite> of target class indices).
For each sample in the mini-batch:</p>
<div class="math">
\[\text{loss}(x, y) = \sum_{ij}\frac{\max(0, 1 - (x[y[j]] - x[i]))}{\text{x.size}(0)}

\]</div>
<p>where <span class="math">\(x \in \left\{0, \; \cdots , \; \text{x.size}(0) - 1\right\}\)</span>, <span class="math">\(y \in \left\{0, \; \cdots , \; \text{y.size}(0) - 1\right\}\)</span>, <span class="math">\(0 \leq y[j] \leq \text{x.size}(0)-1\)</span>, and <span class="math">\(i \neq y[j]\)</span> for all <span class="math">\(i\)</span> and <span class="math">\(j\)</span>.</p>
<p><span class="math">\(y\)</span> and <span class="math">\(x\)</span> must have the same size.</p>
<p>The criterion only considers a contiguous block of non-negative targets that
starts at the front.</p>
<p>This allows for different samples to have variable amounts of target classes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((C)\)</span> or <span class="math">\((N, C)\)</span> where <cite>N</cite> is the batch size and <cite>C</cite>
is the number of classes.</p></li>
<li><p>Target: <span class="math">\((C)\)</span> or <span class="math">\((N, C)\)</span>, label targets padded by -1 ensuring same shape as the input.</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N)\)</span>.</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MultiLabelMarginLoss</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">FloatTensor</span><span class="p">([[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># for target y, only consider labels 3 and 0, not after label -1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">LongTensor</span><span class="p">([[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># 0.25 * ((1-(0.1-0.2)) + (1-(0.1-0.4)) + (1-(0.8-0.2)) + (1-(0.8-0.4)))</span>
<span class="go">tensor(0.8500)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="smoothl1loss">
<h3><span class="hidden-section">SmoothL1Loss</span><a class="headerlink" href="#smoothl1loss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.SmoothL1Loss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">SmoothL1Loss</code><span class="sig-paren">(</span><em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#SmoothL1Loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.SmoothL1Loss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that uses a squared term if the absolute
element-wise error falls below 1 and an L1 term otherwise.
It is less sensitive to outliers than the <cite>MSELoss</cite> and in some cases
prevents exploding gradients (e.g. see <cite>Fast R-CNN</cite> paper by Ross Girshick).
Also known as the Huber loss:</p>
<div class="math">
\[\text{loss}(x, y) = \frac{1}{n} \sum_{i} z_{i}

\]</div>
<p>where <span class="math">\(z_{i}\)</span> is given by:</p>
<div class="math">
\[z_{i} =
\begin{cases}
0.5 (x_i - y_i)^2, & \text{if } |x_i - y_i| < 1 \\
|x_i - y_i| - 0.5, & \text{otherwise }
\end{cases}

\]</div>
<p><span class="math">\(x\)</span> and <span class="math">\(y\)</span> arbitrary shapes with a total of <span class="math">\(n\)</span> elements each
the sum operation still operates over all the elements, and divides by <span class="math">\(n\)</span>.</p>
<p>The division by <span class="math">\(n\)</span> can be avoided if sets <code class="docutils literal notranslate"><span class="pre">reduction</span> <span class="pre">=</span> <span class="pre">'sum'</span></code>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, *)\)</span> where <span class="math">\(*\)</span> means, any number of additional
dimensions</p></li>
<li><p>Target: <span class="math">\((N, *)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then
<span class="math">\((N, *)\)</span>, same shape as the input</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="softmarginloss">
<h3><span class="hidden-section">SoftMarginLoss</span><a class="headerlink" href="#softmarginloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.SoftMarginLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">SoftMarginLoss</code><span class="sig-paren">(</span><em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#SoftMarginLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.SoftMarginLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that optimizes a two-class classification
logistic loss between input tensor <span class="math">\(x\)</span> and target tensor <span class="math">\(y\)</span>
(containing 1 or -1).</p>
<div class="math">
\[\text{loss}(x, y) = \sum_i \frac{\log(1 + \exp(-y[i]*x[i]))}{\text{x.nelement}()}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((*)\)</span> where <span class="math">\(*\)</span> means, any number of additional
dimensions</p></li>
<li><p>Target: <span class="math">\((*)\)</span>, same shape as the input</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then same shape as the input</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="multilabelsoftmarginloss">
<h3><span class="hidden-section">MultiLabelSoftMarginLoss</span><a class="headerlink" href="#multilabelsoftmarginloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MultiLabelSoftMarginLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MultiLabelSoftMarginLoss</code><span class="sig-paren">(</span><em>weight=None</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#MultiLabelSoftMarginLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MultiLabelSoftMarginLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that optimizes a multi-label one-versus-all
loss based on max-entropy, between input <span class="math">\(x\)</span> and target <span class="math">\(y\)</span> of size
<span class="math">\((N, C)\)</span>.
For each sample in the minibatch:</p>
<div class="math">
\[loss(x, y) = - \frac{1}{C} * \sum_i y[i] * \log((1 + \exp(-x[i]))^{-1})
                 + (1-y[i]) * \log\left(\frac{\exp(-x[i])}{(1 + \exp(-x[i]))}\right)

\]</div>
<p>where <span class="math">\(i \in \left\{0, \; \cdots , \; \text{x.nElement}() - 1\right\}\)</span>,
<span class="math">\(y[i] \in \left\{0, \; 1\right\}\)</span>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to each
class. If given, it has to be a Tensor of size <cite>C</cite>. Otherwise, it is
treated as if having all ones.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C)\)</span> where <cite>N</cite> is the batch size and <cite>C</cite> is the number of classes.</p></li>
<li><p>Target: <span class="math">\((N, C)\)</span>, label targets padded by -1 ensuring same shape as the input.</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N)\)</span>.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="cosineembeddingloss">
<h3><span class="hidden-section">CosineEmbeddingLoss</span><a class="headerlink" href="#cosineembeddingloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.CosineEmbeddingLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">CosineEmbeddingLoss</code><span class="sig-paren">(</span><em>margin=0.0</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#CosineEmbeddingLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.CosineEmbeddingLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that measures the loss given input tensors
<span class="math">\(x_1\)</span>, <span class="math">\(x_2\)</span> and a <cite>Tensor</cite> label <span class="math">\(y\)</span> with values 1 or -1.
This is used for measuring whether two inputs are similar or dissimilar,
using the cosine distance, and is typically used for learning nonlinear
embeddings or semi-supervised learning.</p>
<p>The loss function for each sample is:</p>
<div class="math">
\[\text{loss}(x, y) =
\begin{cases}
1 - \cos(x_1, x_2), & \text{if } y = 1 \\
\max(0, \cos(x_1, x_2) - \text{margin}), & \text{if } y = -1
\end{cases}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>margin</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Should be a number from <span class="math">\(-1\)</span> to <span class="math">\(1\)</span>,
<span class="math">\(0\)</span> to <span class="math">\(0.5\)</span> is suggested. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">margin</span></code> is missing, the
default value is <span class="math">\(0\)</span>.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="multimarginloss">
<h3><span class="hidden-section">MultiMarginLoss</span><a class="headerlink" href="#multimarginloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.MultiMarginLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">MultiMarginLoss</code><span class="sig-paren">(</span><em>p=1</em>, <em>margin=1.0</em>, <em>weight=None</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#MultiMarginLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.MultiMarginLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that optimizes a multi-class classification hinge
loss (margin-based loss) between input <span class="math">\(x\)</span> (a 2D mini-batch <cite>Tensor</cite>) and
output <span class="math">\(y\)</span> (which is a 1D tensor of target class indices,
<span class="math">\(0 \leq y \leq \text{x.size}(1)-1\)</span>):</p>
<p>For each mini-batch sample, the loss in terms of the 1D input <span class="math">\(x\)</span> and scalar
output <span class="math">\(y\)</span> is:</p>
<div class="math">
\[\text{loss}(x, y) = \frac{\sum_i \max(0, \text{margin} - x[y] + x[i]))^p}{\text{x.size}(0)}

\]</div>
<p>where <span class="math">\(x \in \left\{0, \; \cdots , \; \text{x.size}(0) - 1\right\}\)</span>
and <span class="math">\(i \neq y\)</span>.</p>
<p>Optionally, you can give non-equal weighting on the classes by passing
a 1D <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> tensor into the constructor.</p>
<p>The loss function then becomes:</p>
<div class="math">
\[\text{loss}(x, y) = \frac{\sum_i \max(0, w[y] * (\text{margin} - x[y] + x[i]))^p)}{\text{x.size}(0)}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Has a default value of <span class="math">\(1\)</span>. <span class="math">\(1\)</span> and <span class="math">\(2\)</span>
are the only supported values.</p></li>
<li><p><strong>margin</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Has a default value of <span class="math">\(1\)</span>.</p></li>
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to each
class. If given, it has to be a Tensor of size <cite>C</cite>. Otherwise, it is
treated as if having all ones.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="tripletmarginloss">
<h3><span class="hidden-section">TripletMarginLoss</span><a class="headerlink" href="#tripletmarginloss" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.TripletMarginLoss">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">TripletMarginLoss</code><span class="sig-paren">(</span><em>margin=1.0</em>, <em>p=2.0</em>, <em>eps=1e-06</em>, <em>swap=False</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/loss.html#TripletMarginLoss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.TripletMarginLoss" title="Permalink to this definition">¶</a></dt>
<dd><p>Creates a criterion that measures the triplet loss given an input
tensors <span class="math">\(x1\)</span>, <span class="math">\(x2\)</span>, <span class="math">\(x3\)</span> and a margin with a value greater than <span class="math">\(0\)</span>.
This is used for measuring a relative similarity between samples. A triplet
is composed by <cite>a</cite>, <cite>p</cite> and <cite>n</cite> (i.e., <cite>anchor</cite>, <cite>positive examples</cite> and <cite>negative
examples</cite> respectively). The shapes of all input tensors should be
<span class="math">\((N, D)\)</span>.</p>
<p>The distance swap is described in detail in the paper <a class="reference external" href="http://www.bmva.org/bmvc/2016/papers/paper119/index.html">Learning shallow
convolutional feature descriptors with triplet losses</a> by
V. Balntas, E. Riba et al.</p>
<p>The loss function for each sample in the mini-batch is:</p>
<div class="math">
\[L(a, p, n) = \max \{d(a_i, p_i) - d(a_i, n_i) + {\rm margin}, 0\}

\]</div>
<p>where</p>
<div class="math">
\[d(x_i, y_i) = \left\lVert {\bf x}_i - {\bf y}_i \right\rVert_p

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>margin</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Default: <span class="math">\(1\)</span>.</p></li>
<li><p><strong>p</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – The norm degree for pairwise distance. Default: <span class="math">\(2\)</span>.</p></li>
<li><p><strong>swap</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – The distance swap is described in detail in the paper
<cite>Learning shallow convolutional feature descriptors with triplet losses</cite> by
V. Balntas, E. Riba et al. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there are multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, D)\)</span> where <span class="math">\(D\)</span> is the vector dimension.</p></li>
<li><p>Output: scalar. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code> is <code class="docutils literal notranslate"><span class="pre">'none'</span></code>, then <span class="math">\((N)\)</span>.</p></li>
</ul>
</dd>
</dl>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">triplet_loss</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">TripletMarginLoss</span><span class="p">(</span><span class="n">margin</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">p</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input3</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">triplet_loss</span><span class="p">(</span><span class="n">input1</span><span class="p">,</span> <span class="n">input2</span><span class="p">,</span> <span class="n">input3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="vision-layers">
<h2>Vision layers<a class="headerlink" href="#vision-layers" title="Permalink to this headline">¶</a></h2>
<div class="section" id="pixelshuffle">
<h3><span class="hidden-section">PixelShuffle</span><a class="headerlink" href="#pixelshuffle" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.PixelShuffle">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">PixelShuffle</code><span class="sig-paren">(</span><em>upscale_factor</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/pixelshuffle.html#PixelShuffle"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.PixelShuffle" title="Permalink to this definition">¶</a></dt>
<dd><p>Rearranges elements in a tensor of shape <span class="math">\((*, C \times r^2, H, W)\)</span>
to a tensor of shape <span class="math">\((*, C, H \times r, W \times r)\)</span>.</p>
<p>This is useful for implementing efficient sub-pixel convolution
with a stride of <span class="math">\(1/r\)</span>.</p>
<p>Look at the paper:
<a class="reference external" href="https://arxiv.org/abs/1609.05158">Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network</a>
by Shi et. al (2016) for more details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>upscale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – factor to increase spatial resolution by</p>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, L, H_{in}, W_{in})\)</span> where <span class="math">\(L=C \times \text{upscale\_factor}^2\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span> where
<span class="math">\(H_{out} = H_{in} \times \text{upscale\_factor}\)</span>
and <span class="math">\(W_{out} = W_{in} \times \text{upscale\_factor}\)</span></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">pixel_shuffle</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">PixelShuffle</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">pixel_shuffle</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([1, 1, 12, 12])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="upsample">
<h3><span class="hidden-section">Upsample</span><a class="headerlink" href="#upsample" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.Upsample">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">Upsample</code><span class="sig-paren">(</span><em>size=None</em>, <em>scale_factor=None</em>, <em>mode='nearest'</em>, <em>align_corners=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/upsampling.html#Upsample"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.Upsample" title="Permalink to this definition">¶</a></dt>
<dd><p>Upsamples a given multi-channel 1D (temporal), 2D (spatial) or 3D (volumetric) data.</p>
<p>The input data is assumed to be of the form
<cite>minibatch x channels x [optional depth] x [optional height] x width</cite>.
Hence, for spatial inputs, we expect a 4D Tensor and for volumetric inputs, we expect a 5D Tensor.</p>
<p>The algorithms available for upsampling are nearest neighbor and linear,
bilinear, bicubic and trilinear for 3D, 4D and 5D input Tensor,
respectively.</p>
<p>One can either give a <code class="xref py py-attr docutils literal notranslate"><span class="pre">scale_factor</span></code> or the target output <code class="xref py py-attr docutils literal notranslate"><span class="pre">size</span></code> to
calculate the output size. (You cannot give both, as it is ambiguous)</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em><em>, </em><em>optional</em>) – output spatial sizes</p></li>
<li><p><strong>scale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – multiplier for spatial size. Has to match input size if it is a tuple.</p></li>
<li><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a><em>, </em><em>optional</em>) – the upsampling algorithm: one of <code class="docutils literal notranslate"><span class="pre">'nearest'</span></code>,
<code class="docutils literal notranslate"><span class="pre">'linear'</span></code>, <code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code>, <code class="docutils literal notranslate"><span class="pre">'bicubic'</span></code> and <code class="docutils literal notranslate"><span class="pre">'trilinear'</span></code>.
Default: <code class="docutils literal notranslate"><span class="pre">'nearest'</span></code></p></li>
<li><p><strong>align_corners</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, the corner pixels of the input
and output tensors are aligned, and thus preserving the values at
those pixels. This only has effect when <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code> is
<code class="docutils literal notranslate"><span class="pre">'linear'</span></code>, <code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code>, or <code class="docutils literal notranslate"><span class="pre">'trilinear'</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, W_{in})\)</span>, <span class="math">\((N, C, H_{in}, W_{in})\)</span> or <span class="math">\((N, C, D_{in}, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, W_{out})\)</span>, <span class="math">\((N, C, H_{out}, W_{out})\)</span>
or <span class="math">\((N, C, D_{out}, H_{out}, W_{out})\)</span>, where</p></li>
</ul>
</dd>
</dl>
<div class="math">
\[D_{out} = \left\lfloor D_{in} \times \text{scale\_factor} \right\rfloor

\]</div>
<div class="math">
\[H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor

\]</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>With <code class="docutils literal notranslate"><span class="pre">align_corners</span> <span class="pre">=</span> <span class="pre">True</span></code>, the linearly interpolating modes
(<cite>linear</cite>, <cite>bilinear</cite>, <cite>bicubic</cite>, and <cite>trilinear</cite>) don’t proportionally
align the output and input pixels, and thus the output values can depend
on the input size. This was the default behavior for these modes up to
version 0.3.1. Since then, the default behavior is
<code class="docutils literal notranslate"><span class="pre">align_corners</span> <span class="pre">=</span> <span class="pre">False</span></code>. See below for concrete examples on how this
affects the outputs.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If you want downsampling/general resizing, you should use <code class="xref py py-func docutils literal notranslate"><span class="pre">interpolate()</span></code>.</p>
</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[[ 1.,  2.],</span>
<span class="go">          [ 3.,  4.]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Upsample</span><span class="p">(</span><span class="n">scale_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;nearest&#39;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.,  1.,  2.,  2.],</span>
<span class="go">          [ 1.,  1.,  2.,  2.],</span>
<span class="go">          [ 3.,  3.,  4.,  4.],</span>
<span class="go">          [ 3.,  3.,  4.,  4.]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Upsample</span><span class="p">(</span><span class="n">scale_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;bilinear&#39;</span><span class="p">)</span>  <span class="c1"># align_corners=False</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.0000,  1.2500,  1.7500,  2.0000],</span>
<span class="go">          [ 1.5000,  1.7500,  2.2500,  2.5000],</span>
<span class="go">          [ 2.5000,  2.7500,  3.2500,  3.5000],</span>
<span class="go">          [ 3.0000,  3.2500,  3.7500,  4.0000]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Upsample</span><span class="p">(</span><span class="n">scale_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;bilinear&#39;</span><span class="p">,</span> <span class="n">align_corners</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.0000,  1.3333,  1.6667,  2.0000],</span>
<span class="go">          [ 1.6667,  2.0000,  2.3333,  2.6667],</span>
<span class="go">          [ 2.3333,  2.6667,  3.0000,  3.3333],</span>
<span class="go">          [ 3.0000,  3.3333,  3.6667,  4.0000]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Try scaling the same data in a larger tensor</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input_3x3</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input_3x3</span><span class="p">[:,</span> <span class="p">:,</span> <span class="p">:</span><span class="mi">2</span><span class="p">,</span> <span class="p">:</span><span class="mi">2</span><span class="p">]</span><span class="o">.</span><span class="n">copy_</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.,  2.],</span>
<span class="go">          [ 3.,  4.]]]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input_3x3</span>
<span class="go">tensor([[[[ 1.,  2.,  0.],</span>
<span class="go">          [ 3.,  4.,  0.],</span>
<span class="go">          [ 0.,  0.,  0.]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Upsample</span><span class="p">(</span><span class="n">scale_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;bilinear&#39;</span><span class="p">)</span>  <span class="c1"># align_corners=False</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Notice that values in top left corner are the same with the small input (except at boundary)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="n">input_3x3</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.0000,  1.2500,  1.7500,  1.5000,  0.5000,  0.0000],</span>
<span class="go">          [ 1.5000,  1.7500,  2.2500,  1.8750,  0.6250,  0.0000],</span>
<span class="go">          [ 2.5000,  2.7500,  3.2500,  2.6250,  0.8750,  0.0000],</span>
<span class="go">          [ 2.2500,  2.4375,  2.8125,  2.2500,  0.7500,  0.0000],</span>
<span class="go">          [ 0.7500,  0.8125,  0.9375,  0.7500,  0.2500,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Upsample</span><span class="p">(</span><span class="n">scale_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;bilinear&#39;</span><span class="p">,</span> <span class="n">align_corners</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Notice that values in top left corner are now changed</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="n">input_3x3</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.0000,  1.4000,  1.8000,  1.6000,  0.8000,  0.0000],</span>
<span class="go">          [ 1.8000,  2.2000,  2.6000,  2.2400,  1.1200,  0.0000],</span>
<span class="go">          [ 2.6000,  3.0000,  3.4000,  2.8800,  1.4400,  0.0000],</span>
<span class="go">          [ 2.4000,  2.7200,  3.0400,  2.5600,  1.2800,  0.0000],</span>
<span class="go">          [ 1.2000,  1.3600,  1.5200,  1.2800,  0.6400,  0.0000],</span>
<span class="go">          [ 0.0000,  0.0000,  0.0000,  0.0000,  0.0000,  0.0000]]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="upsamplingnearest2d">
<h3><span class="hidden-section">UpsamplingNearest2d</span><a class="headerlink" href="#upsamplingnearest2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.UpsamplingNearest2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">UpsamplingNearest2d</code><span class="sig-paren">(</span><em>size=None</em>, <em>scale_factor=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/upsampling.html#UpsamplingNearest2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.UpsamplingNearest2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D nearest neighbor upsampling to an input signal composed of several input
channels.</p>
<p>To specify the scale, it takes either the <code class="xref py py-attr docutils literal notranslate"><span class="pre">size</span></code> or the <code class="xref py py-attr docutils literal notranslate"><span class="pre">scale_factor</span></code>
as it’s constructor argument.</p>
<p>When <code class="xref py py-attr docutils literal notranslate"><span class="pre">size</span></code> is given, it is the output size of the image <cite>(h, w)</cite>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em><em>, </em><em>optional</em>) – output spatial sizes</p></li>
<li><p><strong>scale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – multiplier for
spatial size.</p></li>
</ul>
</dd>
</dl>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This class is deprecated in favor of <code class="xref py py-func docutils literal notranslate"><span class="pre">interpolate()</span></code>.</p>
</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span> where</p></li>
</ul>
</dd>
</dl>
<div class="math">
\[H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor

\]</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[[ 1.,  2.],</span>
<span class="go">          [ 3.,  4.]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">UpsamplingNearest2d</span><span class="p">(</span><span class="n">scale_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.,  1.,  2.,  2.],</span>
<span class="go">          [ 1.,  1.,  2.,  2.],</span>
<span class="go">          [ 3.,  3.,  4.,  4.],</span>
<span class="go">          [ 3.,  3.,  4.,  4.]]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="upsamplingbilinear2d">
<h3><span class="hidden-section">UpsamplingBilinear2d</span><a class="headerlink" href="#upsamplingbilinear2d" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.UpsamplingBilinear2d">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">UpsamplingBilinear2d</code><span class="sig-paren">(</span><em>size=None</em>, <em>scale_factor=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/modules/upsampling.html#UpsamplingBilinear2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.UpsamplingBilinear2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D bilinear upsampling to an input signal composed of several input
channels.</p>
<p>To specify the scale, it takes either the <code class="xref py py-attr docutils literal notranslate"><span class="pre">size</span></code> or the <code class="xref py py-attr docutils literal notranslate"><span class="pre">scale_factor</span></code>
as it’s constructor argument.</p>
<p>When <code class="xref py py-attr docutils literal notranslate"><span class="pre">size</span></code> is given, it is the output size of the image <cite>(h, w)</cite>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em><em>, </em><em>optional</em>) – output spatial sizes</p></li>
<li><p><strong>scale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – multiplier for
spatial size.</p></li>
</ul>
</dd>
</dl>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This class is deprecated in favor of <code class="xref py py-func docutils literal notranslate"><span class="pre">interpolate()</span></code>. It is
equivalent to <code class="docutils literal notranslate"><span class="pre">nn.functional.interpolate(...,</span> <span class="pre">mode='bilinear',</span> <span class="pre">align_corners=True)</span></code>.</p>
</div>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((N, C, H_{in}, W_{in})\)</span></p></li>
<li><p>Output: <span class="math">\((N, C, H_{out}, W_{out})\)</span> where</p></li>
</ul>
</dd>
</dl>
<div class="math">
\[H_{out} = \left\lfloor H_{in} \times \text{scale\_factor} \right\rfloor

\]</div>
<div class="math">
\[W_{out} = \left\lfloor W_{in} \times \text{scale\_factor} \right\rfloor

\]</div>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span>
<span class="go">tensor([[[[ 1.,  2.],</span>
<span class="go">          [ 3.,  4.]]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">UpsamplingBilinear2d</span><span class="p">(</span><span class="n">scale_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
<span class="go">tensor([[[[ 1.0000,  1.3333,  1.6667,  2.0000],</span>
<span class="go">          [ 1.6667,  2.0000,  2.3333,  2.6667],</span>
<span class="go">          [ 2.3333,  2.6667,  3.0000,  3.3333],</span>
<span class="go">          [ 3.0000,  3.3333,  3.6667,  4.0000]]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="dataparallel-layers-multi-gpu-distributed">
<h2>DataParallel layers (multi-GPU, distributed)<a class="headerlink" href="#dataparallel-layers-multi-gpu-distributed" title="Permalink to this headline">¶</a></h2>
<div class="section" id="dataparallel">
<h3><span class="hidden-section">DataParallel</span><a class="headerlink" href="#dataparallel" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.DataParallel">
<em class="property">class </em><code class="descclassname">torch.nn.</code><code class="descname">DataParallel</code><span class="sig-paren">(</span><em>module</em>, <em>device_ids=None</em>, <em>output_device=None</em>, <em>dim=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/parallel/data_parallel.html#DataParallel"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.DataParallel" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements data parallelism at the module level.</p>
<p>This container parallelizes the application of the given <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> by
splitting the input across the specified devices by chunking in the batch
dimension (other objects will be copied once per device). In the forward
pass, the module is replicated on each device, and each replica handles a
portion of the input. During the backwards pass, gradients from each replica
are summed into the original module.</p>
<p>The batch size should be larger than the number of GPUs used.</p>
<p>See also: <a class="reference internal" href="notes/cuda.html#cuda-nn-dataparallel-instead"><span class="std std-ref">Use nn.DataParallel instead of multiprocessing</span></a></p>
<p>Arbitrary positional and keyword inputs are allowed to be passed into
DataParallel but some types are specially handled. tensors will be
<strong>scattered</strong> on dim specified (default 0). tuple, list and dict types will
be shallow copied. The other types will be shared among different threads
and can be corrupted if written to in the model’s forward pass.</p>
<p>The parallelized <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> must have its parameters and buffers on
<code class="docutils literal notranslate"><span class="pre">device_ids[0]</span></code> before running this <a class="reference internal" href="#torch.nn.DataParallel" title="torch.nn.DataParallel"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataParallel</span></code></a>
module.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>In each forward, <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> is <strong>replicated</strong> on each device, so any
updates to the running module in <code class="docutils literal notranslate"><span class="pre">forward</span></code> will be lost. For example,
if <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> has a counter attribute that is incremented in each
<code class="docutils literal notranslate"><span class="pre">forward</span></code>, it will always stay at the initial value because the update
is done on the replicas which are destroyed after <code class="docutils literal notranslate"><span class="pre">forward</span></code>. However,
<a class="reference internal" href="#torch.nn.DataParallel" title="torch.nn.DataParallel"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataParallel</span></code></a> guarantees that the replica on
<code class="docutils literal notranslate"><span class="pre">device[0]</span></code> will have its parameters and buffers sharing storage with
the base parallelized <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code>. So <strong>in-place</strong> updates to the
parameters or buffers on <code class="docutils literal notranslate"><span class="pre">device[0]</span></code> will be recorded. E.g.,
<a class="reference internal" href="#torch.nn.BatchNorm2d" title="torch.nn.BatchNorm2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">BatchNorm2d</span></code></a> and <a class="reference internal" href="#torch.nn.utils.spectral_norm" title="torch.nn.utils.spectral_norm"><code class="xref py py-func docutils literal notranslate"><span class="pre">spectral_norm()</span></code></a>
rely on this behavior to update the buffers.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Forward and backward hooks defined on <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> and its submodules
will be invoked <code class="docutils literal notranslate"><span class="pre">len(device_ids)</span></code> times, each with inputs located on
a particular device. Particularly, the hooks are only guaranteed to be
executed in correct order with respect to operations on corresponding
devices. For example, it is not guaranteed that hooks set via
<a class="reference internal" href="#torch.nn.Module.register_forward_pre_hook" title="torch.nn.Module.register_forward_pre_hook"><code class="xref py py-meth docutils literal notranslate"><span class="pre">register_forward_pre_hook()</span></code></a> be executed before
<cite>all</cite> <code class="docutils literal notranslate"><span class="pre">len(device_ids)</span></code> <a class="reference internal" href="#torch.nn.Module.forward" title="torch.nn.Module.forward"><code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code></a> calls, but
that each such hook be executed before the corresponding
<a class="reference internal" href="#torch.nn.Module.forward" title="torch.nn.Module.forward"><code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code></a> call of that device.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>When <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> returns a scalar (i.e., 0-dimensional tensor) in
<code class="xref py py-func docutils literal notranslate"><span class="pre">forward()</span></code>, this wrapper will return a vector of length equal to
number of devices used in data parallelism, containing the result from
each device.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>There is a subtlety in using the
<code class="docutils literal notranslate"><span class="pre">pack</span> <span class="pre">sequence</span> <span class="pre">-&gt;</span> <span class="pre">recurrent</span> <span class="pre">network</span> <span class="pre">-&gt;</span> <span class="pre">unpack</span> <span class="pre">sequence</span></code> pattern in a
<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> wrapped in <a class="reference internal" href="#torch.nn.DataParallel" title="torch.nn.DataParallel"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataParallel</span></code></a>.
See <a class="reference internal" href="notes/faq.html#pack-rnn-unpack-with-data-parallelism"><span class="std std-ref">My recurrent network doesn’t work with data parallelism</span></a> section in FAQ for
details.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – module to be parallelized</p></li>
<li><p><strong>device_ids</strong> (<em>list of python:int</em><em> or </em><a class="reference internal" href="tensor_attributes.html#torch.torch.device" title="torch.torch.device"><em>torch.device</em></a>) – CUDA devices (default: all devices)</p></li>
<li><p><strong>output_device</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference internal" href="tensor_attributes.html#torch.torch.device" title="torch.torch.device"><em>torch.device</em></a>) – device location of output (default: device_ids[0])</p></li>
</ul>
</dd>
<dt class="field-even">Variables</dt>
<dd class="field-even"><p><strong>~DataParallel.module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – the module to be parallelized</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">net</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">DataParallel</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">device_ids</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">net</span><span class="p">(</span><span class="n">input_var</span><span class="p">)</span>  <span class="c1"># input_var can be on any device, including CPU</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="distributeddataparallel">
<h3><span class="hidden-section">DistributedDataParallel</span><a class="headerlink" href="#distributeddataparallel" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.parallel.DistributedDataParallel">
<em class="property">class </em><code class="descclassname">torch.nn.parallel.</code><code class="descname">DistributedDataParallel</code><span class="sig-paren">(</span><em>module</em>, <em>device_ids=None</em>, <em>output_device=None</em>, <em>dim=0</em>, <em>broadcast_buffers=True</em>, <em>process_group=None</em>, <em>bucket_cap_mb=25</em>, <em>find_unused_parameters=False</em>, <em>check_reduction=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/parallel/distributed.html#DistributedDataParallel"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.parallel.DistributedDataParallel" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements distributed data parallelism that is based on
<code class="docutils literal notranslate"><span class="pre">torch.distributed</span></code> package at the module level.</p>
<p>This container parallelizes the application of the given module by
splitting the input across the specified devices by chunking in the batch
dimension. The module is replicated on each machine and each device, and
each such replica handles a portion of the input. During the backwards
pass, gradients from each node are averaged.</p>
<p>The batch size should be larger than the number of GPUs used locally.</p>
<p>See also: <a class="reference internal" href="distributed.html#distributed-basics"><span class="std std-ref">Basics</span></a> and <a class="reference internal" href="notes/cuda.html#cuda-nn-dataparallel-instead"><span class="std std-ref">Use nn.DataParallel instead of multiprocessing</span></a>.
The same constraints on input as in <a class="reference internal" href="#torch.nn.DataParallel" title="torch.nn.DataParallel"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.DataParallel</span></code></a> apply.</p>
<p>Creation of this class requires that <code class="docutils literal notranslate"><span class="pre">torch.distributed</span></code> to be already
initialized, by calling <a class="reference internal" href="distributed.html#torch.distributed.init_process_group" title="torch.distributed.init_process_group"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.distributed.init_process_group()</span></code></a>.</p>
<p><code class="docutils literal notranslate"><span class="pre">DistributedDataParallel</span></code> can be used in the following two ways:</p>
<ol class="arabic simple">
<li><p>Single-Process Multi-GPU</p></li>
</ol>
<p>In this case, a single process will be
spawned on each host/node and each process will operate on all the GPUs
of the node where it’s running. To use <code class="docutils literal notranslate"><span class="pre">DistributedDataParallel</span></code> in
this way, you can simply construct the model as the following:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="n">init_process_group</span><span class="p">(</span><span class="n">backend</span><span class="o">=</span><span class="s2">&quot;nccl&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">model</span> <span class="o">=</span> <span class="n">DistributedDataParallel</span><span class="p">(</span><span class="n">model</span><span class="p">)</span> <span class="c1"># device_ids will include all GPU devices by default</span>
</pre></div>
</div>
<ol class="arabic simple" start="2">
<li><p>Multi-Process Single-GPU</p></li>
</ol>
<p>This is the highly recommended way to use <code class="docutils literal notranslate"><span class="pre">DistributedDataParallel</span></code>, with
multiple processes, each of which operates on a single GPU. This is
currently the fastest approach to do data parallel training using PyTorch
and applies to both single-node(multi-GPU) and multi-node data
parallel training. It is proven to be significantly faster than
<a class="reference internal" href="#torch.nn.DataParallel" title="torch.nn.DataParallel"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.DataParallel</span></code></a> for single-node multi-GPU data
parallel training.</p>
<p>Here is how to use it: on each host with N GPUs, you should spawn up N
processes, while ensuring that each process individually works on a single GPU
from 0 to N-1. Therefore, it is your job to ensure that your training script
operates on a single given GPU by calling:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
</pre></div>
</div>
<p>where i is from 0 to N-1. In each process, you should refer the following
to construct this module:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="n">init_process_group</span><span class="p">(</span><span class="n">backend</span><span class="o">=</span><span class="s1">&#39;nccl&#39;</span><span class="p">,</span> <span class="n">world_size</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">init_method</span><span class="o">=</span><span class="s1">&#39;...&#39;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">model</span> <span class="o">=</span> <span class="n">DistributedDataParallel</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">device_ids</span><span class="o">=</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">output_device</span><span class="o">=</span><span class="n">i</span><span class="p">)</span>
</pre></div>
</div>
<p>In order to spawn up multiple processes per node, you can use either
<code class="docutils literal notranslate"><span class="pre">torch.distributed.launch</span></code> or <code class="docutils literal notranslate"><span class="pre">torch.multiprocessing.spawn</span></code></p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><code class="docutils literal notranslate"><span class="pre">nccl</span></code> backend is currently the fastest and
highly recommended backend to be used with Multi-Process Single-GPU
distributed training and this applies to both single-node and multi-node
distributed training</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This module also supports mixed-precision distributed training.
This means that your model can have different types of parameters such
as mixed types of fp16 and fp32, the gradient reduction on these
mixed types of parameters will just work fine.
Also note that <code class="docutils literal notranslate"><span class="pre">nccl</span></code> backend is currently the fastest and highly
recommended backend for fp16/fp32 mixed-precision training.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If you use <code class="docutils literal notranslate"><span class="pre">torch.save</span></code> on one process to checkpoint the module,
and <code class="docutils literal notranslate"><span class="pre">torch.load</span></code> on some other processes to recover it, make sure that
<code class="docutils literal notranslate"><span class="pre">map_location</span></code> is configured properly for every process. Without
<code class="docutils literal notranslate"><span class="pre">map_location</span></code>, <code class="docutils literal notranslate"><span class="pre">torch.load</span></code> would recover the module to devices
where the module was saved from.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module works only with the <code class="docutils literal notranslate"><span class="pre">gloo</span></code> and <code class="docutils literal notranslate"><span class="pre">nccl</span></code> backends.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Constructor, forward method, and differentiation of the output (or a
function of the output of this module) is a distributed synchronization
point. Take that into account in case different processes might be
executing different code.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module assumes all parameters are registered in the model by the
time it is created. No parameters should be added nor removed later.
Same applies to buffers.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module assumes all parameters are registered in the model of each
distributed processes are in the same order. The module itself will
conduct gradient all-reduction following the reverse order of the
registered parameters of the model. In other words, it is users’
responsibility to ensure that each distributed process has the exact
same model and thus the exact same parameter registration order.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module assumes all buffers and gradients are dense.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module doesn’t work with <a class="reference internal" href="autograd.html#torch.autograd.grad" title="torch.autograd.grad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.autograd.grad()</span></code></a> (i.e. it will
only work if gradients are to be accumulated in <code class="docutils literal notranslate"><span class="pre">.grad</span></code> attributes of
parameters).</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>If you plan on using this module with a <code class="docutils literal notranslate"><span class="pre">nccl</span></code> backend or a <code class="docutils literal notranslate"><span class="pre">gloo</span></code>
backend (that uses Infiniband), together with a DataLoader that uses
multiple workers, please change the multiprocessing start method to
<code class="docutils literal notranslate"><span class="pre">forkserver</span></code> (Python 3 only) or <code class="docutils literal notranslate"><span class="pre">spawn</span></code>. Unfortunately
Gloo (that uses Infiniband) and NCCL2 are not fork safe, and you will
likely experience deadlocks if you don’t change this setting.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Forward and backward hooks defined on <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> and its submodules
won’t be invoked anymore, unless the hooks are initialized in the
<code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code> method.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>You should never try to change your model’s parameters after wrapping
up your model with DistributedDataParallel. In other words, when
wrapping up your model with DistributedDataParallel, the constructor of
DistributedDataParallel will register the additional gradient
reduction functions on all the parameters of the model itself at the
time of construction. If you change the model’s parameters after
the DistributedDataParallel construction, this is not supported and
unexpected behaviors can happen, since some parameters’ gradient
reduction functions might not get called.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Parameters are never broadcast between processes. The module performs
an all-reduce step on gradients and assumes that they will be modified
by the optimizer in all processes in the same way. Buffers
(e.g. BatchNorm stats) are broadcast from the module in process of rank
0, to all other replicas in the system in every iteration.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – module to be parallelized</p></li>
<li><p><strong>device_ids</strong> (<em>list of python:int</em><em> or </em><a class="reference internal" href="tensor_attributes.html#torch.torch.device" title="torch.torch.device"><em>torch.device</em></a>) – CUDA devices. This should
only be provided when the input module resides on a single
CUDA device. For single-device modules, the <code class="docutils literal notranslate"><span class="pre">i``th</span>
<span class="pre">:attr:`module`</span> <span class="pre">replica</span> <span class="pre">is</span> <span class="pre">placed</span> <span class="pre">on</span> <span class="pre">``device_ids[i]</span></code>. For
multi-device modules and CPU modules, device_ids must be None
or an empty list, and input data for the forward pass must be
placed on the correct device. (default: all devices for
single-device modules)</p></li>
<li><p><strong>output_device</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><a class="reference internal" href="tensor_attributes.html#torch.torch.device" title="torch.torch.device"><em>torch.device</em></a>) – device location of output for
single-device CUDA modules. For multi-device modules and
CPU modules, it must be None, and the module itself
dictates the output location. (default: device_ids[0] for
single-device modules)</p></li>
<li><p><strong>broadcast_buffers</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – flag that enables syncing (broadcasting) buffers of
the module at beginning of the forward function.
(default: <code class="docutils literal notranslate"><span class="pre">True</span></code>)</p></li>
<li><p><strong>process_group</strong> – the process group to be used for distributed data
all-reduction. If <code class="docutils literal notranslate"><span class="pre">None</span></code>, the default process group, which
is created by <code class="docutils literal notranslate"><span class="pre">`torch.distributed.init_process_group`</span></code>,
will be used. (default: <code class="docutils literal notranslate"><span class="pre">None</span></code>)</p></li>
<li><p><strong>bucket_cap_mb</strong> – DistributedDataParallel will bucket parameters into
multiple buckets so that gradient reduction of each
bucket can potentially overlap with backward computation.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">bucket_cap_mb</span></code> controls the bucket size in MegaBytes (MB)
(default: 25)</p></li>
<li><p><strong>find_unused_parameters</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – Traverse the autograd graph of all tensors
contained in the return value of the wrapped
module’s <code class="docutils literal notranslate"><span class="pre">forward</span></code> function.
Parameters that don’t receive gradients as
part of this graph are preemptively marked
as being ready to be reduced.
(default: <code class="docutils literal notranslate"><span class="pre">False</span></code>)</p></li>
<li><p><strong>check_reduction</strong> – when setting to <code class="docutils literal notranslate"><span class="pre">True</span></code>, it enables DistributedDataParallel
to automatically check if the previous iteration’s
backward reductions were successfully issued at the
beginning of every iteration’s forward function.
You normally don’t need this option enabled unless you
are observing weird behaviors such as different ranks
are getting different gradients, which should not
happen if DistributedDataParallel is correctly used.
(default: <code class="docutils literal notranslate"><span class="pre">False</span></code>)</p></li>
</ul>
</dd>
<dt class="field-even">Variables</dt>
<dd class="field-even"><p><strong>~DistributedDataParallel.module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – the module to be parallelized</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="n">init_process_group</span><span class="p">(</span><span class="n">backend</span><span class="o">=</span><span class="s1">&#39;nccl&#39;</span><span class="p">,</span> <span class="n">world_size</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">init_method</span><span class="o">=</span><span class="s1">&#39;...&#39;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">net</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">DistributedDataParallel</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">pg</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="distributeddataparallelcpu">
<h3><span class="hidden-section">DistributedDataParallelCPU</span><a class="headerlink" href="#distributeddataparallelcpu" title="Permalink to this headline">¶</a></h3>
<dl class="class">
<dt id="torch.nn.parallel.DistributedDataParallelCPU">
<em class="property">class </em><code class="descclassname">torch.nn.parallel.</code><code class="descname">DistributedDataParallelCPU</code><span class="sig-paren">(</span><em>module</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/parallel/distributed_cpu.html#DistributedDataParallelCPU"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.parallel.DistributedDataParallelCPU" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements distributed data parallelism for CPU at the module level.</p>
<p>This module supports the <code class="docutils literal notranslate"><span class="pre">mpi</span></code> and <code class="docutils literal notranslate"><span class="pre">gloo</span></code> backends.</p>
<p>This container parallelizes the application of the given module by splitting
the input across the specified devices by chunking in the batch
dimension. The module is replicated on each machine, and each such replica
handles a portion of the input. During the backwards pass, gradients from
each node are averaged.</p>
<p>This module could be used in conjunction with the DistributedSampler,
(see <a class="reference internal" href="data.html#torch.utils.data.distributed.DistributedSampler" title="torch.utils.data.distributed.DistributedSampler"><code class="xref py py-class docutils literal notranslate"><span class="pre">DistributedSampler</span></code></a>)
which will load a subset of the original dataset for each node with the same
batch size. So strong scaling should be configured like this:</p>
<p>n = 1, batch size = 12</p>
<p>n = 2, batch size = 64</p>
<p>n = 4, batch size = 32</p>
<p>n = 8, batch size = 16</p>
<p>Creation of this class requires the distributed package to be already
initialized in the process group mode
(see <a class="reference internal" href="distributed.html#torch.distributed.init_process_group" title="torch.distributed.init_process_group"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.distributed.init_process_group()</span></code></a>).</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Constructor, forward method, and differentiation of the output (or a
function of the output of this module) is a distributed synchronization
point. Take that into account in case different node might be
executing different code.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module assumes all parameters are registered in the model by the
time it is created. No parameters should be added nor removed later.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module assumes all gradients are dense.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This module doesn’t work with <a class="reference internal" href="autograd.html#torch.autograd.grad" title="torch.autograd.grad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.autograd.grad()</span></code></a> (i.e. it will
only work if gradients are to be accumulated in <code class="docutils literal notranslate"><span class="pre">.grad</span></code> attributes of
parameters).</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Forward and backward hooks defined on <code class="xref py py-attr docutils literal notranslate"><span class="pre">module</span></code> and its submodules
won’t be invoked anymore, unless the hooks are initialized in the
<code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code> method.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Parameters are broadcast between nodes in the __init__() function. The
module performs an all-reduce step on gradients and assumes that they
will be modified by the optimizer in all nodes in the same way.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>module</strong> – module to be parallelized</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">torch</span><span class="o">.</span><span class="n">distributed</span><span class="o">.</span><span class="n">init_process_group</span><span class="p">(</span><span class="n">world_size</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">init_method</span><span class="o">=</span><span class="s1">&#39;...&#39;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">net</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">DistributedDataParallelCPU</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="utilities">
<h2>Utilities<a class="headerlink" href="#utilities" title="Permalink to this headline">¶</a></h2>
<div class="section" id="clip-grad-norm">
<h3><span class="hidden-section">clip_grad_norm_</span><a class="headerlink" href="#clip-grad-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.clip_grad_norm_">
<code class="descclassname">torch.nn.utils.</code><code class="descname">clip_grad_norm_</code><span class="sig-paren">(</span><em>parameters</em>, <em>max_norm</em>, <em>norm_type=2</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/clip_grad.html#clip_grad_norm_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.clip_grad_norm_" title="Permalink to this definition">¶</a></dt>
<dd><p>Clips gradient norm of an iterable of parameters.</p>
<p>The norm is computed over all gradients together, as if they were
concatenated into a single vector. Gradients are modified in-place.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>parameters</strong> (<em>Iterable</em><em>[</em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>] or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – an iterable of Tensors or a
single Tensor that will have gradients normalized</p></li>
<li><p><strong>max_norm</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – max norm of the gradients</p></li>
<li><p><strong>norm_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – type of the used p-norm. Can be <code class="docutils literal notranslate"><span class="pre">'inf'</span></code> for
infinity norm.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Total norm of the parameters (viewed as a single vector).</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="clip-grad-value">
<h3><span class="hidden-section">clip_grad_value_</span><a class="headerlink" href="#clip-grad-value" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.clip_grad_value_">
<code class="descclassname">torch.nn.utils.</code><code class="descname">clip_grad_value_</code><span class="sig-paren">(</span><em>parameters</em>, <em>clip_value</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/clip_grad.html#clip_grad_value_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.clip_grad_value_" title="Permalink to this definition">¶</a></dt>
<dd><p>Clips gradient of an iterable of parameters at specified value.</p>
<p>Gradients are modified in-place.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>parameters</strong> (<em>Iterable</em><em>[</em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>] or </em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – an iterable of Tensors or a
single Tensor that will have gradients normalized</p></li>
<li><p><strong>clip_value</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – maximum allowed value of the gradients.
The gradients are clipped in the range
<span class="math">\(\left[\text{-clip\_value}, \text{clip\_value}\right]\)</span></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="parameters-to-vector">
<h3><span class="hidden-section">parameters_to_vector</span><a class="headerlink" href="#parameters-to-vector" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.parameters_to_vector">
<code class="descclassname">torch.nn.utils.</code><code class="descname">parameters_to_vector</code><span class="sig-paren">(</span><em>parameters</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/convert_parameters.html#parameters_to_vector"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.parameters_to_vector" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert parameters to one vector</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>parameters</strong> (<em>Iterable</em><em>[</em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>]</em>) – an iterator of Tensors that are the
parameters of a model.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>The parameters represented by a single vector</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="vector-to-parameters">
<h3><span class="hidden-section">vector_to_parameters</span><a class="headerlink" href="#vector-to-parameters" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.vector_to_parameters">
<code class="descclassname">torch.nn.utils.</code><code class="descname">vector_to_parameters</code><span class="sig-paren">(</span><em>vec</em>, <em>parameters</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/convert_parameters.html#vector_to_parameters"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.vector_to_parameters" title="Permalink to this definition">¶</a></dt>
<dd><p>Convert one vector to the parameters</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>vec</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – a single vector represents the parameters of a model.</p></li>
<li><p><strong>parameters</strong> (<em>Iterable</em><em>[</em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>]</em>) – an iterator of Tensors that are the
parameters of a model.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="weight-norm">
<h3><span class="hidden-section">weight_norm</span><a class="headerlink" href="#weight-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.weight_norm">
<code class="descclassname">torch.nn.utils.</code><code class="descname">weight_norm</code><span class="sig-paren">(</span><em>module</em>, <em>name='weight'</em>, <em>dim=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/weight_norm.html#weight_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.weight_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies weight normalization to a parameter in the given module.</p>
<div class="math">
\[\mathbf{w} = g \dfrac{\mathbf{v}}{\|\mathbf{v}\|}

\]</div>
<p>Weight normalization is a reparameterization that decouples the magnitude
of a weight tensor from its direction. This replaces the parameter specified
by <code class="xref py py-attr docutils literal notranslate"><span class="pre">name</span></code> (e.g. <code class="docutils literal notranslate"><span class="pre">'weight'</span></code>) with two parameters: one specifying the magnitude
(e.g. <code class="docutils literal notranslate"><span class="pre">'weight_g'</span></code>) and one specifying the direction (e.g. <code class="docutils literal notranslate"><span class="pre">'weight_v'</span></code>).
Weight normalization is implemented via a hook that recomputes the weight
tensor from the magnitude and direction before every <code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code>
call.</p>
<p>By default, with <code class="docutils literal notranslate"><span class="pre">dim=0</span></code>, the norm is computed independently per output
channel/plane. To compute a norm over the entire weight tensor, use
<code class="docutils literal notranslate"><span class="pre">dim=None</span></code>.</p>
<p>See <a class="reference external" href="https://arxiv.org/abs/1602.07868">https://arxiv.org/abs/1602.07868</a></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – containing module</p></li>
<li><p><strong>name</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a><em>, </em><em>optional</em>) – name of weight parameter</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – dimension over which to compute the norm</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>The original module with the weight norm hook</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">weight_norm</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">40</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s1">&#39;weight&#39;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span>
<span class="go">Linear(in_features=20, out_features=40, bias=True)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="o">.</span><span class="n">weight_g</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([40, 1])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="o">.</span><span class="n">weight_v</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([40, 20])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="remove-weight-norm">
<h3><span class="hidden-section">remove_weight_norm</span><a class="headerlink" href="#remove-weight-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.remove_weight_norm">
<code class="descclassname">torch.nn.utils.</code><code class="descname">remove_weight_norm</code><span class="sig-paren">(</span><em>module</em>, <em>name='weight'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/weight_norm.html#remove_weight_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.remove_weight_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes the weight normalization reparameterization from a module.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – containing module</p></li>
<li><p><strong>name</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a><em>, </em><em>optional</em>) – name of weight parameter</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">weight_norm</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">40</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">remove_weight_norm</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="spectral-norm">
<h3><span class="hidden-section">spectral_norm</span><a class="headerlink" href="#spectral-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.spectral_norm">
<code class="descclassname">torch.nn.utils.</code><code class="descname">spectral_norm</code><span class="sig-paren">(</span><em>module</em>, <em>name='weight'</em>, <em>n_power_iterations=1</em>, <em>eps=1e-12</em>, <em>dim=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/spectral_norm.html#spectral_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.spectral_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies spectral normalization to a parameter in the given module.</p>
<div class="math">
\[\mathbf{W}_{SN} = \dfrac{\mathbf{W}}{\sigma(\mathbf{W})},
\sigma(\mathbf{W}) = \max_{\mathbf{h}: \mathbf{h} \ne 0} \dfrac{\|\mathbf{W} \mathbf{h}\|_2}{\|\mathbf{h}\|_2}

\]</div>
<p>Spectral normalization stabilizes the training of discriminators (critics)
in Generative Adversarial Networks (GANs) by rescaling the weight tensor
with spectral norm <span class="math">\(\sigma\)</span> of the weight matrix calculated using
power iteration method. If the dimension of the weight tensor is greater
than 2, it is reshaped to 2D in power iteration method to get spectral
norm. This is implemented via a hook that calculates spectral norm and
rescales weight before every <code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code> call.</p>
<p>See <a class="reference external" href="https://arxiv.org/abs/1802.05957">Spectral Normalization for Generative Adversarial Networks</a> .</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>nn.Module</em></a>) – containing module</p></li>
<li><p><strong>name</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a><em>, </em><em>optional</em>) – name of weight parameter</p></li>
<li><p><strong>n_power_iterations</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – number of power iterations to
calculate spectral norm</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – epsilon for numerical stability in
calculating norms</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – dimension corresponding to number of outputs,
the default is <code class="docutils literal notranslate"><span class="pre">0</span></code>, except for modules that are instances of
ConvTranspose{1,2,3}d, when it is <code class="docutils literal notranslate"><span class="pre">1</span></code></p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>The original module with the spectral norm hook</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">spectral_norm</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">40</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span>
<span class="go">Linear(in_features=20, out_features=40, bias=True)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">m</span><span class="o">.</span><span class="n">weight_u</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([40])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="remove-spectral-norm">
<h3><span class="hidden-section">remove_spectral_norm</span><a class="headerlink" href="#remove-spectral-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.remove_spectral_norm">
<code class="descclassname">torch.nn.utils.</code><code class="descname">remove_spectral_norm</code><span class="sig-paren">(</span><em>module</em>, <em>name='weight'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/spectral_norm.html#remove_spectral_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.remove_spectral_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Removes the spectral normalization reparameterization from a module.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – containing module</p></li>
<li><p><strong>name</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a><em>, </em><em>optional</em>) – name of weight parameter</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">m</span> <span class="o">=</span> <span class="n">spectral_norm</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">40</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">remove_spectral_norm</span><span class="p">(</span><span class="n">m</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="packedsequence">
<h3><span class="hidden-section">PackedSequence</span><a class="headerlink" href="#packedsequence" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.rnn.PackedSequence">
<code class="descclassname">torch.nn.utils.rnn.</code><code class="descname">PackedSequence</code><span class="sig-paren">(</span><em>data</em>, <em>batch_sizes=None</em>, <em>sorted_indices=None</em>, <em>unsorted_indices=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/rnn.html#PackedSequence"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.rnn.PackedSequence" title="Permalink to this definition">¶</a></dt>
<dd><p>Holds the data and list of <code class="xref py py-attr docutils literal notranslate"><span class="pre">batch_sizes</span></code> of a packed sequence.</p>
<p>All RNN modules accept packed sequences as inputs.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Instances of this class should never be created manually. They are meant
to be instantiated by functions like <a class="reference internal" href="#torch.nn.utils.rnn.pack_padded_sequence" title="torch.nn.utils.rnn.pack_padded_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">pack_padded_sequence()</span></code></a>.</p>
<p>Batch sizes represent the number elements at each sequence step in
the batch, not the varying sequence lengths passed to
<a class="reference internal" href="#torch.nn.utils.rnn.pack_padded_sequence" title="torch.nn.utils.rnn.pack_padded_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">pack_padded_sequence()</span></code></a>.  For instance, given data <code class="docutils literal notranslate"><span class="pre">abc</span></code> and <code class="docutils literal notranslate"><span class="pre">x</span></code>
the <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">PackedSequence</span></code></a> would contain data <code class="docutils literal notranslate"><span class="pre">axbc</span></code> with
<code class="docutils literal notranslate"><span class="pre">batch_sizes=[2,1,1]</span></code>.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Variables</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>~PackedSequence.data</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – Tensor containing packed sequence</p></li>
<li><p><strong>~PackedSequence.batch_sizes</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – Tensor of integers holding
information about the batch size at each sequence step</p></li>
<li><p><strong>~PackedSequence.sorted_indices</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – Tensor of integers holding how this
<a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">PackedSequence</span></code></a> is constructed from sequences.</p></li>
<li><p><strong>~PackedSequence.unsorted_indices</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – Tensor of integers holding how this
to recover the original sequences with correct order.</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><code class="xref py py-attr docutils literal notranslate"><span class="pre">data</span></code> can be on arbitrary device and of arbitrary dtype.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">sorted_indices</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">unsorted_indices</span></code> must be <code class="docutils literal notranslate"><span class="pre">torch.int64</span></code>
tensors on the same device as <code class="xref py py-attr docutils literal notranslate"><span class="pre">data</span></code>.</p>
<p>However, <code class="xref py py-attr docutils literal notranslate"><span class="pre">batch_sizes</span></code> should always be a CPU <code class="docutils literal notranslate"><span class="pre">torch.int64</span></code> tensor.</p>
<p>This invariant is maintained throughout <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">PackedSequence</span></code></a> class,
and all functions that construct a <cite>:class:PackedSequence</cite> in PyTorch
(i.e., they only pass in tensors conforming to this constraint).</p>
</div>
</dd></dl>

</div>
<div class="section" id="pack-padded-sequence">
<h3><span class="hidden-section">pack_padded_sequence</span><a class="headerlink" href="#pack-padded-sequence" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.rnn.pack_padded_sequence">
<code class="descclassname">torch.nn.utils.rnn.</code><code class="descname">pack_padded_sequence</code><span class="sig-paren">(</span><em>input</em>, <em>lengths</em>, <em>batch_first=False</em>, <em>enforce_sorted=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/rnn.html#pack_padded_sequence"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.rnn.pack_padded_sequence" title="Permalink to this definition">¶</a></dt>
<dd><p>Packs a Tensor containing padded sequences of variable length.</p>
<p><code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> can be of size <code class="docutils literal notranslate"><span class="pre">T</span> <span class="pre">x</span> <span class="pre">B</span> <span class="pre">x</span> <span class="pre">*</span></code> where <cite>T</cite> is the length of the
longest sequence (equal to <code class="docutils literal notranslate"><span class="pre">lengths[0]</span></code>), <code class="docutils literal notranslate"><span class="pre">B</span></code> is the batch size, and
<code class="docutils literal notranslate"><span class="pre">*</span></code> is any number of dimensions (including 0). If <code class="docutils literal notranslate"><span class="pre">batch_first</span></code> is
<code class="docutils literal notranslate"><span class="pre">True</span></code>, <code class="docutils literal notranslate"><span class="pre">B</span> <span class="pre">x</span> <span class="pre">T</span> <span class="pre">x</span> <span class="pre">*</span></code> <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> is expected.</p>
<p>For unsorted sequences, use <cite>enforce_sorted = False</cite>. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">enforce_sorted</span></code> is
<code class="docutils literal notranslate"><span class="pre">True</span></code>, the sequences should be sorted by length in a decreasing order, i.e.
<code class="docutils literal notranslate"><span class="pre">input[:,0]</span></code> should be the longest sequence, and <code class="docutils literal notranslate"><span class="pre">input[:,B-1]</span></code> the shortest
one. <cite>enforce_sorted = True</cite> is only necessary for ONNX export.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This function accepts any input that has at least two dimensions. You
can apply it to pack the labels, and use the output of the RNN with
them to compute the loss directly. A Tensor can be retrieved from
a <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">PackedSequence</span></code></a> object by accessing its <code class="docutils literal notranslate"><span class="pre">.data</span></code> attribute.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – padded batch of variable length sequences.</p></li>
<li><p><strong>lengths</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – list of sequences lengths of each batch element.</p></li>
<li><p><strong>batch_first</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, the input is expected in <code class="docutils literal notranslate"><span class="pre">B</span> <span class="pre">x</span> <span class="pre">T</span> <span class="pre">x</span> <span class="pre">*</span></code>
format.</p></li>
<li><p><strong>enforce_sorted</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, the input is expected to
contain sequences sorted by length in a decreasing order. If
<code class="docutils literal notranslate"><span class="pre">False</span></code>, this condition is not checked. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>a <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">PackedSequence</span></code></a> object</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="pad-packed-sequence">
<h3><span class="hidden-section">pad_packed_sequence</span><a class="headerlink" href="#pad-packed-sequence" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.rnn.pad_packed_sequence">
<code class="descclassname">torch.nn.utils.rnn.</code><code class="descname">pad_packed_sequence</code><span class="sig-paren">(</span><em>sequence</em>, <em>batch_first=False</em>, <em>padding_value=0.0</em>, <em>total_length=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/rnn.html#pad_packed_sequence"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.rnn.pad_packed_sequence" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads a packed batch of variable length sequences.</p>
<p>It is an inverse operation to <a class="reference internal" href="#torch.nn.utils.rnn.pack_padded_sequence" title="torch.nn.utils.rnn.pack_padded_sequence"><code class="xref py py-func docutils literal notranslate"><span class="pre">pack_padded_sequence()</span></code></a>.</p>
<p>The returned Tensor’s data will be of size <code class="docutils literal notranslate"><span class="pre">T</span> <span class="pre">x</span> <span class="pre">B</span> <span class="pre">x</span> <span class="pre">*</span></code>, where <cite>T</cite> is the length
of the longest sequence and <cite>B</cite> is the batch size. If <code class="docutils literal notranslate"><span class="pre">batch_first</span></code> is True,
the data will be transposed into <code class="docutils literal notranslate"><span class="pre">B</span> <span class="pre">x</span> <span class="pre">T</span> <span class="pre">x</span> <span class="pre">*</span></code> format.</p>
<p>Batch elements will be ordered decreasingly by their length.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><code class="xref py py-attr docutils literal notranslate"><span class="pre">total_length</span></code> is useful to implement the
<code class="docutils literal notranslate"><span class="pre">pack</span> <span class="pre">sequence</span> <span class="pre">-&gt;</span> <span class="pre">recurrent</span> <span class="pre">network</span> <span class="pre">-&gt;</span> <span class="pre">unpack</span> <span class="pre">sequence</span></code> pattern in a
<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><code class="xref py py-class docutils literal notranslate"><span class="pre">Module</span></code></a> wrapped in <a class="reference internal" href="#torch.nn.DataParallel" title="torch.nn.DataParallel"><code class="xref py py-class docutils literal notranslate"><span class="pre">DataParallel</span></code></a>.
See <a class="reference internal" href="notes/faq.html#pack-rnn-unpack-with-data-parallelism"><span class="std std-ref">this FAQ section</span></a> for
details.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>sequence</strong> (<em>PackedSequence</em>) – batch to pad</p></li>
<li><p><strong>batch_first</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, the output will be in <code class="docutils literal notranslate"><span class="pre">B</span> <span class="pre">x</span> <span class="pre">T</span> <span class="pre">x</span> <span class="pre">*</span></code>
format.</p></li>
<li><p><strong>padding_value</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – values for padded elements.</p></li>
<li><p><strong>total_length</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – if not <code class="docutils literal notranslate"><span class="pre">None</span></code>, the output will be padded to
have length <code class="xref py py-attr docutils literal notranslate"><span class="pre">total_length</span></code>. This method will throw <a class="reference external" href="https://docs.python.org/3/library/exceptions.html#ValueError" title="(in Python v3.7)"><code class="xref py py-class docutils literal notranslate"><span class="pre">ValueError</span></code></a>
if <code class="xref py py-attr docutils literal notranslate"><span class="pre">total_length</span></code> is less than the max sequence length in
<code class="xref py py-attr docutils literal notranslate"><span class="pre">sequence</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Tuple of Tensor containing the padded sequence, and a Tensor
containing the list of lengths of each sequence in the batch.</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="pad-sequence">
<h3><span class="hidden-section">pad_sequence</span><a class="headerlink" href="#pad-sequence" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.rnn.pad_sequence">
<code class="descclassname">torch.nn.utils.rnn.</code><code class="descname">pad_sequence</code><span class="sig-paren">(</span><em>sequences</em>, <em>batch_first=False</em>, <em>padding_value=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/rnn.html#pad_sequence"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.rnn.pad_sequence" title="Permalink to this definition">¶</a></dt>
<dd><p>Pad a list of variable length Tensors with <code class="docutils literal notranslate"><span class="pre">padding_value</span></code></p>
<p><code class="docutils literal notranslate"><span class="pre">pad_sequence</span></code> stacks a list of Tensors along a new dimension,
and pads them to equal length. For example, if the input is list of
sequences with size <code class="docutils literal notranslate"><span class="pre">L</span> <span class="pre">x</span> <span class="pre">*</span></code> and if batch_first is False, and <code class="docutils literal notranslate"><span class="pre">T</span> <span class="pre">x</span> <span class="pre">B</span> <span class="pre">x</span> <span class="pre">*</span></code>
otherwise.</p>
<p><cite>B</cite> is batch size. It is equal to the number of elements in <code class="docutils literal notranslate"><span class="pre">sequences</span></code>.
<cite>T</cite> is length of the longest sequence.
<cite>L</cite> is length of the sequence.
<cite>*</cite> is any number of trailing dimensions, including none.</p>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">torch.nn.utils.rnn</span> <span class="k">import</span> <span class="n">pad_sequence</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">a</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">25</span><span class="p">,</span> <span class="mi">300</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">22</span><span class="p">,</span> <span class="mi">300</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">c</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">15</span><span class="p">,</span> <span class="mi">300</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pad_sequence</span><span class="p">([</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">c</span><span class="p">])</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
<span class="go">torch.Size([25, 3, 300])</span>
</pre></div>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This function returns a Tensor of size <code class="docutils literal notranslate"><span class="pre">T</span> <span class="pre">x</span> <span class="pre">B</span> <span class="pre">x</span> <span class="pre">*</span></code> or <code class="docutils literal notranslate"><span class="pre">B</span> <span class="pre">x</span> <span class="pre">T</span> <span class="pre">x</span> <span class="pre">*</span></code>
where <cite>T</cite> is the length of the longest sequence. This function assumes
trailing dimensions and type of all the Tensors in sequences are same.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>sequences</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a><em>[</em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>]</em>) – list of variable length sequences.</p></li>
<li><p><strong>batch_first</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – output will be in <code class="docutils literal notranslate"><span class="pre">B</span> <span class="pre">x</span> <span class="pre">T</span> <span class="pre">x</span> <span class="pre">*</span></code> if True, or in
<code class="docutils literal notranslate"><span class="pre">T</span> <span class="pre">x</span> <span class="pre">B</span> <span class="pre">x</span> <span class="pre">*</span></code> otherwise</p></li>
<li><p><strong>padding_value</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – value for padded elements. Default: 0.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Tensor of size <code class="docutils literal notranslate"><span class="pre">T</span> <span class="pre">x</span> <span class="pre">B</span> <span class="pre">x</span> <span class="pre">*</span></code> if <code class="xref py py-attr docutils literal notranslate"><span class="pre">batch_first</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>.
Tensor of size <code class="docutils literal notranslate"><span class="pre">B</span> <span class="pre">x</span> <span class="pre">T</span> <span class="pre">x</span> <span class="pre">*</span></code> otherwise</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="pack-sequence">
<h3><span class="hidden-section">pack_sequence</span><a class="headerlink" href="#pack-sequence" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.utils.rnn.pack_sequence">
<code class="descclassname">torch.nn.utils.rnn.</code><code class="descname">pack_sequence</code><span class="sig-paren">(</span><em>sequences</em>, <em>enforce_sorted=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/utils/rnn.html#pack_sequence"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.utils.rnn.pack_sequence" title="Permalink to this definition">¶</a></dt>
<dd><p>Packs a list of variable length Tensors</p>
<p><code class="docutils literal notranslate"><span class="pre">sequences</span></code> should be a list of Tensors of size <code class="docutils literal notranslate"><span class="pre">L</span> <span class="pre">x</span> <span class="pre">*</span></code>, where <cite>L</cite> is
the length of a sequence and <cite>*</cite> is any number of trailing dimensions,
including zero.</p>
<p>For unsorted sequences, use <cite>enforce_sorted = False</cite>. If <code class="docutils literal notranslate"><span class="pre">enforce_sorted</span></code>
is <code class="docutils literal notranslate"><span class="pre">True</span></code>, the sequences should be sorted in the order of decreasing length.
<code class="docutils literal notranslate"><span class="pre">enforce_sorted</span> <span class="pre">=</span> <span class="pre">True</span></code> is only necessary for ONNX export.</p>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">torch.nn.utils.rnn</span> <span class="k">import</span> <span class="n">pack_sequence</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">a</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">c</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">6</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pack_sequence</span><span class="p">([</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">c</span><span class="p">])</span>
<span class="go">PackedSequence(data=tensor([ 1,  4,  6,  2,  5,  3]), batch_sizes=tensor([ 3,  2,  1]))</span>
</pre></div>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>sequences</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a><em>[</em><a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>]</em>) – A list of sequences of decreasing length.</p></li>
<li><p><strong>enforce_sorted</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, checks that the input
contains sequences sorted by length in a decreasing order. If
<code class="docutils literal notranslate"><span class="pre">False</span></code>, this condition is not checked. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>a <a class="reference internal" href="#torch.nn.utils.rnn.PackedSequence" title="torch.nn.utils.rnn.PackedSequence"><code class="xref py py-class docutils literal notranslate"><span class="pre">PackedSequence</span></code></a> object</p>
</dd>
</dl>
</dd></dl>

</div>
</div>
</div>
<div class="section" id="torch-nn-functional">
<h1>torch.nn.functional<a class="headerlink" href="#torch-nn-functional" title="Permalink to this headline">¶</a></h1>
<div class="section" id="convolution-functions">
<h2>Convolution functions<a class="headerlink" href="#convolution-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="id21">
<h3><span class="hidden-section">conv1d</span><a class="headerlink" href="#id21" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.conv1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">conv1d</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>bias=None</em>, <em>stride=1</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>groups=1</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.conv1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D convolution over an input signal composed of several input
planes.</p>
<p>See <a class="reference internal" href="#torch.nn.Conv1d" title="torch.nn.Conv1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv1d</span></code></a> for details and output shape.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\((\text{minibatch} , \text{in\_channels} , iW)\)</span></p></li>
<li><p><strong>weight</strong> – filters of shape <span class="math">\((\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kW)\)</span></p></li>
<li><p><strong>bias</strong> – optional bias of shape <span class="math">\((\text{out\_channels})\)</span>. Default: <code class="docutils literal notranslate"><span class="pre">None</span></code></p></li>
<li><p><strong>stride</strong> – the stride of the convolving kernel. Can be a single number or
a one-element tuple <cite>(sW,)</cite>. Default: 1</p></li>
<li><p><strong>padding</strong> – implicit paddings on both sides of the input. Can be a
single number or a one-element tuple <cite>(padW,)</cite>. Default: 0</p></li>
<li><p><strong>dilation</strong> – the spacing between kernel elements. Can be a single number or
a one-element tuple <cite>(dW,)</cite>. Default: 1</p></li>
<li><p><strong>groups</strong> – split input into groups, <span class="math">\(\text{in\_channels}\)</span> should be divisible by
the number of groups. Default: 1</p></li>
<li><p><strong>padding_mode</strong> – the type of paddings applied to both sided can be: <cite>zeros</cite> or <cite>circular</cite>. Default: <cite>zeros</cite></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">filters</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">33</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">conv1d</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">filters</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="id22">
<h3><span class="hidden-section">conv2d</span><a class="headerlink" href="#id22" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.conv2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">conv2d</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>bias=None</em>, <em>stride=1</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>groups=1</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.conv2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D convolution over an input image composed of several input
planes.</p>
<p>See <a class="reference internal" href="#torch.nn.Conv2d" title="torch.nn.Conv2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv2d</span></code></a> for details and output shape.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\((\text{minibatch} , \text{in\_channels} , iH , iW)\)</span></p></li>
<li><p><strong>weight</strong> – filters of shape <span class="math">\((\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kH , kW)\)</span></p></li>
<li><p><strong>bias</strong> – optional bias tensor of shape <span class="math">\((\text{out\_channels})\)</span>. Default: <code class="docutils literal notranslate"><span class="pre">None</span></code></p></li>
<li><p><strong>stride</strong> – the stride of the convolving kernel. Can be a single number or a
tuple <cite>(sH, sW)</cite>. Default: 1</p></li>
<li><p><strong>padding</strong> – implicit paddings on both sides of the input. Can be a
single number or a tuple <cite>(padH, padW)</cite>. Default: 0</p></li>
<li><p><strong>dilation</strong> – the spacing between kernel elements. Can be a single number or
a tuple <cite>(dH, dW)</cite>. Default: 1</p></li>
<li><p><strong>groups</strong> – split input into groups, <span class="math">\(\text{in\_channels}\)</span> should be divisible by the
number of groups. Default: 1</p></li>
<li><p><strong>padding_mode</strong> – the type of paddings applied to both sided can be: <cite>zeros</cite> or <cite>circular</cite>. Default: <cite>zeros</cite></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With square kernels and equal stride</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">filters</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">,</span><span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">conv2d</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">filters</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="id23">
<h3><span class="hidden-section">conv3d</span><a class="headerlink" href="#id23" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.conv3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">conv3d</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>bias=None</em>, <em>stride=1</em>, <em>padding=0</em>, <em>dilation=1</em>, <em>groups=1</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.conv3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D convolution over an input image composed of several input
planes.</p>
<p>See <a class="reference internal" href="#torch.nn.Conv3d" title="torch.nn.Conv3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Conv3d</span></code></a> for details and output shape.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\((\text{minibatch} , \text{in\_channels} , iT , iH , iW)\)</span></p></li>
<li><p><strong>weight</strong> – filters of shape <span class="math">\((\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kT , kH , kW)\)</span></p></li>
<li><p><strong>bias</strong> – optional bias tensor of shape <span class="math">\((\text{out\_channels})\)</span>. Default: None</p></li>
<li><p><strong>stride</strong> – the stride of the convolving kernel. Can be a single number or a
tuple <cite>(sT, sH, sW)</cite>. Default: 1</p></li>
<li><p><strong>padding</strong> – implicit paddings on both sides of the input. Can be a
single number or a tuple <cite>(padT, padH, padW)</cite>. Default: 0</p></li>
<li><p><strong>dilation</strong> – the spacing between kernel elements. Can be a single number or
a tuple <cite>(dT, dH, dW)</cite>. Default: 1</p></li>
<li><p><strong>groups</strong> – split input into groups, <span class="math">\(\text{in\_channels}\)</span> should be divisible by
the number of groups. Default: 1</p></li>
<li><p><strong>padding_mode</strong> – the type of paddings applied to both sided can be: <cite>zeros</cite> or <cite>circular</cite>. Default: <cite>zeros</cite></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">filters</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">33</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">conv3d</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">filters</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="conv-transpose1d">
<h3><span class="hidden-section">conv_transpose1d</span><a class="headerlink" href="#conv-transpose1d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.conv_transpose1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">conv_transpose1d</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>bias=None</em>, <em>stride=1</em>, <em>padding=0</em>, <em>output_padding=0</em>, <em>groups=1</em>, <em>dilation=1</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.conv_transpose1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D transposed convolution operator over an input signal
composed of several input planes, sometimes also called “deconvolution”.</p>
<p>See <a class="reference internal" href="#torch.nn.ConvTranspose1d" title="torch.nn.ConvTranspose1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">ConvTranspose1d</span></code></a> for details and output shape.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\((\text{minibatch} , \text{in\_channels} , iW)\)</span></p></li>
<li><p><strong>weight</strong> – filters of shape <span class="math">\((\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kW)\)</span></p></li>
<li><p><strong>bias</strong> – optional bias of shape <span class="math">\((\text{out\_channels})\)</span>. Default: None</p></li>
<li><p><strong>stride</strong> – the stride of the convolving kernel. Can be a single number or a
tuple <code class="docutils literal notranslate"><span class="pre">(sW,)</span></code>. Default: 1</p></li>
<li><p><strong>padding</strong> – <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> zero-padding will be added to both
sides of each dimension in the input. Can be a single number or a tuple
<code class="docutils literal notranslate"><span class="pre">(padW,)</span></code>. Default: 0</p></li>
<li><p><strong>output_padding</strong> – additional size added to one side of each dimension in the
output shape. Can be a single number or a tuple <code class="docutils literal notranslate"><span class="pre">(out_padW)</span></code>. Default: 0</p></li>
<li><p><strong>groups</strong> – split input into groups, <span class="math">\(\text{in\_channels}\)</span> should be divisible by the
number of groups. Default: 1</p></li>
<li><p><strong>dilation</strong> – the spacing between kernel elements. Can be a single number or
a tuple <code class="docutils literal notranslate"><span class="pre">(dW,)</span></code>. Default: 1</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weights</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">conv_transpose1d</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">weights</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="conv-transpose2d">
<h3><span class="hidden-section">conv_transpose2d</span><a class="headerlink" href="#conv-transpose2d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.conv_transpose2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">conv_transpose2d</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>bias=None</em>, <em>stride=1</em>, <em>padding=0</em>, <em>output_padding=0</em>, <em>groups=1</em>, <em>dilation=1</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.conv_transpose2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D transposed convolution operator over an input image
composed of several input planes, sometimes also called “deconvolution”.</p>
<p>See <a class="reference internal" href="#torch.nn.ConvTranspose2d" title="torch.nn.ConvTranspose2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">ConvTranspose2d</span></code></a> for details and output shape.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\((\text{minibatch} , \text{in\_channels} , iH , iW)\)</span></p></li>
<li><p><strong>weight</strong> – filters of shape <span class="math">\((\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kH , kW)\)</span></p></li>
<li><p><strong>bias</strong> – optional bias of shape <span class="math">\((\text{out\_channels})\)</span>. Default: None</p></li>
<li><p><strong>stride</strong> – the stride of the convolving kernel. Can be a single number or a
tuple <code class="docutils literal notranslate"><span class="pre">(sH,</span> <span class="pre">sW)</span></code>. Default: 1</p></li>
<li><p><strong>padding</strong> – <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> zero-padding will be added to both
sides of each dimension in the input. Can be a single number or a tuple
<code class="docutils literal notranslate"><span class="pre">(padH,</span> <span class="pre">padW)</span></code>. Default: 0</p></li>
<li><p><strong>output_padding</strong> – additional size added to one side of each dimension in the
output shape. Can be a single number or a tuple <code class="docutils literal notranslate"><span class="pre">(out_padH,</span> <span class="pre">out_padW)</span></code>.
Default: 0</p></li>
<li><p><strong>groups</strong> – split input into groups, <span class="math">\(\text{in\_channels}\)</span> should be divisible by the
number of groups. Default: 1</p></li>
<li><p><strong>dilation</strong> – the spacing between kernel elements. Can be a single number or
a tuple <code class="docutils literal notranslate"><span class="pre">(dH,</span> <span class="pre">dW)</span></code>. Default: 1</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># With square kernels and equal stride</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weights</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">conv_transpose2d</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">weights</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="conv-transpose3d">
<h3><span class="hidden-section">conv_transpose3d</span><a class="headerlink" href="#conv-transpose3d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.conv_transpose3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">conv_transpose3d</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>bias=None</em>, <em>stride=1</em>, <em>padding=0</em>, <em>output_padding=0</em>, <em>groups=1</em>, <em>dilation=1</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.conv_transpose3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D transposed convolution operator over an input image
composed of several input planes, sometimes also called “deconvolution”</p>
<p>See <a class="reference internal" href="#torch.nn.ConvTranspose3d" title="torch.nn.ConvTranspose3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">ConvTranspose3d</span></code></a> for details and output shape.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\((\text{minibatch} , \text{in\_channels} , iT , iH , iW)\)</span></p></li>
<li><p><strong>weight</strong> – filters of shape <span class="math">\((\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kT , kH , kW)\)</span></p></li>
<li><p><strong>bias</strong> – optional bias of shape <span class="math">\((\text{out\_channels})\)</span>. Default: None</p></li>
<li><p><strong>stride</strong> – the stride of the convolving kernel. Can be a single number or a
tuple <code class="docutils literal notranslate"><span class="pre">(sT,</span> <span class="pre">sH,</span> <span class="pre">sW)</span></code>. Default: 1</p></li>
<li><p><strong>padding</strong> – <code class="docutils literal notranslate"><span class="pre">dilation</span> <span class="pre">*</span> <span class="pre">(kernel_size</span> <span class="pre">-</span> <span class="pre">1)</span> <span class="pre">-</span> <span class="pre">padding</span></code> zero-padding will be added to both
sides of each dimension in the input. Can be a single number or a tuple
<code class="docutils literal notranslate"><span class="pre">(padT,</span> <span class="pre">padH,</span> <span class="pre">padW)</span></code>. Default: 0</p></li>
<li><p><strong>output_padding</strong> – additional size added to one side of each dimension in the
output shape. Can be a single number or a tuple
<code class="docutils literal notranslate"><span class="pre">(out_padT,</span> <span class="pre">out_padH,</span> <span class="pre">out_padW)</span></code>. Default: 0</p></li>
<li><p><strong>groups</strong> – split input into groups, <span class="math">\(\text{in\_channels}\)</span> should be divisible by the
number of groups. Default: 1</p></li>
<li><p><strong>dilation</strong> – the spacing between kernel elements. Can be a single number or
a tuple <cite>(dT, dH, dW)</cite>. Default: 1</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">inputs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weights</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">conv_transpose3d</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">weights</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="id24">
<h3><span class="hidden-section">unfold</span><a class="headerlink" href="#id24" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.unfold">
<code class="descclassname">torch.nn.functional.</code><code class="descname">unfold</code><span class="sig-paren">(</span><em>input</em>, <em>kernel_size</em>, <em>dilation=1</em>, <em>padding=0</em>, <em>stride=1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#unfold"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.unfold" title="Permalink to this definition">¶</a></dt>
<dd><p>Extracts sliding local blocks from an batched input tensor.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Currently, only 4-D input tensors (batched image-like tensors) are
supported.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>More than one element of the unfolded tensor may refer to a single
memory location. As a result, in-place operations (especially ones that
are vectorized) may result in incorrect behavior. If you need to write
to the tensor, please clone it first.</p>
</div>
<p>See <a class="reference internal" href="#torch.nn.Unfold" title="torch.nn.Unfold"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.Unfold</span></code></a> for details</p>
</dd></dl>

</div>
<div class="section" id="id25">
<h3><span class="hidden-section">fold</span><a class="headerlink" href="#id25" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.fold">
<code class="descclassname">torch.nn.functional.</code><code class="descname">fold</code><span class="sig-paren">(</span><em>input</em>, <em>output_size</em>, <em>kernel_size</em>, <em>dilation=1</em>, <em>padding=0</em>, <em>stride=1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#fold"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.fold" title="Permalink to this definition">¶</a></dt>
<dd><p>Combines an array of sliding local blocks into a large containing
tensor.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Currently, only 4-D output tensors (batched image-like tensors) are
supported.</p>
</div>
<p>See <a class="reference internal" href="#torch.nn.Fold" title="torch.nn.Fold"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.Fold</span></code></a> for details</p>
</dd></dl>

</div>
</div>
<div class="section" id="pooling-functions">
<h2>Pooling functions<a class="headerlink" href="#pooling-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="avg-pool1d">
<h3><span class="hidden-section">avg_pool1d</span><a class="headerlink" href="#avg-pool1d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.avg_pool1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">avg_pool1d</code><span class="sig-paren">(</span><em>input</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>ceil_mode=False</em>, <em>count_include_pad=True</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.avg_pool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D average pooling over an input signal composed of several
input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AvgPool1d" title="torch.nn.AvgPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AvgPool1d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\((\text{minibatch} , \text{in\_channels} , iW)\)</span></p></li>
<li><p><strong>kernel_size</strong> – the size of the window. Can be a single number or a
tuple <cite>(kW,)</cite></p></li>
<li><p><strong>stride</strong> – the stride of the window. Can be a single number or a tuple
<cite>(sW,)</cite>. Default: <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero paddings on both sides of the input. Can be a
single number or a tuple <cite>(padW,)</cite>. Default: 0</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> to compute the
output shape. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
<li><p><strong>count_include_pad</strong> – when True, will include the zero-padding in the
averaging calculation. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># pool of square window of size=3, stride=2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">7</span><span class="p">]]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">avg_pool1d</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">kernel_size</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="go">tensor([[[ 2.,  4.,  6.]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="avg-pool2d">
<h3><span class="hidden-section">avg_pool2d</span><a class="headerlink" href="#avg-pool2d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.avg_pool2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">avg_pool2d</code><span class="sig-paren">(</span><em>input</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>ceil_mode=False</em>, <em>count_include_pad=True</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.avg_pool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies 2D average-pooling operation in <span class="math">\(kH \times kW\)</span> regions by step size
<span class="math">\(sH \times sW\)</span> steps. The number of output features is equal to the number of
input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AvgPool2d" title="torch.nn.AvgPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AvgPool2d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor <span class="math">\((\text{minibatch} , \text{in\_channels} , iH , iW)\)</span></p></li>
<li><p><strong>kernel_size</strong> – size of the pooling region. Can be a single number or a
tuple <cite>(kH, kW)</cite></p></li>
<li><p><strong>stride</strong> – stride of the pooling operation. Can be a single number or a
tuple <cite>(sH, sW)</cite>. Default: <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero paddings on both sides of the input. Can be a
single number or a tuple <cite>(padH, padW)</cite>. Default: 0</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> in the formula
to compute the output shape. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
<li><p><strong>count_include_pad</strong> – when True, will include the zero-padding in the
averaging calculation. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="avg-pool3d">
<h3><span class="hidden-section">avg_pool3d</span><a class="headerlink" href="#avg-pool3d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.avg_pool3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">avg_pool3d</code><span class="sig-paren">(</span><em>input</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>ceil_mode=False</em>, <em>count_include_pad=True</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.avg_pool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies 3D average-pooling operation in <span class="math">\(kT \times kH \times kW\)</span> regions by step
size <span class="math">\(sT \times sH \times sW\)</span> steps. The number of output features is equal to
<span class="math">\(\lfloor\frac{\text{input planes}}{sT}\rfloor\)</span>.</p>
<p>See <a class="reference internal" href="#torch.nn.AvgPool3d" title="torch.nn.AvgPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AvgPool3d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor <span class="math">\((\text{minibatch} , \text{in\_channels} , iT \times iH , iW)\)</span></p></li>
<li><p><strong>kernel_size</strong> – size of the pooling region. Can be a single number or a
tuple <cite>(kT, kH, kW)</cite></p></li>
<li><p><strong>stride</strong> – stride of the pooling operation. Can be a single number or a
tuple <cite>(sT, sH, sW)</cite>. Default: <code class="xref py py-attr docutils literal notranslate"><span class="pre">kernel_size</span></code></p></li>
<li><p><strong>padding</strong> – implicit zero paddings on both sides of the input. Can be a
single number or a tuple <cite>(padT, padH, padW)</cite>, Default: 0</p></li>
<li><p><strong>ceil_mode</strong> – when True, will use <cite>ceil</cite> instead of <cite>floor</cite> in the formula
to compute the output shape</p></li>
<li><p><strong>count_include_pad</strong> – when True, will include the zero-padding in the
averaging calculation</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="max-pool1d">
<h3><span class="hidden-section">max_pool1d</span><a class="headerlink" href="#max-pool1d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.max_pool1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">max_pool1d</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#torch.nn.functional.max_pool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D max pooling over an input signal composed of several input
planes.</p>
<p>See <a class="reference internal" href="#torch.nn.MaxPool1d" title="torch.nn.MaxPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool1d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="max-pool2d">
<h3><span class="hidden-section">max_pool2d</span><a class="headerlink" href="#max-pool2d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.max_pool2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">max_pool2d</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#torch.nn.functional.max_pool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D max pooling over an input signal composed of several input
planes.</p>
<p>See <a class="reference internal" href="#torch.nn.MaxPool2d" title="torch.nn.MaxPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool2d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="max-pool3d">
<h3><span class="hidden-section">max_pool3d</span><a class="headerlink" href="#max-pool3d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.max_pool3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">max_pool3d</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#torch.nn.functional.max_pool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D max pooling over an input signal composed of several input
planes.</p>
<p>See <a class="reference internal" href="#torch.nn.MaxPool3d" title="torch.nn.MaxPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool3d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="max-unpool1d">
<h3><span class="hidden-section">max_unpool1d</span><a class="headerlink" href="#max-unpool1d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.max_unpool1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">max_unpool1d</code><span class="sig-paren">(</span><em>input</em>, <em>indices</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>output_size=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#max_unpool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.max_unpool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes a partial inverse of <code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool1d</span></code>.</p>
<p>See <a class="reference internal" href="#torch.nn.MaxUnpool1d" title="torch.nn.MaxUnpool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxUnpool1d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="max-unpool2d">
<h3><span class="hidden-section">max_unpool2d</span><a class="headerlink" href="#max-unpool2d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.max_unpool2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">max_unpool2d</code><span class="sig-paren">(</span><em>input</em>, <em>indices</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>output_size=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#max_unpool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.max_unpool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes a partial inverse of <code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool2d</span></code>.</p>
<p>See <a class="reference internal" href="#torch.nn.MaxUnpool2d" title="torch.nn.MaxUnpool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxUnpool2d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="max-unpool3d">
<h3><span class="hidden-section">max_unpool3d</span><a class="headerlink" href="#max-unpool3d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.max_unpool3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">max_unpool3d</code><span class="sig-paren">(</span><em>input</em>, <em>indices</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>padding=0</em>, <em>output_size=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#max_unpool3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.max_unpool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes a partial inverse of <code class="xref py py-class docutils literal notranslate"><span class="pre">MaxPool3d</span></code>.</p>
<p>See <a class="reference internal" href="#torch.nn.MaxUnpool3d" title="torch.nn.MaxUnpool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">MaxUnpool3d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="lp-pool1d">
<h3><span class="hidden-section">lp_pool1d</span><a class="headerlink" href="#lp-pool1d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.lp_pool1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">lp_pool1d</code><span class="sig-paren">(</span><em>input</em>, <em>norm_type</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>ceil_mode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#lp_pool1d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.lp_pool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D power-average pooling over an input signal composed of
several input planes. If the sum of all inputs to the power of <cite>p</cite> is
zero, the gradient is set to zero as well.</p>
<p>See <a class="reference internal" href="#torch.nn.LPPool1d" title="torch.nn.LPPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">LPPool1d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="lp-pool2d">
<h3><span class="hidden-section">lp_pool2d</span><a class="headerlink" href="#lp-pool2d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.lp_pool2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">lp_pool2d</code><span class="sig-paren">(</span><em>input</em>, <em>norm_type</em>, <em>kernel_size</em>, <em>stride=None</em>, <em>ceil_mode=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#lp_pool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.lp_pool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D power-average pooling over an input signal composed of
several input planes. If the sum of all inputs to the power of <cite>p</cite> is
zero, the gradient is set to zero as well.</p>
<p>See <a class="reference internal" href="#torch.nn.LPPool2d" title="torch.nn.LPPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">LPPool2d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="adaptive-max-pool1d">
<h3><span class="hidden-section">adaptive_max_pool1d</span><a class="headerlink" href="#adaptive-max-pool1d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.adaptive_max_pool1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">adaptive_max_pool1d</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#torch.nn.functional.adaptive_max_pool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D adaptive max pooling over an input signal composed of
several input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AdaptiveMaxPool1d" title="torch.nn.AdaptiveMaxPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaptiveMaxPool1d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output_size</strong> – the target output size (single integer)</p></li>
<li><p><strong>return_indices</strong> – whether to return pooling indices. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="adaptive-max-pool2d">
<h3><span class="hidden-section">adaptive_max_pool2d</span><a class="headerlink" href="#adaptive-max-pool2d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.adaptive_max_pool2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">adaptive_max_pool2d</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#torch.nn.functional.adaptive_max_pool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D adaptive max pooling over an input signal composed of
several input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AdaptiveMaxPool2d" title="torch.nn.AdaptiveMaxPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaptiveMaxPool2d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output_size</strong> – the target output size (single integer or
double-integer tuple)</p></li>
<li><p><strong>return_indices</strong> – whether to return pooling indices. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="adaptive-max-pool3d">
<h3><span class="hidden-section">adaptive_max_pool3d</span><a class="headerlink" href="#adaptive-max-pool3d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.adaptive_max_pool3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">adaptive_max_pool3d</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="headerlink" href="#torch.nn.functional.adaptive_max_pool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D adaptive max pooling over an input signal composed of
several input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AdaptiveMaxPool3d" title="torch.nn.AdaptiveMaxPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaptiveMaxPool3d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output_size</strong> – the target output size (single integer or
triple-integer tuple)</p></li>
<li><p><strong>return_indices</strong> – whether to return pooling indices. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="adaptive-avg-pool1d">
<h3><span class="hidden-section">adaptive_avg_pool1d</span><a class="headerlink" href="#adaptive-avg-pool1d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.adaptive_avg_pool1d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">adaptive_avg_pool1d</code><span class="sig-paren">(</span><em>input</em>, <em>output_size</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.adaptive_avg_pool1d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 1D adaptive average pooling over an input signal composed of
several input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AdaptiveAvgPool1d" title="torch.nn.AdaptiveAvgPool1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaptiveAvgPool1d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>output_size</strong> – the target output size (single integer)</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="adaptive-avg-pool2d">
<h3><span class="hidden-section">adaptive_avg_pool2d</span><a class="headerlink" href="#adaptive-avg-pool2d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.adaptive_avg_pool2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">adaptive_avg_pool2d</code><span class="sig-paren">(</span><em>input</em>, <em>output_size</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#adaptive_avg_pool2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.adaptive_avg_pool2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 2D adaptive average pooling over an input signal composed of
several input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AdaptiveAvgPool2d" title="torch.nn.AdaptiveAvgPool2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaptiveAvgPool2d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>output_size</strong> – the target output size (single integer or
double-integer tuple)</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="adaptive-avg-pool3d">
<h3><span class="hidden-section">adaptive_avg_pool3d</span><a class="headerlink" href="#adaptive-avg-pool3d" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.adaptive_avg_pool3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">adaptive_avg_pool3d</code><span class="sig-paren">(</span><em>input</em>, <em>output_size</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#adaptive_avg_pool3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.adaptive_avg_pool3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a 3D adaptive average pooling over an input signal composed of
several input planes.</p>
<p>See <a class="reference internal" href="#torch.nn.AdaptiveAvgPool3d" title="torch.nn.AdaptiveAvgPool3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">AdaptiveAvgPool3d</span></code></a> for details and output shape.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>output_size</strong> – the target output size (single integer or
triple-integer tuple)</p>
</dd>
</dl>
</dd></dl>

</div>
</div>
<div class="section" id="non-linear-activation-functions">
<h2>Non-linear activation functions<a class="headerlink" href="#non-linear-activation-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="id26">
<h3><span class="hidden-section">threshold</span><a class="headerlink" href="#id26" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.threshold">
<code class="descclassname">torch.nn.functional.</code><code class="descname">threshold</code><span class="sig-paren">(</span><em>input</em>, <em>threshold</em>, <em>value</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#threshold"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.threshold" title="Permalink to this definition">¶</a></dt>
<dd><p>Thresholds each element of the input Tensor.</p>
<p>See <a class="reference internal" href="#torch.nn.Threshold" title="torch.nn.Threshold"><code class="xref py py-class docutils literal notranslate"><span class="pre">Threshold</span></code></a> for more details.</p>
</dd></dl>

<dl class="function">
<dt id="torch.nn.functional.threshold_">
<code class="descclassname">torch.nn.functional.</code><code class="descname">threshold_</code><span class="sig-paren">(</span><em>input</em>, <em>threshold</em>, <em>value</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.threshold_" title="Permalink to this definition">¶</a></dt>
<dd><p>In-place version of <a class="reference internal" href="#torch.nn.functional.threshold" title="torch.nn.functional.threshold"><code class="xref py py-func docutils literal notranslate"><span class="pre">threshold()</span></code></a>.</p>
</dd></dl>

</div>
<div class="section" id="id27">
<h3><span class="hidden-section">relu</span><a class="headerlink" href="#id27" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.relu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">relu</code><span class="sig-paren">(</span><em>input</em>, <em>inplace=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#relu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.relu" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the rectified linear unit function element-wise. See
<a class="reference internal" href="#torch.nn.ReLU" title="torch.nn.ReLU"><code class="xref py py-class docutils literal notranslate"><span class="pre">ReLU</span></code></a> for more details.</p>
</dd></dl>

<dl class="function">
<dt id="torch.nn.functional.relu_">
<code class="descclassname">torch.nn.functional.</code><code class="descname">relu_</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.relu_" title="Permalink to this definition">¶</a></dt>
<dd><p>In-place version of <a class="reference internal" href="#torch.nn.functional.relu" title="torch.nn.functional.relu"><code class="xref py py-func docutils literal notranslate"><span class="pre">relu()</span></code></a>.</p>
</dd></dl>

</div>
<div class="section" id="id28">
<h3><span class="hidden-section">hardtanh</span><a class="headerlink" href="#id28" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.hardtanh">
<code class="descclassname">torch.nn.functional.</code><code class="descname">hardtanh</code><span class="sig-paren">(</span><em>input</em>, <em>min_val=-1.</em>, <em>max_val=1.</em>, <em>inplace=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#hardtanh"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.hardtanh" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the HardTanh function element-wise. See <a class="reference internal" href="#torch.nn.Hardtanh" title="torch.nn.Hardtanh"><code class="xref py py-class docutils literal notranslate"><span class="pre">Hardtanh</span></code></a> for more
details.</p>
</dd></dl>

<dl class="function">
<dt id="torch.nn.functional.hardtanh_">
<code class="descclassname">torch.nn.functional.</code><code class="descname">hardtanh_</code><span class="sig-paren">(</span><em>input</em>, <em>min_val=-1.</em>, <em>max_val=1.</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.hardtanh_" title="Permalink to this definition">¶</a></dt>
<dd><p>In-place version of <a class="reference internal" href="#torch.nn.functional.hardtanh" title="torch.nn.functional.hardtanh"><code class="xref py py-func docutils literal notranslate"><span class="pre">hardtanh()</span></code></a>.</p>
</dd></dl>

</div>
<div class="section" id="id29">
<h3><span class="hidden-section">relu6</span><a class="headerlink" href="#id29" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.relu6">
<code class="descclassname">torch.nn.functional.</code><code class="descname">relu6</code><span class="sig-paren">(</span><em>input</em>, <em>inplace=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#relu6"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.relu6" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function <span class="math">\(\text{ReLU6}(x) = \min(\max(0,x), 6)\)</span>.</p>
<p>See <a class="reference internal" href="#torch.nn.ReLU6" title="torch.nn.ReLU6"><code class="xref py py-class docutils literal notranslate"><span class="pre">ReLU6</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id30">
<h3><span class="hidden-section">elu</span><a class="headerlink" href="#id30" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.elu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">elu</code><span class="sig-paren">(</span><em>input</em>, <em>alpha=1.0</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#elu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.elu" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise,
<span class="math">\(\text{ELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1))\)</span>.</p>
<p>See <a class="reference internal" href="#torch.nn.ELU" title="torch.nn.ELU"><code class="xref py py-class docutils literal notranslate"><span class="pre">ELU</span></code></a> for more details.</p>
</dd></dl>

<dl class="function">
<dt id="torch.nn.functional.elu_">
<code class="descclassname">torch.nn.functional.</code><code class="descname">elu_</code><span class="sig-paren">(</span><em>input</em>, <em>alpha=1.</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.elu_" title="Permalink to this definition">¶</a></dt>
<dd><p>In-place version of <a class="reference internal" href="#torch.nn.functional.elu" title="torch.nn.functional.elu"><code class="xref py py-func docutils literal notranslate"><span class="pre">elu()</span></code></a>.</p>
</dd></dl>

</div>
<div class="section" id="id31">
<h3><span class="hidden-section">selu</span><a class="headerlink" href="#id31" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.selu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">selu</code><span class="sig-paren">(</span><em>input</em>, <em>inplace=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#selu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.selu" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise,
<span class="math">\(\text{SELU}(x) = scale * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))\)</span>,
with <span class="math">\(\alpha=1.6732632423543772848170429916717\)</span> and
<span class="math">\(scale=1.0507009873554804934193349852946\)</span>.</p>
<p>See <a class="reference internal" href="#torch.nn.SELU" title="torch.nn.SELU"><code class="xref py py-class docutils literal notranslate"><span class="pre">SELU</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id32">
<h3><span class="hidden-section">celu</span><a class="headerlink" href="#id32" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.celu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">celu</code><span class="sig-paren">(</span><em>input</em>, <em>alpha=1.</em>, <em>inplace=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#celu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.celu" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise,
<span class="math">\(\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))\)</span>.</p>
<p>See <a class="reference internal" href="#torch.nn.CELU" title="torch.nn.CELU"><code class="xref py py-class docutils literal notranslate"><span class="pre">CELU</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="leaky-relu">
<h3><span class="hidden-section">leaky_relu</span><a class="headerlink" href="#leaky-relu" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.leaky_relu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">leaky_relu</code><span class="sig-paren">(</span><em>input</em>, <em>negative_slope=0.01</em>, <em>inplace=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#leaky_relu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.leaky_relu" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise,
<span class="math">\(\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)\)</span></p>
<p>See <a class="reference internal" href="#torch.nn.LeakyReLU" title="torch.nn.LeakyReLU"><code class="xref py py-class docutils literal notranslate"><span class="pre">LeakyReLU</span></code></a> for more details.</p>
</dd></dl>

<dl class="function">
<dt id="torch.nn.functional.leaky_relu_">
<code class="descclassname">torch.nn.functional.</code><code class="descname">leaky_relu_</code><span class="sig-paren">(</span><em>input</em>, <em>negative_slope=0.01</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.leaky_relu_" title="Permalink to this definition">¶</a></dt>
<dd><p>In-place version of <a class="reference internal" href="#torch.nn.functional.leaky_relu" title="torch.nn.functional.leaky_relu"><code class="xref py py-func docutils literal notranslate"><span class="pre">leaky_relu()</span></code></a>.</p>
</dd></dl>

</div>
<div class="section" id="id33">
<h3><span class="hidden-section">prelu</span><a class="headerlink" href="#id33" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.prelu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">prelu</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#prelu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.prelu" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise the function
<span class="math">\(\text{PReLU}(x) = \max(0,x) + \text{weight} * \min(0,x)\)</span> where weight is a
learnable parameter.</p>
<p>See <a class="reference internal" href="#torch.nn.PReLU" title="torch.nn.PReLU"><code class="xref py py-class docutils literal notranslate"><span class="pre">PReLU</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id34">
<h3><span class="hidden-section">rrelu</span><a class="headerlink" href="#id34" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.rrelu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">rrelu</code><span class="sig-paren">(</span><em>input</em>, <em>lower=1./8</em>, <em>upper=1./3</em>, <em>training=False</em>, <em>inplace=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#rrelu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.rrelu" title="Permalink to this definition">¶</a></dt>
<dd><p>Randomized leaky ReLU.</p>
<p>See <a class="reference internal" href="#torch.nn.RReLU" title="torch.nn.RReLU"><code class="xref py py-class docutils literal notranslate"><span class="pre">RReLU</span></code></a> for more details.</p>
</dd></dl>

<dl class="function">
<dt id="torch.nn.functional.rrelu_">
<code class="descclassname">torch.nn.functional.</code><code class="descname">rrelu_</code><span class="sig-paren">(</span><em>input</em>, <em>lower=1./8</em>, <em>upper=1./3</em>, <em>training=False</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.rrelu_" title="Permalink to this definition">¶</a></dt>
<dd><p>In-place version of <a class="reference internal" href="#torch.nn.functional.rrelu" title="torch.nn.functional.rrelu"><code class="xref py py-func docutils literal notranslate"><span class="pre">rrelu()</span></code></a>.</p>
</dd></dl>

</div>
<div class="section" id="glu">
<h3><span class="hidden-section">glu</span><a class="headerlink" href="#glu" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.glu">
<code class="descclassname">torch.nn.functional.</code><code class="descname">glu</code><span class="sig-paren">(</span><em>input</em>, <em>dim=-1</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#glu"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.glu" title="Permalink to this definition">¶</a></dt>
<dd><p>The gated linear unit. Computes:</p>
<div class="math">
\[\text{GLU}(a, b) = a \otimes \sigma(b)

\]</div>
<p>where <cite>input</cite> is split in half along <cite>dim</cite> to form <cite>a</cite> and <cite>b</cite>, <span class="math">\(\sigma\)</span>
is the sigmoid function and <span class="math">\(\otimes\)</span> is the element-wise product between matrices.</p>
<p>See <a class="reference external" href="https://arxiv.org/abs/1612.08083">Language Modeling with Gated Convolutional Networks</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input tensor</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – dimension on which to split the input. Default: -1</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="id35">
<h3><span class="hidden-section">logsigmoid</span><a class="headerlink" href="#id35" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.logsigmoid">
<code class="descclassname">torch.nn.functional.</code><code class="descname">logsigmoid</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.logsigmoid" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise <span class="math">\(\text{LogSigmoid}(x_i) = \log \left(\frac{1}{1 + \exp(-x_i)}\right)\)</span></p>
<p>See <a class="reference internal" href="#torch.nn.LogSigmoid" title="torch.nn.LogSigmoid"><code class="xref py py-class docutils literal notranslate"><span class="pre">LogSigmoid</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id36">
<h3><span class="hidden-section">hardshrink</span><a class="headerlink" href="#id36" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.hardshrink">
<code class="descclassname">torch.nn.functional.</code><code class="descname">hardshrink</code><span class="sig-paren">(</span><em>input</em>, <em>lambd=0.5</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#hardshrink"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.hardshrink" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the hard shrinkage function element-wise</p>
<p>See <a class="reference internal" href="#torch.nn.Hardshrink" title="torch.nn.Hardshrink"><code class="xref py py-class docutils literal notranslate"><span class="pre">Hardshrink</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id37">
<h3><span class="hidden-section">tanhshrink</span><a class="headerlink" href="#id37" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.tanhshrink">
<code class="descclassname">torch.nn.functional.</code><code class="descname">tanhshrink</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#tanhshrink"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.tanhshrink" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise, <span class="math">\(\text{Tanhshrink}(x) = x - \text{Tanh}(x)\)</span></p>
<p>See <a class="reference internal" href="#torch.nn.Tanhshrink" title="torch.nn.Tanhshrink"><code class="xref py py-class docutils literal notranslate"><span class="pre">Tanhshrink</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id38">
<h3><span class="hidden-section">softsign</span><a class="headerlink" href="#id38" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.softsign">
<code class="descclassname">torch.nn.functional.</code><code class="descname">softsign</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#softsign"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.softsign" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise, the function <span class="math">\(\text{SoftSign}(x) = \frac{x}{1 + |x|}\)</span></p>
<p>See <a class="reference internal" href="#torch.nn.Softsign" title="torch.nn.Softsign"><code class="xref py py-class docutils literal notranslate"><span class="pre">Softsign</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id39">
<h3><span class="hidden-section">softplus</span><a class="headerlink" href="#id39" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.softplus">
<code class="descclassname">torch.nn.functional.</code><code class="descname">softplus</code><span class="sig-paren">(</span><em>input</em>, <em>beta=1</em>, <em>threshold=20</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.softplus" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</div>
<div class="section" id="id40">
<h3><span class="hidden-section">softmin</span><a class="headerlink" href="#id40" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.softmin">
<code class="descclassname">torch.nn.functional.</code><code class="descname">softmin</code><span class="sig-paren">(</span><em>input</em>, <em>dim=None</em>, <em>_stacklevel=3</em>, <em>dtype=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#softmin"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.softmin" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a softmin function.</p>
<p>Note that <span class="math">\(\text{Softmin}(x) = \text{Softmax}(-x)\)</span>. See softmax definition for mathematical formula.</p>
<p>See <a class="reference internal" href="#torch.nn.Softmin" title="torch.nn.Softmin"><code class="xref py py-class docutils literal notranslate"><span class="pre">Softmin</span></code></a> for more details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – A dimension along which softmin will be computed (so every slice
along dim will sum to 1).</p></li>
<li><p><strong>dtype</strong> (<code class="xref py py-class docutils literal notranslate"><span class="pre">torch.dtype</span></code>, optional) – the desired data type of returned tensor.
If specified, the input tensor is casted to <code class="xref py py-attr docutils literal notranslate"><span class="pre">dtype</span></code> before the operation
is performed. This is useful for preventing data type overflows. Default: None.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="id41">
<h3><span class="hidden-section">softmax</span><a class="headerlink" href="#id41" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.softmax">
<code class="descclassname">torch.nn.functional.</code><code class="descname">softmax</code><span class="sig-paren">(</span><em>input</em>, <em>dim=None</em>, <em>_stacklevel=3</em>, <em>dtype=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#softmax"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.softmax" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a softmax function.</p>
<p>Softmax is defined as:</p>
<p><span class="math">\(\text{Softmax}(x_{i}) = \frac{exp(x_i)}{\sum_j exp(x_j)}\)</span></p>
<p>It is applied to all slices along dim, and will re-scale them so that the elements
lie in the range <cite>[0, 1]</cite> and sum to 1.</p>
<p>See <a class="reference internal" href="#torch.nn.Softmax" title="torch.nn.Softmax"><code class="xref py py-class docutils literal notranslate"><span class="pre">Softmax</span></code></a> for more details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – A dimension along which softmax will be computed.</p></li>
<li><p><strong>dtype</strong> (<code class="xref py py-class docutils literal notranslate"><span class="pre">torch.dtype</span></code>, optional) – the desired data type of returned tensor.
If specified, the input tensor is casted to <code class="xref py py-attr docutils literal notranslate"><span class="pre">dtype</span></code> before the operation
is performed. This is useful for preventing data type overflows. Default: None.</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This function doesn’t work directly with NLLLoss,
which expects the Log to be computed between the Softmax and itself.
Use log_softmax instead (it’s faster and has better numerical properties).</p>
</div>
</dd></dl>

</div>
<div class="section" id="id42">
<h3><span class="hidden-section">softshrink</span><a class="headerlink" href="#id42" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.softshrink">
<code class="descclassname">torch.nn.functional.</code><code class="descname">softshrink</code><span class="sig-paren">(</span><em>input</em>, <em>lambd=0.5</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.softshrink" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the soft shrinkage function elementwise</p>
<p>See <a class="reference internal" href="#torch.nn.Softshrink" title="torch.nn.Softshrink"><code class="xref py py-class docutils literal notranslate"><span class="pre">Softshrink</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="gumbel-softmax">
<h3><span class="hidden-section">gumbel_softmax</span><a class="headerlink" href="#gumbel-softmax" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.gumbel_softmax">
<code class="descclassname">torch.nn.functional.</code><code class="descname">gumbel_softmax</code><span class="sig-paren">(</span><em>logits</em>, <em>tau=1</em>, <em>hard=False</em>, <em>eps=1e-10</em>, <em>dim=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#gumbel_softmax"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.gumbel_softmax" title="Permalink to this definition">¶</a></dt>
<dd><p>Samples from the <a class="reference external" href="https://arxiv.org/abs/1611.00712https://arxiv.org/abs/1611.01144">Gumbel-Softmax distribution</a> and optionally discretizes.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>logits</strong> – <cite>[…, num_features]</cite> unnormalized log probabilities</p></li>
<li><p><strong>tau</strong> – non-negative scalar temperature</p></li>
<li><p><strong>hard</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, the returned samples will be discretized as one-hot vectors,
but will be differentiated as if it is the soft sample in autograd</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – A dimension along which softmax will be computed. Default: -1.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Sampled tensor of same shape as <cite>logits</cite> from the Gumbel-Softmax distribution.
If <code class="docutils literal notranslate"><span class="pre">hard=True</span></code>, the returned samples will be one-hot, otherwise they will
be probability distributions that sum to 1 across <cite>dim</cite>.</p>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This function is here for legacy reasons, may be removed from nn.Functional in the future.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The main trick for <cite>hard</cite> is to do  <cite>y_hard - y_soft.detach() + y_soft</cite></p>
<p>It achieves two things:
- makes the output value exactly one-hot
(since we add then subtract y_soft value)
- makes the gradient equal to y_soft gradient
(since we strip all other gradients)</p>
</div>
<dl>
<dt>Examples::</dt><dd><div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">logits</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">32</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Sample soft categorical using reparametrization trick:</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">gumbel_softmax</span><span class="p">(</span><span class="n">logits</span><span class="p">,</span> <span class="n">tau</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">hard</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Sample hard categorical using &quot;Straight-through&quot; trick:</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">gumbel_softmax</span><span class="p">(</span><span class="n">logits</span><span class="p">,</span> <span class="n">tau</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">hard</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</pre></div>
</div>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="log-softmax">
<h3><span class="hidden-section">log_softmax</span><a class="headerlink" href="#log-softmax" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.log_softmax">
<code class="descclassname">torch.nn.functional.</code><code class="descname">log_softmax</code><span class="sig-paren">(</span><em>input</em>, <em>dim=None</em>, <em>_stacklevel=3</em>, <em>dtype=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#log_softmax"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.log_softmax" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a softmax followed by a logarithm.</p>
<p>While mathematically equivalent to log(softmax(x)), doing these two
operations separately is slower, and numerically unstable. This function
uses an alternative formulation to compute the output and gradient correctly.</p>
<p>See <a class="reference internal" href="#torch.nn.LogSoftmax" title="torch.nn.LogSoftmax"><code class="xref py py-class docutils literal notranslate"><span class="pre">LogSoftmax</span></code></a> for more details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – A dimension along which log_softmax will be computed.</p></li>
<li><p><strong>dtype</strong> (<code class="xref py py-class docutils literal notranslate"><span class="pre">torch.dtype</span></code>, optional) – the desired data type of returned tensor.
If specified, the input tensor is casted to <code class="xref py py-attr docutils literal notranslate"><span class="pre">dtype</span></code> before the operation
is performed. This is useful for preventing data type overflows. Default: None.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="id43">
<h3><span class="hidden-section">tanh</span><a class="headerlink" href="#id43" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.tanh">
<code class="descclassname">torch.nn.functional.</code><code class="descname">tanh</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#tanh"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.tanh" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies element-wise,
<span class="math">\(\text{Tanh}(x) = \tanh(x) = \frac{\exp(x) - \exp(-x)}{\exp(x) + \exp(-x)}\)</span></p>
<p>See <a class="reference internal" href="#torch.nn.Tanh" title="torch.nn.Tanh"><code class="xref py py-class docutils literal notranslate"><span class="pre">Tanh</span></code></a> for more details.</p>
</dd></dl>

</div>
<div class="section" id="id44">
<h3><span class="hidden-section">sigmoid</span><a class="headerlink" href="#id44" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.sigmoid">
<code class="descclassname">torch.nn.functional.</code><code class="descname">sigmoid</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#sigmoid"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.sigmoid" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies the element-wise function <span class="math">\(\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}\)</span></p>
<p>See <a class="reference internal" href="#torch.nn.Sigmoid" title="torch.nn.Sigmoid"><code class="xref py py-class docutils literal notranslate"><span class="pre">Sigmoid</span></code></a> for more details.</p>
</dd></dl>

</div>
</div>
<div class="section" id="normalization-functions">
<h2>Normalization functions<a class="headerlink" href="#normalization-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="batch-norm">
<h3><span class="hidden-section">batch_norm</span><a class="headerlink" href="#batch-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.batch_norm">
<code class="descclassname">torch.nn.functional.</code><code class="descname">batch_norm</code><span class="sig-paren">(</span><em>input</em>, <em>running_mean</em>, <em>running_var</em>, <em>weight=None</em>, <em>bias=None</em>, <em>training=False</em>, <em>momentum=0.1</em>, <em>eps=1e-05</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#batch_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.batch_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Batch Normalization for each channel across a batch of data.</p>
<p>See <a class="reference internal" href="#torch.nn.BatchNorm1d" title="torch.nn.BatchNorm1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">BatchNorm1d</span></code></a>, <a class="reference internal" href="#torch.nn.BatchNorm2d" title="torch.nn.BatchNorm2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">BatchNorm2d</span></code></a>,
<a class="reference internal" href="#torch.nn.BatchNorm3d" title="torch.nn.BatchNorm3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">BatchNorm3d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="instance-norm">
<h3><span class="hidden-section">instance_norm</span><a class="headerlink" href="#instance-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.instance_norm">
<code class="descclassname">torch.nn.functional.</code><code class="descname">instance_norm</code><span class="sig-paren">(</span><em>input</em>, <em>running_mean=None</em>, <em>running_var=None</em>, <em>weight=None</em>, <em>bias=None</em>, <em>use_input_stats=True</em>, <em>momentum=0.1</em>, <em>eps=1e-05</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#instance_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.instance_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Instance Normalization for each channel in each data sample in a
batch.</p>
<p>See <a class="reference internal" href="#torch.nn.InstanceNorm1d" title="torch.nn.InstanceNorm1d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm1d</span></code></a>, <a class="reference internal" href="#torch.nn.InstanceNorm2d" title="torch.nn.InstanceNorm2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm2d</span></code></a>,
<a class="reference internal" href="#torch.nn.InstanceNorm3d" title="torch.nn.InstanceNorm3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">InstanceNorm3d</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="layer-norm">
<h3><span class="hidden-section">layer_norm</span><a class="headerlink" href="#layer-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.layer_norm">
<code class="descclassname">torch.nn.functional.</code><code class="descname">layer_norm</code><span class="sig-paren">(</span><em>input</em>, <em>normalized_shape</em>, <em>weight=None</em>, <em>bias=None</em>, <em>eps=1e-05</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#layer_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.layer_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies Layer Normalization for last certain number of dimensions.</p>
<p>See <a class="reference internal" href="#torch.nn.LayerNorm" title="torch.nn.LayerNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LayerNorm</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="local-response-norm">
<h3><span class="hidden-section">local_response_norm</span><a class="headerlink" href="#local-response-norm" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.local_response_norm">
<code class="descclassname">torch.nn.functional.</code><code class="descname">local_response_norm</code><span class="sig-paren">(</span><em>input</em>, <em>size</em>, <em>alpha=0.0001</em>, <em>beta=0.75</em>, <em>k=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#local_response_norm"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.local_response_norm" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies local response normalization over an input signal composed of
several input planes, where channels occupy the second dimension.
Applies normalization across channels.</p>
<p>See <a class="reference internal" href="#torch.nn.LocalResponseNorm" title="torch.nn.LocalResponseNorm"><code class="xref py py-class docutils literal notranslate"><span class="pre">LocalResponseNorm</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="normalize">
<h3><span class="hidden-section">normalize</span><a class="headerlink" href="#normalize" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.normalize">
<code class="descclassname">torch.nn.functional.</code><code class="descname">normalize</code><span class="sig-paren">(</span><em>input</em>, <em>p=2</em>, <em>dim=1</em>, <em>eps=1e-12</em>, <em>out=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#normalize"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.normalize" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs <span class="math">\(L_p\)</span> normalization of inputs over specified dimension.</p>
<p>For a tensor <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> of sizes <span class="math">\((n_0, ..., n_{dim}, ..., n_k)\)</span>, each
<span class="math">\(n_{dim}\)</span> -element vector <span class="math">\(v\)</span> along dimension <code class="xref py py-attr docutils literal notranslate"><span class="pre">dim</span></code> is transformed as</p>
<div class="math">
\[v = \frac{v}{\max(\lVert v \rVert_p, \epsilon)}.

\]</div>
<p>With the default arguments it uses the Euclidean norm over vectors along dimension <span class="math">\(1\)</span> for normalization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of any shape</p></li>
<li><p><strong>p</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – the exponent value in the norm formulation. Default: 2</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – the dimension to reduce. Default: 1</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – small value to avoid division by zero. Default: 1e-12</p></li>
<li><p><strong>out</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – the output tensor. If <code class="xref py py-attr docutils literal notranslate"><span class="pre">out</span></code> is used, this
operation won’t be differentiable.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
</div>
<div class="section" id="linear-functions">
<h2>Linear functions<a class="headerlink" href="#linear-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="id45">
<h3><span class="hidden-section">linear</span><a class="headerlink" href="#id45" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.linear">
<code class="descclassname">torch.nn.functional.</code><code class="descname">linear</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>bias=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#linear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.linear" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies a linear transformation to the incoming data: <span class="math">\(y = xA^T + b\)</span>.</p>
<p>Shape:</p>
<blockquote>
<div><ul class="simple">
<li><p>Input: <span class="math">\((N, *, in\_features)\)</span> where <cite>*</cite> means any number of
additional dimensions</p></li>
<li><p>Weight: <span class="math">\((out\_features, in\_features)\)</span></p></li>
<li><p>Bias: <span class="math">\((out\_features)\)</span></p></li>
<li><p>Output: <span class="math">\((N, *, out\_features)\)</span></p></li>
</ul>
</div></blockquote>
</dd></dl>

</div>
<div class="section" id="id46">
<h3><span class="hidden-section">bilinear</span><a class="headerlink" href="#id46" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.bilinear">
<code class="descclassname">torch.nn.functional.</code><code class="descname">bilinear</code><span class="sig-paren">(</span><em>input1</em>, <em>input2</em>, <em>weight</em>, <em>bias=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#bilinear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.bilinear" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</div>
</div>
<div class="section" id="dropout-functions">
<h2>Dropout functions<a class="headerlink" href="#dropout-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="id47">
<h3><span class="hidden-section">dropout</span><a class="headerlink" href="#id47" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.dropout">
<code class="descclassname">torch.nn.functional.</code><code class="descname">dropout</code><span class="sig-paren">(</span><em>input</em>, <em>p=0.5</em>, <em>training=True</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#dropout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.dropout" title="Permalink to this definition">¶</a></dt>
<dd><p>During training, randomly zeroes some of the elements of the input
tensor with probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">p</span></code> using samples from a Bernoulli
distribution.</p>
<p>See <a class="reference internal" href="#torch.nn.Dropout" title="torch.nn.Dropout"><code class="xref py py-class docutils literal notranslate"><span class="pre">Dropout</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> – probability of an element to be zeroed. Default: 0.5</p></li>
<li><p><strong>training</strong> – apply dropout if is <code class="docutils literal notranslate"><span class="pre">True</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>inplace</strong> – If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, will do this operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="alpha-dropout">
<h3><span class="hidden-section">alpha_dropout</span><a class="headerlink" href="#alpha-dropout" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.alpha_dropout">
<code class="descclassname">torch.nn.functional.</code><code class="descname">alpha_dropout</code><span class="sig-paren">(</span><em>input</em>, <em>p=0.5</em>, <em>training=False</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#alpha_dropout"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.alpha_dropout" title="Permalink to this definition">¶</a></dt>
<dd><p>Applies alpha dropout to the input.</p>
<p>See <a class="reference internal" href="#torch.nn.AlphaDropout" title="torch.nn.AlphaDropout"><code class="xref py py-class docutils literal notranslate"><span class="pre">AlphaDropout</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="id48">
<h3><span class="hidden-section">dropout2d</span><a class="headerlink" href="#id48" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.dropout2d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">dropout2d</code><span class="sig-paren">(</span><em>input</em>, <em>p=0.5</em>, <em>training=True</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#dropout2d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.dropout2d" title="Permalink to this definition">¶</a></dt>
<dd><p>Randomly zero out entire channels (a channel is a 2D feature map,
e.g., the <span class="math">\(j\)</span>-th channel of the <span class="math">\(i\)</span>-th sample in the
batched input is a 2D tensor <span class="math">\(\text{input}[i, j]\)</span>) of the input tensor).
Each channel will be zeroed out independently on every forward call with
probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">p</span></code> using samples from a Bernoulli distribution.</p>
<p>See <a class="reference internal" href="#torch.nn.Dropout2d" title="torch.nn.Dropout2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Dropout2d</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> – probability of a channel to be zeroed. Default: 0.5</p></li>
<li><p><strong>training</strong> – apply dropout if is <code class="docutils literal notranslate"><span class="pre">True</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>inplace</strong> – If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, will do this operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="id49">
<h3><span class="hidden-section">dropout3d</span><a class="headerlink" href="#id49" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.dropout3d">
<code class="descclassname">torch.nn.functional.</code><code class="descname">dropout3d</code><span class="sig-paren">(</span><em>input</em>, <em>p=0.5</em>, <em>training=True</em>, <em>inplace=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#dropout3d"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.dropout3d" title="Permalink to this definition">¶</a></dt>
<dd><p>Randomly zero out entire channels (a channel is a 3D feature map,
e.g., the <span class="math">\(j\)</span>-th channel of the <span class="math">\(i\)</span>-th sample in the
batched input is a 3D tensor <span class="math">\(\text{input}[i, j]\)</span>) of the input tensor).
Each channel will be zeroed out independently on every forward call with
probability <code class="xref py py-attr docutils literal notranslate"><span class="pre">p</span></code> using samples from a Bernoulli distribution.</p>
<p>See <a class="reference internal" href="#torch.nn.Dropout3d" title="torch.nn.Dropout3d"><code class="xref py py-class docutils literal notranslate"><span class="pre">Dropout3d</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>p</strong> – probability of a channel to be zeroed. Default: 0.5</p></li>
<li><p><strong>training</strong> – apply dropout if is <code class="docutils literal notranslate"><span class="pre">True</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>inplace</strong> – If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, will do this operation in-place. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
</div>
<div class="section" id="sparse-functions">
<h2>Sparse functions<a class="headerlink" href="#sparse-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="id50">
<h3><span class="hidden-section">embedding</span><a class="headerlink" href="#id50" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.embedding">
<code class="descclassname">torch.nn.functional.</code><code class="descname">embedding</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>padding_idx=None</em>, <em>max_norm=None</em>, <em>norm_type=2.0</em>, <em>scale_grad_by_freq=False</em>, <em>sparse=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#embedding"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.embedding" title="Permalink to this definition">¶</a></dt>
<dd><p>A simple lookup table that looks up embeddings in a fixed dictionary and size.</p>
<p>This module is often used to retrieve word embeddings using indices.
The input to the module is a list of indices, and the embedding matrix,
and the output is the corresponding word embeddings.</p>
<p>See <a class="reference internal" href="#torch.nn.Embedding" title="torch.nn.Embedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.Embedding</span></code></a> for more details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<em>LongTensor</em>) – Tensor containing indices into the embedding matrix</p></li>
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – The embedding matrix with number of rows equal to the maximum possible index + 1,
and number of columns equal to the embedding size</p></li>
<li><p><strong>padding_idx</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – If given, pads the output with the embedding vector at <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding_idx</span></code>
(initialized to zeros) whenever it encounters the index.</p></li>
<li><p><strong>max_norm</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – If given, each embedding vector with norm larger than <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>
is renormalized to have norm <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>.
Note: this will modify <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> in-place.</p></li>
<li><p><strong>norm_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – The p of the p-norm to compute for the <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code> option. Default <code class="docutils literal notranslate"><span class="pre">2</span></code>.</p></li>
<li><p><strong>scale_grad_by_freq</strong> (<em>boolean</em><em>, </em><em>optional</em>) – If given, this will scale gradients by the inverse of frequency of
the words in the mini-batch. Default <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>sparse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, gradient w.r.t. <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> will be a sparse tensor. See Notes under
<a class="reference internal" href="#torch.nn.Embedding" title="torch.nn.Embedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.Embedding</span></code></a> for more details regarding sparse gradients.</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: LongTensor of arbitrary shape containing the indices to extract</p></li>
<li><dl class="simple">
<dt>Weight: Embedding matrix of floating point type with shape <cite>(V, embedding_dim)</cite>,</dt><dd><p>where V = maximum index + 1 and embedding_dim = the embedding size</p>
</dd>
</dl>
</li>
<li><p>Output: <cite>(*, embedding_dim)</cite>, where <cite>*</cite> is the input shape</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># a batch of 2 samples of 4 indices each</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">],[</span><span class="mi">4</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">9</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># an embedding matrix containing 10 tensors of size 3</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">embedding_matrix</span><span class="p">)</span>
<span class="go">tensor([[[ 0.8490,  0.9625,  0.6753],</span>
<span class="go">         [ 0.9666,  0.7761,  0.6108],</span>
<span class="go">         [ 0.6246,  0.9751,  0.3618],</span>
<span class="go">         [ 0.4161,  0.2419,  0.7383]],</span>

<span class="go">        [[ 0.6246,  0.9751,  0.3618],</span>
<span class="go">         [ 0.0237,  0.7794,  0.0528],</span>
<span class="go">         [ 0.9666,  0.7761,  0.6108],</span>
<span class="go">         [ 0.3385,  0.8612,  0.1867]]])</span>

<span class="gp">&gt;&gt;&gt; </span><span class="c1"># example with padding_idx</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weights</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">weights</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="p">:]</span><span class="o">.</span><span class="n">zero_</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">weights</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">0</span><span class="p">,</span><span class="mi">5</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">embedding</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">embedding_matrix</span><span class="p">,</span> <span class="n">padding_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="go">tensor([[[ 0.0000,  0.0000,  0.0000],</span>
<span class="go">         [ 0.5609,  0.5384,  0.8720],</span>
<span class="go">         [ 0.0000,  0.0000,  0.0000],</span>
<span class="go">         [ 0.6262,  0.2438,  0.7471]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="embedding-bag">
<h3><span class="hidden-section">embedding_bag</span><a class="headerlink" href="#embedding-bag" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.embedding_bag">
<code class="descclassname">torch.nn.functional.</code><code class="descname">embedding_bag</code><span class="sig-paren">(</span><em>input</em>, <em>weight</em>, <em>offsets=None</em>, <em>max_norm=None</em>, <em>norm_type=2</em>, <em>scale_grad_by_freq=False</em>, <em>mode='mean'</em>, <em>sparse=False</em>, <em>per_sample_weights=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#embedding_bag"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.embedding_bag" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes sums, means or maxes of <cite>bags</cite> of embeddings, without instantiating the
intermediate embeddings.</p>
<p>See <a class="reference internal" href="#torch.nn.EmbeddingBag" title="torch.nn.EmbeddingBag"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.EmbeddingBag</span></code></a> for more details.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<em>LongTensor</em>) – Tensor containing bags of indices into the embedding matrix</p></li>
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – The embedding matrix with number of rows equal to the maximum possible index + 1,
and number of columns equal to the embedding size</p></li>
<li><p><strong>offsets</strong> (<em>LongTensor</em><em>, </em><em>optional</em>) – Only used when <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> is 1D. <code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> determines
the starting index position of each bag (sequence) in <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code>.</p></li>
<li><p><strong>max_norm</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – If given, each embedding vector with norm larger than <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>
is renormalized to have norm <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code>.
Note: this will modify <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> in-place.</p></li>
<li><p><strong>norm_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – The <code class="docutils literal notranslate"><span class="pre">p</span></code> in the <code class="docutils literal notranslate"><span class="pre">p</span></code>-norm to compute for the <code class="xref py py-attr docutils literal notranslate"><span class="pre">max_norm</span></code> option.
Default <code class="docutils literal notranslate"><span class="pre">2</span></code>.</p></li>
<li><p><strong>scale_grad_by_freq</strong> (<em>boolean</em><em>, </em><em>optional</em>) – if given, this will scale gradients by the inverse of frequency of
the words in the mini-batch. Default <code class="docutils literal notranslate"><span class="pre">False</span></code>.
Note: this option is not supported when <code class="docutils literal notranslate"><span class="pre">mode=&quot;max&quot;</span></code>.</p></li>
<li><p><strong>mode</strong> (<em>string</em><em>, </em><em>optional</em>) – <code class="docutils literal notranslate"><span class="pre">&quot;sum&quot;</span></code>, <code class="docutils literal notranslate"><span class="pre">&quot;mean&quot;</span></code> or <code class="docutils literal notranslate"><span class="pre">&quot;max&quot;</span></code>. Specifies the way to reduce the bag.
Default: <code class="docutils literal notranslate"><span class="pre">&quot;mean&quot;</span></code></p></li>
<li><p><strong>sparse</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, gradient w.r.t. <code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> will be a sparse tensor. See Notes under
<a class="reference internal" href="#torch.nn.Embedding" title="torch.nn.Embedding"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.Embedding</span></code></a> for more details regarding sparse gradients.
Note: this option is not supported when <code class="docutils literal notranslate"><span class="pre">mode=&quot;max&quot;</span></code>.</p></li>
<li><p><strong>per_sample_weights</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a tensor of float / double weights, or None
to indicate all weights should be taken to be 1. If specified, <code class="xref py py-attr docutils literal notranslate"><span class="pre">per_sample_weights</span></code>
must have exactly the same shape as input and is treated as having the same
<code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code>, if those are not None.</p></li>
</ul>
</dd>
</dl>
<p>Shape:</p>
<blockquote>
<div><ul>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> (LongTensor) and <code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> (LongTensor, optional)</p>
<ul>
<li><p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> is 2D of shape <cite>(B, N)</cite>,</p>
<p>it will be treated as <code class="docutils literal notranslate"><span class="pre">B</span></code> bags (sequences) each of fixed length <code class="docutils literal notranslate"><span class="pre">N</span></code>, and
this will return <code class="docutils literal notranslate"><span class="pre">B</span></code> values aggregated in a way depending on the <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code>.
<code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> is ignored and required to be <code class="docutils literal notranslate"><span class="pre">None</span></code> in this case.</p>
</li>
<li><p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> is 1D of shape <cite>(N)</cite>,</p>
<p>it will be treated as a concatenation of multiple bags (sequences).
<code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> is required to be a 1D tensor containing the
starting index positions of each bag in <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code>. Therefore,
for <code class="xref py py-attr docutils literal notranslate"><span class="pre">offsets</span></code> of shape <cite>(B)</cite>, <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> will be viewed as
having <code class="docutils literal notranslate"><span class="pre">B</span></code> bags. Empty bags (i.e., having 0-length) will have
returned vectors filled by zeros.</p>
</li>
</ul>
</li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">weight</span></code> (Tensor): the learnable weights of the module of
shape <cite>(num_embeddings, embedding_dim)</cite></p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">per_sample_weights</span></code> (Tensor, optional). Has the same shape as
<code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code>.</p></li>
<li><p><code class="xref py py-attr docutils literal notranslate"><span class="pre">output</span></code>: aggregated embedding values of shape <cite>(B, embedding_dim)</cite></p></li>
</ul>
</div></blockquote>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># an Embedding module containing 10 tensors of size 3</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># a batch of 2 samples of 4 indices each</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">5</span><span class="p">,</span><span class="mi">4</span><span class="p">,</span><span class="mi">3</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">9</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">offsets</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span><span class="mi">4</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">embedding_bag</span><span class="p">(</span><span class="n">embedding_matrix</span><span class="p">,</span> <span class="nb">input</span><span class="p">,</span> <span class="n">offsets</span><span class="p">)</span>
<span class="go">tensor([[ 0.3397,  0.3552,  0.5545],</span>
<span class="go">        [ 0.5893,  0.4386,  0.5882]])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="one-hot">
<h3><span class="hidden-section">one_hot</span><a class="headerlink" href="#one-hot" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.one_hot">
<code class="descclassname">torch.nn.functional.</code><code class="descname">one_hot</code><span class="sig-paren">(</span><em>tensor</em>, <em>num_classes=0</em><span class="sig-paren">)</span> &#x2192; LongTensor<a class="headerlink" href="#torch.nn.functional.one_hot" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes LongTensor with index values of shape <code class="docutils literal notranslate"><span class="pre">(*)</span></code> and returns a tensor
of shape <code class="docutils literal notranslate"><span class="pre">(*,</span> <span class="pre">num_classes)</span></code> that have zeros everywhere except where the
index of last dimension matches the corresponding value of the input tensor,
in which case it will be 1.</p>
<p>See also <a class="reference external" href="https://en.wikipedia.org/wiki/One-hot">One-hot on Wikipedia</a> .</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> (<em>LongTensor</em>) – class values of any shape.</p></li>
<li><p><strong>num_classes</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Total number of classes. If set to -1, the number
of classes will be inferred as one greater than the largest class
value in the input tensor.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>LongTensor that has one more dimension with 1 values at the
index of last dimension indicated by the input, and 0 everywhere
else.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">one_hot</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span> <span class="o">%</span> <span class="mi">3</span><span class="p">)</span>
<span class="go">tensor([[1, 0, 0],</span>
<span class="go">        [0, 1, 0],</span>
<span class="go">        [0, 0, 1],</span>
<span class="go">        [1, 0, 0],</span>
<span class="go">        [0, 1, 0]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">one_hot</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span> <span class="o">%</span> <span class="mi">3</span><span class="p">,</span> <span class="n">num_classes</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
<span class="go">tensor([[1, 0, 0, 0, 0],</span>
<span class="go">        [0, 1, 0, 0, 0],</span>
<span class="go">        [0, 0, 1, 0, 0],</span>
<span class="go">        [1, 0, 0, 0, 0],</span>
<span class="go">        [0, 1, 0, 0, 0]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">F</span><span class="o">.</span><span class="n">one_hot</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">6</span><span class="p">)</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">2</span><span class="p">)</span> <span class="o">%</span> <span class="mi">3</span><span class="p">)</span>
<span class="go">tensor([[[1, 0, 0],</span>
<span class="go">         [0, 1, 0]],</span>
<span class="go">        [[0, 0, 1],</span>
<span class="go">         [1, 0, 0]],</span>
<span class="go">        [[0, 1, 0],</span>
<span class="go">         [0, 0, 1]]])</span>
</pre></div>
</div>
</dd></dl>

</div>
</div>
<div class="section" id="id51">
<h2>Distance functions<a class="headerlink" href="#id51" title="Permalink to this headline">¶</a></h2>
<div class="section" id="pairwise-distance">
<h3><span class="hidden-section">pairwise_distance</span><a class="headerlink" href="#pairwise-distance" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.pairwise_distance">
<code class="descclassname">torch.nn.functional.</code><code class="descname">pairwise_distance</code><span class="sig-paren">(</span><em>x1</em>, <em>x2</em>, <em>p=2.0</em>, <em>eps=1e-06</em>, <em>keepdim=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#pairwise_distance"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.pairwise_distance" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.PairwiseDistance" title="torch.nn.PairwiseDistance"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.PairwiseDistance</span></code></a> for details</p>
</dd></dl>

</div>
<div class="section" id="cosine-similarity">
<h3><span class="hidden-section">cosine_similarity</span><a class="headerlink" href="#cosine-similarity" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.cosine_similarity">
<code class="descclassname">torch.nn.functional.</code><code class="descname">cosine_similarity</code><span class="sig-paren">(</span><em>x1</em>, <em>x2</em>, <em>dim=1</em>, <em>eps=1e-8</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.cosine_similarity" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns cosine similarity between x1 and x2, computed along dim.</p>
<div class="math">
\[\text{similarity} = \dfrac{x_1 \cdot x_2}{\max(\Vert x_1 \Vert _2 \cdot \Vert x_2 \Vert _2, \epsilon)}

\]</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>x1</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – First input.</p></li>
<li><p><strong>x2</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – Second input (of size matching x1).</p></li>
<li><p><strong>dim</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Dimension of vectors. Default: 1</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Small value to avoid division by zero.
Default: 1e-8</p></li>
</ul>
</dd>
</dl>
<dl class="simple">
<dt>Shape:</dt><dd><ul class="simple">
<li><p>Input: <span class="math">\((\ast_1, D, \ast_2)\)</span> where D is at position <cite>dim</cite>.</p></li>
<li><p>Output: <span class="math">\((\ast_1, \ast_2)\)</span> where 1 is at position <cite>dim</cite>.</p></li>
</ul>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">input1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="mi">128</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">cosine_similarity</span><span class="p">(</span><span class="n">input1</span><span class="p">,</span> <span class="n">input2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="pdist">
<h3><span class="hidden-section">pdist</span><a class="headerlink" href="#pdist" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.pdist">
<code class="descclassname">torch.nn.functional.</code><code class="descname">pdist</code><span class="sig-paren">(</span><em>input</em>, <em>p=2</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="headerlink" href="#torch.nn.functional.pdist" title="Permalink to this definition">¶</a></dt>
<dd><p>Computes the p-norm distance between every pair of row vectors in the input.
This is identical to the upper triangular portion, excluding the diagonal, of
<cite>torch.norm(input[:, None] - input, dim=2, p=p)</cite>. This function will be faster
if the rows are contiguous.</p>
<p>If input has shape <span class="math">\(N \times M\)</span> then the output will have shape
<span class="math">\(\frac{1}{2} N (N - 1)\)</span>.</p>
<p>This function is equivalent to <cite>scipy.spatial.distance.pdist(input,
‘minkowski’, p=p)</cite> if <span class="math">\(p \in (0, \infty)\)</span>. When <span class="math">\(p = 0\)</span> it is
equivalent to <cite>scipy.spatial.distance.pdist(input, ‘hamming’) * M</cite>.
When <span class="math">\(p = \infty\)</span>, the closest scipy function is
<cite>scipy.spatial.distance.pdist(xn, lambda x, y: np.abs(x - y).max())</cite>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – input tensor of shape <span class="math">\(N \times M\)</span>.</p></li>
<li><p><strong>p</strong> – p value for the p-norm distance to calculate between each vector pair
<span class="math">\(\in [0, \infty]\)</span>.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
</div>
<div class="section" id="id52">
<h2>Loss functions<a class="headerlink" href="#id52" title="Permalink to this headline">¶</a></h2>
<div class="section" id="binary-cross-entropy">
<h3><span class="hidden-section">binary_cross_entropy</span><a class="headerlink" href="#binary-cross-entropy" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.binary_cross_entropy">
<code class="descclassname">torch.nn.functional.</code><code class="descname">binary_cross_entropy</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>weight=None</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#binary_cross_entropy"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.binary_cross_entropy" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that measures the Binary Cross Entropy
between the target and the output.</p>
<p>See <a class="reference internal" href="#torch.nn.BCELoss" title="torch.nn.BCELoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">BCELoss</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – Tensor of arbitrary shape</p></li>
<li><p><strong>target</strong> – Tensor of the same shape as input</p></li>
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight
if provided it’s repeated to match input tensor shape</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">binary_cross_entropy</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">sigmoid</span><span class="p">(</span><span class="nb">input</span><span class="p">),</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="binary-cross-entropy-with-logits">
<h3><span class="hidden-section">binary_cross_entropy_with_logits</span><a class="headerlink" href="#binary-cross-entropy-with-logits" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.binary_cross_entropy_with_logits">
<code class="descclassname">torch.nn.functional.</code><code class="descname">binary_cross_entropy_with_logits</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>weight=None</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em>, <em>pos_weight=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#binary_cross_entropy_with_logits"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.binary_cross_entropy_with_logits" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that measures Binary Cross Entropy between target and output
logits.</p>
<p>See <a class="reference internal" href="#torch.nn.BCEWithLogitsLoss" title="torch.nn.BCEWithLogitsLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">BCEWithLogitsLoss</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – Tensor of arbitrary shape</p></li>
<li><p><strong>target</strong> – Tensor of the same shape as input</p></li>
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight
if provided it’s repeated to match input tensor shape</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
<li><p><strong>pos_weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a weight of positive examples.
Must be a vector with length equal to the number of classes.</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="o">.</span><span class="n">random_</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">binary_cross_entropy_with_logits</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="poisson-nll-loss">
<h3><span class="hidden-section">poisson_nll_loss</span><a class="headerlink" href="#poisson-nll-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.poisson_nll_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">poisson_nll_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>log_input=True</em>, <em>full=False</em>, <em>size_average=None</em>, <em>eps=1e-08</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#poisson_nll_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.poisson_nll_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>Poisson negative log likelihood loss.</p>
<p>See <a class="reference internal" href="#torch.nn.PoissonNLLLoss" title="torch.nn.PoissonNLLLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">PoissonNLLLoss</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – expectation of underlying Poisson distribution.</p></li>
<li><p><strong>target</strong> – random sample <span class="math">\(target \sim \text{Poisson}(input)\)</span>.</p></li>
<li><p><strong>log_input</strong> – if <code class="docutils literal notranslate"><span class="pre">True</span></code> the loss is computed as
<span class="math">\(\exp(\text{input}) - \text{target} * \text{input}\)</span>, if <code class="docutils literal notranslate"><span class="pre">False</span></code> then loss is
<span class="math">\(\text{input} - \text{target} * \log(\text{input}+\text{eps})\)</span>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>full</strong> – whether to compute full loss, i. e. to add the Stirling
approximation term. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>
<span class="math">\(\text{target} * \log(\text{target}) - \text{target} + 0.5 * \log(2 * \pi * \text{target})\)</span>.</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – Small value to avoid evaluation of <span class="math">\(\log(0)\)</span> when
<code class="xref py py-attr docutils literal notranslate"><span class="pre">log_input`=``False`</span></code>. Default: 1e-8</p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="cosine-embedding-loss">
<h3><span class="hidden-section">cosine_embedding_loss</span><a class="headerlink" href="#cosine-embedding-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.cosine_embedding_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">cosine_embedding_loss</code><span class="sig-paren">(</span><em>input1</em>, <em>input2</em>, <em>target</em>, <em>margin=0</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#cosine_embedding_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.cosine_embedding_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.CosineEmbeddingLoss" title="torch.nn.CosineEmbeddingLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">CosineEmbeddingLoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="cross-entropy">
<h3><span class="hidden-section">cross_entropy</span><a class="headerlink" href="#cross-entropy" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.cross_entropy">
<code class="descclassname">torch.nn.functional.</code><code class="descname">cross_entropy</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>weight=None</em>, <em>size_average=None</em>, <em>ignore_index=-100</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#cross_entropy"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.cross_entropy" title="Permalink to this definition">¶</a></dt>
<dd><p>This criterion combines <cite>log_softmax</cite> and <cite>nll_loss</cite> in a single
function.</p>
<p>See <a class="reference internal" href="#torch.nn.CrossEntropyLoss" title="torch.nn.CrossEntropyLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">CrossEntropyLoss</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – <span class="math">\((N, C)\)</span> where <cite>C = number of classes</cite> or <span class="math">\((N, C, H, W)\)</span>
in case of 2D Loss, or <span class="math">\((N, C, d_1, d_2, ..., d_K)\)</span> where <span class="math">\(K \geq 1\)</span>
in the case of K-dimensional loss.</p></li>
<li><p><strong>target</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – <span class="math">\((N)\)</span> where each value is <span class="math">\(0 \leq \text{targets}[i] \leq C-1\)</span>,
or <span class="math">\((N, d_1, d_2, ..., d_K)\)</span> where <span class="math">\(K \geq 1\)</span> for
K-dimensional loss.</p></li>
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to each
class. If given, has to be a Tensor of size <cite>C</cite></p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>ignore_index</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Specifies a target value that is ignored
and does not contribute to the input gradient. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code> is
<code class="docutils literal notranslate"><span class="pre">True</span></code>, the loss is averaged over non-ignored targets. Default: -100</p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">int64</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">cross_entropy</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="ctc-loss">
<h3><span class="hidden-section">ctc_loss</span><a class="headerlink" href="#ctc-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.ctc_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">ctc_loss</code><span class="sig-paren">(</span><em>log_probs</em>, <em>targets</em>, <em>input_lengths</em>, <em>target_lengths</em>, <em>blank=0</em>, <em>reduction='mean'</em>, <em>zero_infinity=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#ctc_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.ctc_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>The Connectionist Temporal Classification loss.</p>
<p>See <a class="reference internal" href="#torch.nn.CTCLoss" title="torch.nn.CTCLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">CTCLoss</span></code></a> for details.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In some circumstances when using the CUDA backend with CuDNN, this operator
may select a nondeterministic algorithm to increase performance. If this is
undesirable, you can try to make the operation deterministic (potentially at
a performance cost) by setting <code class="docutils literal notranslate"><span class="pre">torch.backends.cudnn.deterministic</span> <span class="pre">=</span>
<span class="pre">True</span></code>.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>log_probs</strong> – <span class="math">\((T, N, C)\)</span> where <cite>C = number of characters in alphabet including blank</cite>,
<cite>T = input length</cite>, and <cite>N = batch size</cite>.
The logarithmized probabilities of the outputs
(e.g. obtained with <a class="reference internal" href="#torch.nn.functional.log_softmax" title="torch.nn.functional.log_softmax"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.log_softmax()</span></code></a>).</p></li>
<li><p><strong>targets</strong> – <span class="math">\((N, S)\)</span> or <cite>(sum(target_lengths))</cite>.
Targets cannot be blank. In the second form, the targets are assumed to be concatenated.</p></li>
<li><p><strong>input_lengths</strong> – <span class="math">\((N)\)</span>.
Lengths of the inputs (must each be <span class="math">\(\leq T\)</span>)</p></li>
<li><p><strong>target_lengths</strong> – <span class="math">\((N)\)</span>.
Lengths of the targets</p></li>
<li><p><strong>blank</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Blank label. Default <span class="math">\(0\)</span>.</p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the output losses will be divided by the target lengths and
then the mean over the batch is taken, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be
summed. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
<li><p><strong>zero_infinity</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Whether to zero infinite losses and the associated gradients.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>
Infinite losses mainly occur when the inputs are too short
to be aligned to the targets.</p></li>
</ul>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">log_probs</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">50</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">20</span><span class="p">)</span><span class="o">.</span><span class="n">log_softmax</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">requires_grad_</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">targets</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">20</span><span class="p">,</span> <span class="p">(</span><span class="mi">16</span><span class="p">,</span> <span class="mi">30</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input_lengths</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">((</span><span class="mi">16</span><span class="p">,),</span> <span class="mi">50</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target_lengths</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span><span class="mi">30</span><span class="p">,(</span><span class="mi">16</span><span class="p">,),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">long</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">ctc_loss</span><span class="p">(</span><span class="n">log_probs</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">input_lengths</span><span class="p">,</span> <span class="n">target_lengths</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="hinge-embedding-loss">
<h3><span class="hidden-section">hinge_embedding_loss</span><a class="headerlink" href="#hinge-embedding-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.hinge_embedding_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">hinge_embedding_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>margin=1.0</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#hinge_embedding_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.hinge_embedding_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.HingeEmbeddingLoss" title="torch.nn.HingeEmbeddingLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">HingeEmbeddingLoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="kl-div">
<h3><span class="hidden-section">kl_div</span><a class="headerlink" href="#kl-div" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.kl_div">
<code class="descclassname">torch.nn.functional.</code><code class="descname">kl_div</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#kl_div"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.kl_div" title="Permalink to this definition">¶</a></dt>
<dd><p>The <a class="reference external" href="https://en.wikipedia.org/wiki/Kullback-Leibler_divergence">Kullback-Leibler divergence</a> Loss.</p>
<p>See <a class="reference internal" href="#torch.nn.KLDivLoss" title="torch.nn.KLDivLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">KLDivLoss</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – Tensor of arbitrary shape</p></li>
<li><p><strong>target</strong> – Tensor of the same shape as input</p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code>.
<code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied
<code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code>: the sum of the output will be divided by the batchsize
<code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the output will be divided by the number of elements in the output
Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated,
and in the meantime, specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>:attr:<code class="docutils literal notranslate"><span class="pre">reduction</span></code> = <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> doesn’t return the true kl divergence value, please use
:attr:<code class="docutils literal notranslate"><span class="pre">reduction</span></code> = <code class="docutils literal notranslate"><span class="pre">'batchmean'</span></code> which aligns with KL math definition.
In the next major release, <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> will be changed to be the same as ‘batchmean’.</p>
</div>
</dd></dl>

</div>
<div class="section" id="l1-loss">
<h3><span class="hidden-section">l1_loss</span><a class="headerlink" href="#l1-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.l1_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">l1_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#l1_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.l1_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that takes the mean element-wise absolute value difference.</p>
<p>See <a class="reference internal" href="#torch.nn.L1Loss" title="torch.nn.L1Loss"><code class="xref py py-class docutils literal notranslate"><span class="pre">L1Loss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="mse-loss">
<h3><span class="hidden-section">mse_loss</span><a class="headerlink" href="#mse-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.mse_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">mse_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#mse_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.mse_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>Measures the element-wise mean squared error.</p>
<p>See <a class="reference internal" href="#torch.nn.MSELoss" title="torch.nn.MSELoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">MSELoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="margin-ranking-loss">
<h3><span class="hidden-section">margin_ranking_loss</span><a class="headerlink" href="#margin-ranking-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.margin_ranking_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">margin_ranking_loss</code><span class="sig-paren">(</span><em>input1</em>, <em>input2</em>, <em>target</em>, <em>margin=0</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#margin_ranking_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.margin_ranking_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.MarginRankingLoss" title="torch.nn.MarginRankingLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">MarginRankingLoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="multilabel-margin-loss">
<h3><span class="hidden-section">multilabel_margin_loss</span><a class="headerlink" href="#multilabel-margin-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.multilabel_margin_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">multilabel_margin_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#multilabel_margin_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.multilabel_margin_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.MultiLabelMarginLoss" title="torch.nn.MultiLabelMarginLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">MultiLabelMarginLoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="multilabel-soft-margin-loss">
<h3><span class="hidden-section">multilabel_soft_margin_loss</span><a class="headerlink" href="#multilabel-soft-margin-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.multilabel_soft_margin_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">multilabel_soft_margin_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>weight=None</em>, <em>size_average=None</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#multilabel_soft_margin_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.multilabel_soft_margin_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.MultiLabelSoftMarginLoss" title="torch.nn.MultiLabelSoftMarginLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">MultiLabelSoftMarginLoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="multi-margin-loss">
<h3><span class="hidden-section">multi_margin_loss</span><a class="headerlink" href="#multi-margin-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.multi_margin_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">multi_margin_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>p=1</em>, <em>margin=1.0</em>, <em>weight=None</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#multi_margin_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.multi_margin_loss" title="Permalink to this definition">¶</a></dt>
<dd><dl class="simple">
<dt>multi_margin_loss(input, target, p=1, margin=1, weight=None, size_average=None,</dt><dd><p>reduce=None, reduction=’mean’) -&gt; Tensor</p>
</dd>
</dl>
<p>See <a class="reference internal" href="#torch.nn.MultiMarginLoss" title="torch.nn.MultiMarginLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">MultiMarginLoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="nll-loss">
<h3><span class="hidden-section">nll_loss</span><a class="headerlink" href="#nll-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.nll_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">nll_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>weight=None</em>, <em>size_average=None</em>, <em>ignore_index=-100</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#nll_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.nll_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>The negative log likelihood loss.</p>
<p>See <a class="reference internal" href="#torch.nn.NLLLoss" title="torch.nn.NLLLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">NLLLoss</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> – <span class="math">\((N, C)\)</span> where <cite>C = number of classes</cite> or <span class="math">\((N, C, H, W)\)</span>
in case of 2D Loss, or <span class="math">\((N, C, d_1, d_2, ..., d_K)\)</span> where <span class="math">\(K \geq 1\)</span>
in the case of K-dimensional loss.</p></li>
<li><p><strong>target</strong> – <span class="math">\((N)\)</span> where each value is <span class="math">\(0 \leq \text{targets}[i] \leq C-1\)</span>,
or <span class="math">\((N, d_1, d_2, ..., d_K)\)</span> where <span class="math">\(K \geq 1\)</span> for
K-dimensional loss.</p></li>
<li><p><strong>weight</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a><em>, </em><em>optional</em>) – a manual rescaling weight given to each
class. If given, has to be a Tensor of size <cite>C</cite></p></li>
<li><p><strong>size_average</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default,
the losses are averaged over each loss element in the batch. Note that for
some losses, there multiple elements per sample. If the field <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
is set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the losses are instead summed for each minibatch. Ignored
when reduce is <code class="docutils literal notranslate"><span class="pre">False</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>ignore_index</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><em>optional</em>) – Specifies a target value that is ignored
and does not contribute to the input gradient. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code> is
<code class="docutils literal notranslate"><span class="pre">True</span></code>, the loss is averaged over non-ignored targets. Default: -100</p></li>
<li><p><strong>reduce</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Deprecated (see <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>). By default, the
losses are averaged or summed over observations for each minibatch depending
on <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. When <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code>, returns a loss per
batch element instead and ignores <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">True</span></code></p></li>
<li><p><strong>reduction</strong> (<em>string</em><em>, </em><em>optional</em>) – Specifies the reduction to apply to the output:
<code class="docutils literal notranslate"><span class="pre">'none'</span></code> | <code class="docutils literal notranslate"><span class="pre">'mean'</span></code> | <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>. <code class="docutils literal notranslate"><span class="pre">'none'</span></code>: no reduction will be applied,
<code class="docutils literal notranslate"><span class="pre">'mean'</span></code>: the sum of the output will be divided by the number of
elements in the output, <code class="docutils literal notranslate"><span class="pre">'sum'</span></code>: the output will be summed. Note: <code class="xref py py-attr docutils literal notranslate"><span class="pre">size_average</span></code>
and <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduce</span></code> are in the process of being deprecated, and in the meantime,
specifying either of those two args will override <code class="xref py py-attr docutils literal notranslate"><span class="pre">reduction</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'mean'</span></code></p></li>
</ul>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># input is of size N x C = 3 x 5</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># each element in target has to have 0 &lt;= value &lt; C</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">nll_loss</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">log_softmax</span><span class="p">(</span><span class="nb">input</span><span class="p">),</span> <span class="n">target</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="smooth-l1-loss">
<h3><span class="hidden-section">smooth_l1_loss</span><a class="headerlink" href="#smooth-l1-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.smooth_l1_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">smooth_l1_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#smooth_l1_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.smooth_l1_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>Function that uses a squared term if the absolute
element-wise error falls below 1 and an L1 term otherwise.</p>
<p>See <a class="reference internal" href="#torch.nn.SmoothL1Loss" title="torch.nn.SmoothL1Loss"><code class="xref py py-class docutils literal notranslate"><span class="pre">SmoothL1Loss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="soft-margin-loss">
<h3><span class="hidden-section">soft_margin_loss</span><a class="headerlink" href="#soft-margin-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.soft_margin_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">soft_margin_loss</code><span class="sig-paren">(</span><em>input</em>, <em>target</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span> &#x2192; Tensor<a class="reference internal" href="_modules/torch/nn/functional.html#soft_margin_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.soft_margin_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.SoftMarginLoss" title="torch.nn.SoftMarginLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">SoftMarginLoss</span></code></a> for details.</p>
</dd></dl>

</div>
<div class="section" id="triplet-margin-loss">
<h3><span class="hidden-section">triplet_margin_loss</span><a class="headerlink" href="#triplet-margin-loss" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.triplet_margin_loss">
<code class="descclassname">torch.nn.functional.</code><code class="descname">triplet_margin_loss</code><span class="sig-paren">(</span><em>anchor</em>, <em>positive</em>, <em>negative</em>, <em>margin=1.0</em>, <em>p=2</em>, <em>eps=1e-06</em>, <em>swap=False</em>, <em>size_average=None</em>, <em>reduce=None</em>, <em>reduction='mean'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#triplet_margin_loss"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.triplet_margin_loss" title="Permalink to this definition">¶</a></dt>
<dd><p>See <a class="reference internal" href="#torch.nn.TripletMarginLoss" title="torch.nn.TripletMarginLoss"><code class="xref py py-class docutils literal notranslate"><span class="pre">TripletMarginLoss</span></code></a> for details</p>
</dd></dl>

</div>
</div>
<div class="section" id="vision-functions">
<h2>Vision functions<a class="headerlink" href="#vision-functions" title="Permalink to this headline">¶</a></h2>
<div class="section" id="pixel-shuffle">
<h3><span class="hidden-section">pixel_shuffle</span><a class="headerlink" href="#pixel-shuffle" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.pixel_shuffle">
<code class="descclassname">torch.nn.functional.</code><code class="descname">pixel_shuffle</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#torch.nn.functional.pixel_shuffle" title="Permalink to this definition">¶</a></dt>
<dd><p>Rearranges elements in a tensor of shape <span class="math">\((*, C \times r^2, H, W)\)</span> to a
tensor of shape <span class="math">\((*, C, H \times r, W \times r)\)</span>.</p>
<p>See <a class="reference internal" href="#torch.nn.PixelShuffle" title="torch.nn.PixelShuffle"><code class="xref py py-class docutils literal notranslate"><span class="pre">PixelShuffle</span></code></a> for details.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the input tensor</p></li>
<li><p><strong>upscale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – factor to increase spatial resolution by</p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="nb">input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">9</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">pixel_shuffle</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">output</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([1, 1, 12, 12])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="pad">
<h3><span class="hidden-section">pad</span><a class="headerlink" href="#pad" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.pad">
<code class="descclassname">torch.nn.functional.</code><code class="descname">pad</code><span class="sig-paren">(</span><em>input</em>, <em>pad</em>, <em>mode='constant'</em>, <em>value=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#pad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.pad" title="Permalink to this definition">¶</a></dt>
<dd><p>Pads tensor.</p>
<dl class="simple">
<dt>Padding size:</dt><dd><p>The padding size by which to pad some dimensions of <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code>
are described starting from the last dimension and moving forward.
<span class="math">\(\left\lfloor\frac{\text{len(pad)}}{2}\right\rfloor\)</span> dimensions
of <code class="docutils literal notranslate"><span class="pre">input</span></code> will be padded.
For example, to pad only the last dimension of the input tensor, then
<a class="reference internal" href="#torch.nn.functional.pad" title="torch.nn.functional.pad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">pad</span></code></a> has the form
<span class="math">\((\text{padding\_left}, \text{padding\_right})\)</span>;
to pad the last 2 dimensions of the input tensor, then use
<span class="math">\((\text{padding\_left}, \text{padding\_right},\)</span>
<span class="math">\(\text{padding\_top}, \text{padding\_bottom})\)</span>;
to pad the last 3 dimensions, use
<span class="math">\((\text{padding\_left}, \text{padding\_right},\)</span>
<span class="math">\(\text{padding\_top}, \text{padding\_bottom}\)</span>
<span class="math">\(\text{padding\_front}, \text{padding\_back})\)</span>.</p>
</dd>
<dt>Padding mode:</dt><dd><p>See <a class="reference internal" href="#torch.nn.ConstantPad2d" title="torch.nn.ConstantPad2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.ConstantPad2d</span></code></a>, <a class="reference internal" href="#torch.nn.ReflectionPad2d" title="torch.nn.ReflectionPad2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.ReflectionPad2d</span></code></a>, and
<a class="reference internal" href="#torch.nn.ReplicationPad2d" title="torch.nn.ReplicationPad2d"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.nn.ReplicationPad2d</span></code></a> for concrete examples on how each of the
padding modes works. Constant padding is implemented for arbitrary dimensions.
Replicate padding is implemented for padding the last 3 dimensions of 5D input
tensor, or the last 2 dimensions of 4D input tensor, or the last dimension of
3D input tensor. Reflect padding is only implemented for padding the last 2
dimensions of 4D input tensor, or the last dimension of 3D input tensor.</p>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – N-dimensional tensor</p></li>
<li><p><strong>pad</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.7)"><em>tuple</em></a>) – m-elements tuple, where
<span class="math">\(\frac{m}{2} \leq\)</span> input dimensions and <span class="math">\(m\)</span> is even.</p></li>
<li><p><strong>mode</strong> – <code class="docutils literal notranslate"><span class="pre">'constant'</span></code>, <code class="docutils literal notranslate"><span class="pre">'reflect'</span></code>, <code class="docutils literal notranslate"><span class="pre">'replicate'</span></code> or <code class="docutils literal notranslate"><span class="pre">'circular'</span></code>.
Default: <code class="docutils literal notranslate"><span class="pre">'constant'</span></code></p></li>
<li><p><strong>value</strong> – fill value for <code class="docutils literal notranslate"><span class="pre">'constant'</span></code> padding. Default: <code class="docutils literal notranslate"><span class="pre">0</span></code></p></li>
</ul>
</dd>
</dl>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">t4d</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">p1d</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> <span class="c1"># pad last dim by 1 on each side</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">t4d</span><span class="p">,</span> <span class="n">p1d</span><span class="p">,</span> <span class="s2">&quot;constant&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>  <span class="c1"># effectively zero padding</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">out</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([3, 3, 4, 4])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">p2d</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span> <span class="c1"># pad last dim by (1, 1) and 2nd to last by (2, 2)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">t4d</span><span class="p">,</span> <span class="n">p2d</span><span class="p">,</span> <span class="s2">&quot;constant&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">out</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([3, 3, 8, 4])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">t4d</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">p3d</span> <span class="o">=</span> <span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span> <span class="c1"># pad by (0, 1), (2, 1), and (3, 3)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">t4d</span><span class="p">,</span> <span class="n">p3d</span><span class="p">,</span> <span class="s2">&quot;constant&quot;</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">out</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
<span class="go">torch.Size([3, 9, 7, 3])</span>
</pre></div>
</div>
</dd></dl>

</div>
<div class="section" id="interpolate">
<h3><span class="hidden-section">interpolate</span><a class="headerlink" href="#interpolate" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.interpolate">
<code class="descclassname">torch.nn.functional.</code><code class="descname">interpolate</code><span class="sig-paren">(</span><em>input</em>, <em>size=None</em>, <em>scale_factor=None</em>, <em>mode='nearest'</em>, <em>align_corners=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#interpolate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.interpolate" title="Permalink to this definition">¶</a></dt>
<dd><p>Down/up samples the input to either the given <code class="xref py py-attr docutils literal notranslate"><span class="pre">size</span></code> or the given
<code class="xref py py-attr docutils literal notranslate"><span class="pre">scale_factor</span></code></p>
<p>The algorithm used for interpolation is determined by <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code>.</p>
<p>Currently temporal, spatial and volumetric sampling are supported, i.e.
expected inputs are 3-D, 4-D or 5-D in shape.</p>
<p>The input dimensions are interpreted in the form:
<cite>mini-batch x channels x [optional depth] x [optional height] x width</cite>.</p>
<p>The modes available for resizing are: <cite>nearest</cite>, <cite>linear</cite> (3D-only),
<cite>bilinear</cite>, <cite>bicubic</cite> (4D-only), <cite>trilinear</cite> (5D-only), <cite>area</cite></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the input tensor</p></li>
<li><p><strong>size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em>) – output spatial size.</p></li>
<li><p><strong>scale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em>) – multiplier for spatial size. Has to match input size if it is a tuple.</p></li>
<li><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – algorithm used for upsampling:
<code class="docutils literal notranslate"><span class="pre">'nearest'</span></code> | <code class="docutils literal notranslate"><span class="pre">'linear'</span></code> | <code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code> | <code class="docutils literal notranslate"><span class="pre">'bicubic'</span></code> |
<code class="docutils literal notranslate"><span class="pre">'trilinear'</span></code> | <code class="docutils literal notranslate"><span class="pre">'area'</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'nearest'</span></code></p></li>
<li><p><strong>align_corners</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Geometrically, we consider the pixels of the
input and output as squares rather than points.
If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, the input and output tensors are aligned by the
center points of their corner pixels. If set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the input and
output tensors are aligned by the corner points of their corner
pixels, and the interpolation uses edge value padding for out-of-boundary values.
This only has effect when <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code> is <code class="docutils literal notranslate"><span class="pre">'linear'</span></code>,
<code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code>, <code class="docutils literal notranslate"><span class="pre">'bicubic'</span></code>, or <code class="docutils literal notranslate"><span class="pre">'trilinear'</span></code>.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>With <code class="docutils literal notranslate"><span class="pre">align_corners</span> <span class="pre">=</span> <span class="pre">True</span></code>, the linearly interpolating modes
(<cite>linear</cite>, <cite>bilinear</cite>, and <cite>trilinear</cite>) don’t proportionally align the
output and input pixels, and thus the output values can depend on the
input size. This was the default behavior for these modes up to version
0.3.1. Since then, the default behavior is <code class="docutils literal notranslate"><span class="pre">align_corners</span> <span class="pre">=</span> <span class="pre">False</span></code>.
See <a class="reference internal" href="#torch.nn.Upsample" title="torch.nn.Upsample"><code class="xref py py-class docutils literal notranslate"><span class="pre">Upsample</span></code></a> for concrete examples on how this
affects the outputs.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
</dd></dl>

</div>
<div class="section" id="id53">
<h3><span class="hidden-section">upsample</span><a class="headerlink" href="#id53" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.upsample">
<code class="descclassname">torch.nn.functional.</code><code class="descname">upsample</code><span class="sig-paren">(</span><em>input</em>, <em>size=None</em>, <em>scale_factor=None</em>, <em>mode='nearest'</em>, <em>align_corners=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#upsample"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.upsample" title="Permalink to this definition">¶</a></dt>
<dd><p>Upsamples the input to either the given <code class="xref py py-attr docutils literal notranslate"><span class="pre">size</span></code> or the given
<code class="xref py py-attr docutils literal notranslate"><span class="pre">scale_factor</span></code></p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This function is deprecated in favor of <a class="reference internal" href="#torch.nn.functional.interpolate" title="torch.nn.functional.interpolate"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.interpolate()</span></code></a>.
This is equivalent with <code class="docutils literal notranslate"><span class="pre">nn.functional.interpolate(...)</span></code>.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<p>The algorithm used for upsampling is determined by <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code>.</p>
<p>Currently temporal, spatial and volumetric upsampling are supported, i.e.
expected inputs are 3-D, 4-D or 5-D in shape.</p>
<p>The input dimensions are interpreted in the form:
<cite>mini-batch x channels x [optional depth] x [optional height] x width</cite>.</p>
<p>The modes available for upsampling are: <cite>nearest</cite>, <cite>linear</cite> (3D-only),
<cite>bilinear</cite>, <cite>bicubic</cite> (4D-only), <cite>trilinear</cite> (5D-only)</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – the input tensor</p></li>
<li><p><strong>size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em>) – output spatial size.</p></li>
<li><p><strong>scale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em>) – multiplier for spatial size. Has to be an integer.</p></li>
<li><p><strong>mode</strong> (<em>string</em>) – algorithm used for upsampling:
<code class="docutils literal notranslate"><span class="pre">'nearest'</span></code> | <code class="docutils literal notranslate"><span class="pre">'linear'</span></code> | <code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code> | <code class="docutils literal notranslate"><span class="pre">'bicubic'</span></code> |
<code class="docutils literal notranslate"><span class="pre">'trilinear'</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'nearest'</span></code></p></li>
<li><p><strong>align_corners</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – Geometrically, we consider the pixels of the
input and output as squares rather than points.
If set to <code class="docutils literal notranslate"><span class="pre">True</span></code>, the input and output tensors are aligned by the
center points of their corner pixels. If set to <code class="docutils literal notranslate"><span class="pre">False</span></code>, the input and
output tensors are aligned by the corner points of their corner
pixels, and the interpolation uses edge value padding for out-of-boundary values.
This only has effect when <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code> is <code class="docutils literal notranslate"><span class="pre">'linear'</span></code>,
<code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code>, <code class="docutils literal notranslate"><span class="pre">'bicubic'</span></code> or <code class="docutils literal notranslate"><span class="pre">'trilinear'</span></code>.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>With <code class="docutils literal notranslate"><span class="pre">align_corners</span> <span class="pre">=</span> <span class="pre">True</span></code>, the linearly interpolating modes
(<cite>linear</cite>, <cite>bilinear</cite>, and <cite>trilinear</cite>) don’t proportionally align the
output and input pixels, and thus the output values can depend on the
input size. This was the default behavior for these modes up to version
0.3.1. Since then, the default behavior is <code class="docutils literal notranslate"><span class="pre">align_corners</span> <span class="pre">=</span> <span class="pre">False</span></code>.
See <a class="reference internal" href="#torch.nn.Upsample" title="torch.nn.Upsample"><code class="xref py py-class docutils literal notranslate"><span class="pre">Upsample</span></code></a> for concrete examples on how this
affects the outputs.</p>
</div>
</dd></dl>

</div>
<div class="section" id="upsample-nearest">
<h3><span class="hidden-section">upsample_nearest</span><a class="headerlink" href="#upsample-nearest" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.upsample_nearest">
<code class="descclassname">torch.nn.functional.</code><code class="descname">upsample_nearest</code><span class="sig-paren">(</span><em>input</em>, <em>size=None</em>, <em>scale_factor=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#upsample_nearest"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.upsample_nearest" title="Permalink to this definition">¶</a></dt>
<dd><p>Upsamples the input, using nearest neighbours’ pixel values.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This function is deprecated in favor of <a class="reference internal" href="#torch.nn.functional.interpolate" title="torch.nn.functional.interpolate"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.interpolate()</span></code></a>.
This is equivalent with <code class="docutils literal notranslate"><span class="pre">nn.functional.interpolate(...,</span> <span class="pre">mode='nearest')</span></code>.</p>
</div>
<p>Currently spatial and volumetric upsampling are supported (i.e. expected
inputs are 4 or 5 dimensional).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input</p></li>
<li><p><strong>size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>] or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em>) – output spatia
size.</p></li>
<li><p><strong>scale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – multiplier for spatial size. Has to be an integer.</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
</dd></dl>

</div>
<div class="section" id="upsample-bilinear">
<h3><span class="hidden-section">upsample_bilinear</span><a class="headerlink" href="#upsample-bilinear" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.upsample_bilinear">
<code class="descclassname">torch.nn.functional.</code><code class="descname">upsample_bilinear</code><span class="sig-paren">(</span><em>input</em>, <em>size=None</em>, <em>scale_factor=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#upsample_bilinear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.upsample_bilinear" title="Permalink to this definition">¶</a></dt>
<dd><p>Upsamples the input, using bilinear upsampling.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This function is deprecated in favor of <a class="reference internal" href="#torch.nn.functional.interpolate" title="torch.nn.functional.interpolate"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.nn.functional.interpolate()</span></code></a>.
This is equivalent with
<code class="docutils literal notranslate"><span class="pre">nn.functional.interpolate(...,</span> <span class="pre">mode='bilinear',</span> <span class="pre">align_corners=True)</span></code>.</p>
</div>
<p>Expected inputs are spatial (4 dimensional). Use <cite>upsample_trilinear</cite> fo
volumetric (5 dimensional) inputs.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input</p></li>
<li><p><strong>size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em>) – output spatial size.</p></li>
<li><p><strong>scale_factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em> or </em><em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a><em>]</em>) – multiplier for spatial size</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
</dd></dl>

</div>
<div class="section" id="grid-sample">
<h3><span class="hidden-section">grid_sample</span><a class="headerlink" href="#grid-sample" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.grid_sample">
<code class="descclassname">torch.nn.functional.</code><code class="descname">grid_sample</code><span class="sig-paren">(</span><em>input</em>, <em>grid</em>, <em>mode='bilinear'</em>, <em>padding_mode='zeros'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#grid_sample"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.grid_sample" title="Permalink to this definition">¶</a></dt>
<dd><p>Given an <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> and a flow-field <code class="xref py py-attr docutils literal notranslate"><span class="pre">grid</span></code>, computes the
<code class="docutils literal notranslate"><span class="pre">output</span></code> using <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> values and pixel locations from <code class="xref py py-attr docutils literal notranslate"><span class="pre">grid</span></code>.</p>
<p>Currently, only spatial (4-D) and volumetric (5-D) <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> are
supported.</p>
<p>In the spatial (4-D) case, for <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> with shape
<span class="math">\((N, C, H_\text{in}, W_\text{in})\)</span> and <code class="xref py py-attr docutils literal notranslate"><span class="pre">grid</span></code> with shape
<span class="math">\((N, H_\text{out}, W_\text{out}, 2)\)</span>, the output will have shape
<span class="math">\((N, C, H_\text{out}, W_\text{out})\)</span>.</p>
<p>For each output location <code class="docutils literal notranslate"><span class="pre">output[n,</span> <span class="pre">:,</span> <span class="pre">h,</span> <span class="pre">w]</span></code>, the size-2 vector
<code class="docutils literal notranslate"><span class="pre">grid[n,</span> <span class="pre">h,</span> <span class="pre">w]</span></code> specifies <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> pixel locations <code class="docutils literal notranslate"><span class="pre">x</span></code> and <code class="docutils literal notranslate"><span class="pre">y</span></code>,
which are used to interpolate the output value <code class="docutils literal notranslate"><span class="pre">output[n,</span> <span class="pre">:,</span> <span class="pre">h,</span> <span class="pre">w]</span></code>.
In the case of 5D inputs, <code class="docutils literal notranslate"><span class="pre">grid[n,</span> <span class="pre">d,</span> <span class="pre">h,</span> <span class="pre">w]</span></code> specifies the
<code class="docutils literal notranslate"><span class="pre">x</span></code>, <code class="docutils literal notranslate"><span class="pre">y</span></code>, <code class="docutils literal notranslate"><span class="pre">z</span></code> pixel locations for interpolating
<code class="docutils literal notranslate"><span class="pre">output[n,</span> <span class="pre">:,</span> <span class="pre">d,</span> <span class="pre">h,</span> <span class="pre">w]</span></code>. <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code> argument specifies <code class="docutils literal notranslate"><span class="pre">nearest</span></code> or
<code class="docutils literal notranslate"><span class="pre">bilinear</span></code> interpolation method to sample the input pixels.</p>
<p><code class="xref py py-attr docutils literal notranslate"><span class="pre">grid</span></code> specifies the sampling pixel locations normalized by the
<code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code> spatial dimensions. Therefore, it should have most values in
the range of <code class="docutils literal notranslate"><span class="pre">[-1,</span> <span class="pre">1]</span></code>. For example, values <code class="docutils literal notranslate"><span class="pre">x</span> <span class="pre">=</span> <span class="pre">-1,</span> <span class="pre">y</span> <span class="pre">=</span> <span class="pre">-1</span></code> is the
left-top pixel of <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code>, and values  <code class="docutils literal notranslate"><span class="pre">x</span> <span class="pre">=</span> <span class="pre">1,</span> <span class="pre">y</span> <span class="pre">=</span> <span class="pre">1</span></code> is the
right-bottom pixel of <code class="xref py py-attr docutils literal notranslate"><span class="pre">input</span></code>.</p>
<p>If <code class="xref py py-attr docutils literal notranslate"><span class="pre">grid</span></code> has values outside the range of <code class="docutils literal notranslate"><span class="pre">[-1,</span> <span class="pre">1]</span></code>, the corresponding
outputs are handled as defined by <code class="xref py py-attr docutils literal notranslate"><span class="pre">padding_mode</span></code>. Options are</p>
<blockquote>
<div><ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">padding_mode=&quot;zeros&quot;</span></code>: use <code class="docutils literal notranslate"><span class="pre">0</span></code> for out-of-bound grid locations,</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">padding_mode=&quot;border&quot;</span></code>: use border values for out-of-bound grid locations,</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">padding_mode=&quot;reflection&quot;</span></code>: use values at locations reflected by
the border for out-of-bound grid locations. For location far away
from the border, it will keep being reflected until becoming in bound,
e.g., (normalized) pixel location <code class="docutils literal notranslate"><span class="pre">x</span> <span class="pre">=</span> <span class="pre">-3.5</span></code> reflects by border <code class="docutils literal notranslate"><span class="pre">-1</span></code>
and becomes <code class="docutils literal notranslate"><span class="pre">x'</span> <span class="pre">=</span> <span class="pre">1.5</span></code>, then reflects by border <code class="docutils literal notranslate"><span class="pre">1</span></code> and becomes
<code class="docutils literal notranslate"><span class="pre">x''</span> <span class="pre">=</span> <span class="pre">-0.5</span></code>.</p></li>
</ul>
</div></blockquote>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This function is often used in building <a class="reference external" href="https://arxiv.org/abs/1506.02025">Spatial Transformer Networks</a> .</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>When using the CUDA backend, this operation may induce nondeterministic
behaviour in be backward that is not easily switched off.
Please see the notes on <a class="reference internal" href="notes/randomness.html"><span class="doc">Reproducibility</span></a> for background.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input of shape <span class="math">\((N, C, H_\text{in}, W_\text{in})\)</span> (4-D case)
or <span class="math">\((N, C, D_\text{in}, H_\text{in}, W_\text{in})\)</span> (5-D case)</p></li>
<li><p><strong>grid</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – flow-field of shape <span class="math">\((N, H_\text{out}, W_\text{out}, 2)\)</span> (4-D case)
or <span class="math">\((N, D_\text{out}, H_\text{out}, W_\text{out}, 3)\)</span> (5-D case)</p></li>
<li><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – interpolation mode to calculate output values
<code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code> | <code class="docutils literal notranslate"><span class="pre">'nearest'</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'bilinear'</span></code></p></li>
<li><p><strong>padding_mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – padding mode for outside grid values
<code class="docutils literal notranslate"><span class="pre">'zeros'</span></code> | <code class="docutils literal notranslate"><span class="pre">'border'</span></code> | <code class="docutils literal notranslate"><span class="pre">'reflection'</span></code>. Default: <code class="docutils literal notranslate"><span class="pre">'zeros'</span></code></p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>output Tensor</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>output (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor">Tensor</a>)</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="affine-grid">
<h3><span class="hidden-section">affine_grid</span><a class="headerlink" href="#affine-grid" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.functional.affine_grid">
<code class="descclassname">torch.nn.functional.</code><code class="descname">affine_grid</code><span class="sig-paren">(</span><em>theta</em>, <em>size</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/functional.html#affine_grid"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.functional.affine_grid" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates a 2d flow field, given a batch of affine matrices <code class="xref py py-attr docutils literal notranslate"><span class="pre">theta</span></code>.
Generally used in conjunction with <a class="reference internal" href="#torch.nn.functional.grid_sample" title="torch.nn.functional.grid_sample"><code class="xref py py-func docutils literal notranslate"><span class="pre">grid_sample()</span></code></a> to
implement Spatial Transformer Networks.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>theta</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – input batch of affine matrices (<span class="math">\(N \times 2 \times 3\)</span>)</p></li>
<li><p><strong>size</strong> (<em>torch.Size</em>) – the target output image size (<span class="math">\(N \times C \times H \times W\)</span>).
Example: torch.Size((32, 3, 24, 24))</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>output Tensor of size (<span class="math">\(N \times H \times W \times 2\)</span>)</p>
</dd>
<dt class="field-odd">Return type</dt>
<dd class="field-odd"><p>output (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor">Tensor</a>)</p>
</dd>
</dl>
</dd></dl>

</div>
</div>
<div class="section" id="dataparallel-functions-multi-gpu-distributed">
<h2>DataParallel functions (multi-GPU, distributed)<a class="headerlink" href="#dataparallel-functions-multi-gpu-distributed" title="Permalink to this headline">¶</a></h2>
<div class="section" id="data-parallel">
<h3><span class="hidden-section">data_parallel</span><a class="headerlink" href="#data-parallel" title="Permalink to this headline">¶</a></h3>
<dl class="function">
<dt id="torch.nn.parallel.data_parallel">
<code class="descclassname">torch.nn.parallel.</code><code class="descname">data_parallel</code><span class="sig-paren">(</span><em>module</em>, <em>inputs</em>, <em>device_ids=None</em>, <em>output_device=None</em>, <em>dim=0</em>, <em>module_kwargs=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/parallel/data_parallel.html#data_parallel"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.parallel.data_parallel" title="Permalink to this definition">¶</a></dt>
<dd><p>Evaluates module(input) in parallel across the GPUs given in device_ids.</p>
<p>This is the functional version of the DataParallel module.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>module</strong> (<a class="reference internal" href="#torch.nn.Module" title="torch.nn.Module"><em>Module</em></a>) – the module to evaluate in parallel</p></li>
<li><p><strong>inputs</strong> (<a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><em>Tensor</em></a>) – inputs to the module</p></li>
<li><p><strong>device_ids</strong> (<em>list of python:int</em><em> or </em><a class="reference internal" href="tensor_attributes.html#torch.torch.device" title="torch.torch.device"><em>torch.device</em></a>) – GPU ids on which to replicate module</p></li>
<li><p><strong>output_device</strong> (<em>list of python:int</em><em> or </em><a class="reference internal" href="tensor_attributes.html#torch.torch.device" title="torch.torch.device"><em>torch.device</em></a>) – GPU location of the output  Use -1 to indicate the CPU.
(default: device_ids[0])</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>a Tensor containing the result of module(input) located on
output_device</p>
</dd>
</dl>
</dd></dl>

</div>
</div>
</div>
<div class="section" id="torch-nn-init">
<h1>torch.nn.init<a class="headerlink" href="#torch-nn-init" title="Permalink to this headline">¶</a></h1>
<dl class="function">
<dt id="torch.nn.init.calculate_gain">
<code class="descclassname">torch.nn.init.</code><code class="descname">calculate_gain</code><span class="sig-paren">(</span><em>nonlinearity</em>, <em>param=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#calculate_gain"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.calculate_gain" title="Permalink to this definition">¶</a></dt>
<dd><p>Return the recommended gain value for the given nonlinearity function.
The values are as follows:</p>
<table class="docutils align-center">
<colgroup>
<col style="width: 24%" />
<col style="width: 76%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>nonlinearity</p></th>
<th class="head"><p>gain</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>Linear / Identity</p></td>
<td><p><span class="math">\(1\)</span></p></td>
</tr>
<tr class="row-odd"><td><p>Conv{1,2,3}D</p></td>
<td><p><span class="math">\(1\)</span></p></td>
</tr>
<tr class="row-even"><td><p>Sigmoid</p></td>
<td><p><span class="math">\(1\)</span></p></td>
</tr>
<tr class="row-odd"><td><p>Tanh</p></td>
<td><p><span class="math">\(\frac{5}{3}\)</span></p></td>
</tr>
<tr class="row-even"><td><p>ReLU</p></td>
<td><p><span class="math">\(\sqrt{2}\)</span></p></td>
</tr>
<tr class="row-odd"><td><p>Leaky Relu</p></td>
<td><p><span class="math">\(\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}\)</span></p></td>
</tr>
</tbody>
</table>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>nonlinearity</strong> – the non-linear function (<cite>nn.functional</cite> name)</p></li>
<li><p><strong>param</strong> – optional parameter for the non-linear function</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">gain</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">calculate_gain</span><span class="p">(</span><span class="s1">&#39;leaky_relu&#39;</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.uniform_">
<code class="descclassname">torch.nn.init.</code><code class="descname">uniform_</code><span class="sig-paren">(</span><em>tensor</em>, <em>a=0.0</em>, <em>b=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#uniform_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.uniform_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input Tensor with values drawn from the uniform
distribution <span class="math">\(\mathcal{U}(a, b)\)</span>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>a</strong> – the lower bound of the uniform distribution</p></li>
<li><p><strong>b</strong> – the upper bound of the uniform distribution</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">uniform_</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.normal_">
<code class="descclassname">torch.nn.init.</code><code class="descname">normal_</code><span class="sig-paren">(</span><em>tensor</em>, <em>mean=0.0</em>, <em>std=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#normal_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.normal_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input Tensor with values drawn from the normal
distribution <span class="math">\(\mathcal{N}(\text{mean}, \text{std})\)</span>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>mean</strong> – the mean of the normal distribution</p></li>
<li><p><strong>std</strong> – the standard deviation of the normal distribution</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">normal_</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.constant_">
<code class="descclassname">torch.nn.init.</code><code class="descname">constant_</code><span class="sig-paren">(</span><em>tensor</em>, <em>val</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#constant_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.constant_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input Tensor with the value <span class="math">\(\text{val}\)</span>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>val</strong> – the value to fill the tensor with</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">constant_</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="mf">0.3</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.eye_">
<code class="descclassname">torch.nn.init.</code><code class="descname">eye_</code><span class="sig-paren">(</span><em>tensor</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#eye_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.eye_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the 2-dimensional input <cite>Tensor</cite> with the identity
matrix. Preserves the identity of the inputs in <cite>Linear</cite> layers, where as
many inputs are preserved as possible.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>tensor</strong> – a 2-dimensional <cite>torch.Tensor</cite></p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">eye_</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.dirac_">
<code class="descclassname">torch.nn.init.</code><code class="descname">dirac_</code><span class="sig-paren">(</span><em>tensor</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#dirac_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.dirac_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the {3, 4, 5}-dimensional input <cite>Tensor</cite> with the Dirac
delta function. Preserves the identity of the inputs in <cite>Convolutional</cite>
layers, where as many input channels are preserved as possible.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>tensor</strong> – a {3, 4, 5}-dimensional <cite>torch.Tensor</cite></p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">dirac_</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.xavier_uniform_">
<code class="descclassname">torch.nn.init.</code><code class="descname">xavier_uniform_</code><span class="sig-paren">(</span><em>tensor</em>, <em>gain=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#xavier_uniform_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.xavier_uniform_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input <cite>Tensor</cite> with values according to the method
described in <cite>Understanding the difficulty of training deep feedforward
neural networks</cite> - Glorot, X. &amp; Bengio, Y. (2010), using a uniform
distribution. The resulting tensor will have values sampled from
<span class="math">\(\mathcal{U}(-a, a)\)</span> where</p>
<div class="math">
\[a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}}

\]</div>
<p>Also known as Glorot initialization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>gain</strong> – an optional scaling factor</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">xavier_uniform_</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="n">gain</span><span class="o">=</span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">calculate_gain</span><span class="p">(</span><span class="s1">&#39;relu&#39;</span><span class="p">))</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.xavier_normal_">
<code class="descclassname">torch.nn.init.</code><code class="descname">xavier_normal_</code><span class="sig-paren">(</span><em>tensor</em>, <em>gain=1.0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#xavier_normal_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.xavier_normal_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input <cite>Tensor</cite> with values according to the method
described in <cite>Understanding the difficulty of training deep feedforward
neural networks</cite> - Glorot, X. &amp; Bengio, Y. (2010), using a normal
distribution. The resulting tensor will have values sampled from
<span class="math">\(\mathcal{N}(0, \text{std})\)</span> where</p>
<div class="math">
\[\text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}}

\]</div>
<p>Also known as Glorot initialization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>gain</strong> – an optional scaling factor</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">xavier_normal_</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.kaiming_uniform_">
<code class="descclassname">torch.nn.init.</code><code class="descname">kaiming_uniform_</code><span class="sig-paren">(</span><em>tensor</em>, <em>a=0</em>, <em>mode='fan_in'</em>, <em>nonlinearity='leaky_relu'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#kaiming_uniform_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.kaiming_uniform_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input <cite>Tensor</cite> with values according to the method
described in <cite>Delving deep into rectifiers: Surpassing human-level
performance on ImageNet classification</cite> - He, K. et al. (2015), using a
uniform distribution. The resulting tensor will have values sampled from
<span class="math">\(\mathcal{U}(-\text{bound}, \text{bound})\)</span> where</p>
<div class="math">
\[\text{bound} = \sqrt{\frac{6}{(1 + a^2) \times \text{fan\_in}}}

\]</div>
<p>Also known as He initialization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>a</strong> – the negative slope of the rectifier used after this layer (0 for ReLU
by default)</p></li>
<li><p><strong>mode</strong> – either <code class="docutils literal notranslate"><span class="pre">'fan_in'</span></code> (default) or <code class="docutils literal notranslate"><span class="pre">'fan_out'</span></code>. Choosing <code class="docutils literal notranslate"><span class="pre">'fan_in'</span></code>
preserves the magnitude of the variance of the weights in the
forward pass. Choosing <code class="docutils literal notranslate"><span class="pre">'fan_out'</span></code> preserves the magnitudes in the
backwards pass.</p></li>
<li><p><strong>nonlinearity</strong> – the non-linear function (<cite>nn.functional</cite> name),
recommended to use only with <code class="docutils literal notranslate"><span class="pre">'relu'</span></code> or <code class="docutils literal notranslate"><span class="pre">'leaky_relu'</span></code> (default).</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">kaiming_uniform_</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;fan_in&#39;</span><span class="p">,</span> <span class="n">nonlinearity</span><span class="o">=</span><span class="s1">&#39;relu&#39;</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.kaiming_normal_">
<code class="descclassname">torch.nn.init.</code><code class="descname">kaiming_normal_</code><span class="sig-paren">(</span><em>tensor</em>, <em>a=0</em>, <em>mode='fan_in'</em>, <em>nonlinearity='leaky_relu'</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#kaiming_normal_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.kaiming_normal_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input <cite>Tensor</cite> with values according to the method
described in <cite>Delving deep into rectifiers: Surpassing human-level
performance on ImageNet classification</cite> - He, K. et al. (2015), using a
normal distribution. The resulting tensor will have values sampled from
<span class="math">\(\mathcal{N}(0, \text{std})\)</span> where</p>
<div class="math">
\[\text{std} = \sqrt{\frac{2}{(1 + a^2) \times \text{fan\_in}}}

\]</div>
<p>Also known as He initialization.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>a</strong> – the negative slope of the rectifier used after this layer (0 for ReLU
by default)</p></li>
<li><p><strong>mode</strong> – either <code class="docutils literal notranslate"><span class="pre">'fan_in'</span></code> (default) or <code class="docutils literal notranslate"><span class="pre">'fan_out'</span></code>. Choosing <code class="docutils literal notranslate"><span class="pre">'fan_in'</span></code>
preserves the magnitude of the variance of the weights in the
forward pass. Choosing <code class="docutils literal notranslate"><span class="pre">'fan_out'</span></code> preserves the magnitudes in the
backwards pass.</p></li>
<li><p><strong>nonlinearity</strong> – the non-linear function (<cite>nn.functional</cite> name),
recommended to use only with <code class="docutils literal notranslate"><span class="pre">'relu'</span></code> or <code class="docutils literal notranslate"><span class="pre">'leaky_relu'</span></code> (default).</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">kaiming_normal_</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;fan_out&#39;</span><span class="p">,</span> <span class="n">nonlinearity</span><span class="o">=</span><span class="s1">&#39;relu&#39;</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.orthogonal_">
<code class="descclassname">torch.nn.init.</code><code class="descname">orthogonal_</code><span class="sig-paren">(</span><em>tensor</em>, <em>gain=1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#orthogonal_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.orthogonal_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the input <cite>Tensor</cite> with a (semi) orthogonal matrix, as
described in <cite>Exact solutions to the nonlinear dynamics of learning in deep
linear neural networks</cite> - Saxe, A. et al. (2013). The input tensor must have
at least 2 dimensions, and for tensors with more than 2 dimensions the
trailing dimensions are flattened.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite>, where <span class="math">\(n \geq 2\)</span></p></li>
<li><p><strong>gain</strong> – optional scaling factor</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">orthogonal_</span><span class="p">(</span><span class="n">w</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="function">
<dt id="torch.nn.init.sparse_">
<code class="descclassname">torch.nn.init.</code><code class="descname">sparse_</code><span class="sig-paren">(</span><em>tensor</em>, <em>sparsity</em>, <em>std=0.01</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/nn/init.html#sparse_"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.nn.init.sparse_" title="Permalink to this definition">¶</a></dt>
<dd><p>Fills the 2D input <cite>Tensor</cite> as a sparse matrix, where the
non-zero elements will be drawn from the normal distribution
<span class="math">\(\mathcal{N}(0, 0.01)\)</span>, as described in <cite>Deep learning via
Hessian-free optimization</cite> - Martens, J. (2010).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – an n-dimensional <cite>torch.Tensor</cite></p></li>
<li><p><strong>sparsity</strong> – The fraction of elements in each column to be set to zero</p></li>
<li><p><strong>std</strong> – the standard deviation of the normal distribution used to generate
the non-zero values</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">w</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">nn</span><span class="o">.</span><span class="n">init</span><span class="o">.</span><span class="n">sparse_</span><span class="p">(</span><span class="n">w</span><span class="p">,</span> <span class="n">sparsity</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</div>


             </article>
             
            </div>
            <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="optim.html" class="btn btn-neutral float-right" title="torch.optim" accesskey="n" rel="next">Next <img src="_static/images/chevron-right-orange.svg" class="next-page"></a>
      
      
        <a href="storage.html" class="btn btn-neutral" title="torch.Storage" accesskey="p" rel="prev"><img src="_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
      
    </div>
  

    <hr>

  
  <div role="contentinfo">
    <p>
        &copy; Copyright 2018, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">torch.nn</a><ul>
<li><a class="reference internal" href="#parameters">Parameters</a></li>
<li><a class="reference internal" href="#containers">Containers</a><ul>
<li><a class="reference internal" href="#module"><span class="hidden-section">Module</span></a></li>
<li><a class="reference internal" href="#sequential"><span class="hidden-section">Sequential</span></a></li>
<li><a class="reference internal" href="#modulelist"><span class="hidden-section">ModuleList</span></a></li>
<li><a class="reference internal" href="#moduledict"><span class="hidden-section">ModuleDict</span></a></li>
<li><a class="reference internal" href="#parameterlist"><span class="hidden-section">ParameterList</span></a></li>
<li><a class="reference internal" href="#parameterdict"><span class="hidden-section">ParameterDict</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#convolution-layers">Convolution layers</a><ul>
<li><a class="reference internal" href="#conv1d"><span class="hidden-section">Conv1d</span></a></li>
<li><a class="reference internal" href="#conv2d"><span class="hidden-section">Conv2d</span></a></li>
<li><a class="reference internal" href="#conv3d"><span class="hidden-section">Conv3d</span></a></li>
<li><a class="reference internal" href="#convtranspose1d"><span class="hidden-section">ConvTranspose1d</span></a></li>
<li><a class="reference internal" href="#convtranspose2d"><span class="hidden-section">ConvTranspose2d</span></a></li>
<li><a class="reference internal" href="#convtranspose3d"><span class="hidden-section">ConvTranspose3d</span></a></li>
<li><a class="reference internal" href="#unfold"><span class="hidden-section">Unfold</span></a></li>
<li><a class="reference internal" href="#fold"><span class="hidden-section">Fold</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#pooling-layers">Pooling layers</a><ul>
<li><a class="reference internal" href="#maxpool1d"><span class="hidden-section">MaxPool1d</span></a></li>
<li><a class="reference internal" href="#maxpool2d"><span class="hidden-section">MaxPool2d</span></a></li>
<li><a class="reference internal" href="#maxpool3d"><span class="hidden-section">MaxPool3d</span></a></li>
<li><a class="reference internal" href="#maxunpool1d"><span class="hidden-section">MaxUnpool1d</span></a></li>
<li><a class="reference internal" href="#maxunpool2d"><span class="hidden-section">MaxUnpool2d</span></a></li>
<li><a class="reference internal" href="#maxunpool3d"><span class="hidden-section">MaxUnpool3d</span></a></li>
<li><a class="reference internal" href="#avgpool1d"><span class="hidden-section">AvgPool1d</span></a></li>
<li><a class="reference internal" href="#avgpool2d"><span class="hidden-section">AvgPool2d</span></a></li>
<li><a class="reference internal" href="#avgpool3d"><span class="hidden-section">AvgPool3d</span></a></li>
<li><a class="reference internal" href="#fractionalmaxpool2d"><span class="hidden-section">FractionalMaxPool2d</span></a></li>
<li><a class="reference internal" href="#lppool1d"><span class="hidden-section">LPPool1d</span></a></li>
<li><a class="reference internal" href="#lppool2d"><span class="hidden-section">LPPool2d</span></a></li>
<li><a class="reference internal" href="#adaptivemaxpool1d"><span class="hidden-section">AdaptiveMaxPool1d</span></a></li>
<li><a class="reference internal" href="#adaptivemaxpool2d"><span class="hidden-section">AdaptiveMaxPool2d</span></a></li>
<li><a class="reference internal" href="#adaptivemaxpool3d"><span class="hidden-section">AdaptiveMaxPool3d</span></a></li>
<li><a class="reference internal" href="#adaptiveavgpool1d"><span class="hidden-section">AdaptiveAvgPool1d</span></a></li>
<li><a class="reference internal" href="#adaptiveavgpool2d"><span class="hidden-section">AdaptiveAvgPool2d</span></a></li>
<li><a class="reference internal" href="#adaptiveavgpool3d"><span class="hidden-section">AdaptiveAvgPool3d</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#padding-layers">Padding layers</a><ul>
<li><a class="reference internal" href="#reflectionpad1d"><span class="hidden-section">ReflectionPad1d</span></a></li>
<li><a class="reference internal" href="#reflectionpad2d"><span class="hidden-section">ReflectionPad2d</span></a></li>
<li><a class="reference internal" href="#replicationpad1d"><span class="hidden-section">ReplicationPad1d</span></a></li>
<li><a class="reference internal" href="#replicationpad2d"><span class="hidden-section">ReplicationPad2d</span></a></li>
<li><a class="reference internal" href="#replicationpad3d"><span class="hidden-section">ReplicationPad3d</span></a></li>
<li><a class="reference internal" href="#zeropad2d"><span class="hidden-section">ZeroPad2d</span></a></li>
<li><a class="reference internal" href="#constantpad1d"><span class="hidden-section">ConstantPad1d</span></a></li>
<li><a class="reference internal" href="#constantpad2d"><span class="hidden-section">ConstantPad2d</span></a></li>
<li><a class="reference internal" href="#constantpad3d"><span class="hidden-section">ConstantPad3d</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#non-linear-activations-weighted-sum-nonlinearity">Non-linear activations (weighted sum, nonlinearity)</a><ul>
<li><a class="reference internal" href="#elu"><span class="hidden-section">ELU</span></a></li>
<li><a class="reference internal" href="#hardshrink"><span class="hidden-section">Hardshrink</span></a></li>
<li><a class="reference internal" href="#hardtanh"><span class="hidden-section">Hardtanh</span></a></li>
<li><a class="reference internal" href="#leakyrelu"><span class="hidden-section">LeakyReLU</span></a></li>
<li><a class="reference internal" href="#logsigmoid"><span class="hidden-section">LogSigmoid</span></a></li>
<li><a class="reference internal" href="#multiheadattention"><span class="hidden-section">MultiheadAttention</span></a></li>
<li><a class="reference internal" href="#prelu"><span class="hidden-section">PReLU</span></a></li>
<li><a class="reference internal" href="#relu"><span class="hidden-section">ReLU</span></a></li>
<li><a class="reference internal" href="#relu6"><span class="hidden-section">ReLU6</span></a></li>
<li><a class="reference internal" href="#rrelu"><span class="hidden-section">RReLU</span></a></li>
<li><a class="reference internal" href="#selu"><span class="hidden-section">SELU</span></a></li>
<li><a class="reference internal" href="#celu"><span class="hidden-section">CELU</span></a></li>
<li><a class="reference internal" href="#sigmoid"><span class="hidden-section">Sigmoid</span></a></li>
<li><a class="reference internal" href="#softplus"><span class="hidden-section">Softplus</span></a></li>
<li><a class="reference internal" href="#softshrink"><span class="hidden-section">Softshrink</span></a></li>
<li><a class="reference internal" href="#softsign"><span class="hidden-section">Softsign</span></a></li>
<li><a class="reference internal" href="#tanh"><span class="hidden-section">Tanh</span></a></li>
<li><a class="reference internal" href="#tanhshrink"><span class="hidden-section">Tanhshrink</span></a></li>
<li><a class="reference internal" href="#threshold"><span class="hidden-section">Threshold</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#non-linear-activations-other">Non-linear activations (other)</a><ul>
<li><a class="reference internal" href="#softmin"><span class="hidden-section">Softmin</span></a></li>
<li><a class="reference internal" href="#softmax"><span class="hidden-section">Softmax</span></a></li>
<li><a class="reference internal" href="#softmax2d"><span class="hidden-section">Softmax2d</span></a></li>
<li><a class="reference internal" href="#logsoftmax"><span class="hidden-section">LogSoftmax</span></a></li>
<li><a class="reference internal" href="#adaptivelogsoftmaxwithloss"><span class="hidden-section">AdaptiveLogSoftmaxWithLoss</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#normalization-layers">Normalization layers</a><ul>
<li><a class="reference internal" href="#batchnorm1d"><span class="hidden-section">BatchNorm1d</span></a></li>
<li><a class="reference internal" href="#batchnorm2d"><span class="hidden-section">BatchNorm2d</span></a></li>
<li><a class="reference internal" href="#batchnorm3d"><span class="hidden-section">BatchNorm3d</span></a></li>
<li><a class="reference internal" href="#groupnorm"><span class="hidden-section">GroupNorm</span></a></li>
<li><a class="reference internal" href="#syncbatchnorm"><span class="hidden-section">SyncBatchNorm</span></a></li>
<li><a class="reference internal" href="#instancenorm1d"><span class="hidden-section">InstanceNorm1d</span></a></li>
<li><a class="reference internal" href="#instancenorm2d"><span class="hidden-section">InstanceNorm2d</span></a></li>
<li><a class="reference internal" href="#instancenorm3d"><span class="hidden-section">InstanceNorm3d</span></a></li>
<li><a class="reference internal" href="#layernorm"><span class="hidden-section">LayerNorm</span></a></li>
<li><a class="reference internal" href="#localresponsenorm"><span class="hidden-section">LocalResponseNorm</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#recurrent-layers">Recurrent layers</a><ul>
<li><a class="reference internal" href="#rnn"><span class="hidden-section">RNN</span></a></li>
<li><a class="reference internal" href="#lstm"><span class="hidden-section">LSTM</span></a></li>
<li><a class="reference internal" href="#gru"><span class="hidden-section">GRU</span></a></li>
<li><a class="reference internal" href="#rnncell"><span class="hidden-section">RNNCell</span></a></li>
<li><a class="reference internal" href="#lstmcell"><span class="hidden-section">LSTMCell</span></a></li>
<li><a class="reference internal" href="#grucell"><span class="hidden-section">GRUCell</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#linear-layers">Linear layers</a><ul>
<li><a class="reference internal" href="#identity"><span class="hidden-section">Identity</span></a></li>
<li><a class="reference internal" href="#linear"><span class="hidden-section">Linear</span></a></li>
<li><a class="reference internal" href="#bilinear"><span class="hidden-section">Bilinear</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#dropout-layers">Dropout layers</a><ul>
<li><a class="reference internal" href="#dropout"><span class="hidden-section">Dropout</span></a></li>
<li><a class="reference internal" href="#dropout2d"><span class="hidden-section">Dropout2d</span></a></li>
<li><a class="reference internal" href="#dropout3d"><span class="hidden-section">Dropout3d</span></a></li>
<li><a class="reference internal" href="#alphadropout"><span class="hidden-section">AlphaDropout</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#sparse-layers">Sparse layers</a><ul>
<li><a class="reference internal" href="#embedding"><span class="hidden-section">Embedding</span></a></li>
<li><a class="reference internal" href="#embeddingbag"><span class="hidden-section">EmbeddingBag</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#distance-functions">Distance functions</a><ul>
<li><a class="reference internal" href="#cosinesimilarity"><span class="hidden-section">CosineSimilarity</span></a></li>
<li><a class="reference internal" href="#pairwisedistance"><span class="hidden-section">PairwiseDistance</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#loss-functions">Loss functions</a><ul>
<li><a class="reference internal" href="#l1loss"><span class="hidden-section">L1Loss</span></a></li>
<li><a class="reference internal" href="#mseloss"><span class="hidden-section">MSELoss</span></a></li>
<li><a class="reference internal" href="#crossentropyloss"><span class="hidden-section">CrossEntropyLoss</span></a></li>
<li><a class="reference internal" href="#ctcloss"><span class="hidden-section">CTCLoss</span></a></li>
<li><a class="reference internal" href="#nllloss"><span class="hidden-section">NLLLoss</span></a></li>
<li><a class="reference internal" href="#poissonnllloss"><span class="hidden-section">PoissonNLLLoss</span></a></li>
<li><a class="reference internal" href="#kldivloss"><span class="hidden-section">KLDivLoss</span></a></li>
<li><a class="reference internal" href="#bceloss"><span class="hidden-section">BCELoss</span></a></li>
<li><a class="reference internal" href="#bcewithlogitsloss"><span class="hidden-section">BCEWithLogitsLoss</span></a></li>
<li><a class="reference internal" href="#marginrankingloss"><span class="hidden-section">MarginRankingLoss</span></a></li>
<li><a class="reference internal" href="#hingeembeddingloss"><span class="hidden-section">HingeEmbeddingLoss</span></a></li>
<li><a class="reference internal" href="#multilabelmarginloss"><span class="hidden-section">MultiLabelMarginLoss</span></a></li>
<li><a class="reference internal" href="#smoothl1loss"><span class="hidden-section">SmoothL1Loss</span></a></li>
<li><a class="reference internal" href="#softmarginloss"><span class="hidden-section">SoftMarginLoss</span></a></li>
<li><a class="reference internal" href="#multilabelsoftmarginloss"><span class="hidden-section">MultiLabelSoftMarginLoss</span></a></li>
<li><a class="reference internal" href="#cosineembeddingloss"><span class="hidden-section">CosineEmbeddingLoss</span></a></li>
<li><a class="reference internal" href="#multimarginloss"><span class="hidden-section">MultiMarginLoss</span></a></li>
<li><a class="reference internal" href="#tripletmarginloss"><span class="hidden-section">TripletMarginLoss</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#vision-layers">Vision layers</a><ul>
<li><a class="reference internal" href="#pixelshuffle"><span class="hidden-section">PixelShuffle</span></a></li>
<li><a class="reference internal" href="#upsample"><span class="hidden-section">Upsample</span></a></li>
<li><a class="reference internal" href="#upsamplingnearest2d"><span class="hidden-section">UpsamplingNearest2d</span></a></li>
<li><a class="reference internal" href="#upsamplingbilinear2d"><span class="hidden-section">UpsamplingBilinear2d</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#dataparallel-layers-multi-gpu-distributed">DataParallel layers (multi-GPU, distributed)</a><ul>
<li><a class="reference internal" href="#dataparallel"><span class="hidden-section">DataParallel</span></a></li>
<li><a class="reference internal" href="#distributeddataparallel"><span class="hidden-section">DistributedDataParallel</span></a></li>
<li><a class="reference internal" href="#distributeddataparallelcpu"><span class="hidden-section">DistributedDataParallelCPU</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#utilities">Utilities</a><ul>
<li><a class="reference internal" href="#clip-grad-norm"><span class="hidden-section">clip_grad_norm_</span></a></li>
<li><a class="reference internal" href="#clip-grad-value"><span class="hidden-section">clip_grad_value_</span></a></li>
<li><a class="reference internal" href="#parameters-to-vector"><span class="hidden-section">parameters_to_vector</span></a></li>
<li><a class="reference internal" href="#vector-to-parameters"><span class="hidden-section">vector_to_parameters</span></a></li>
<li><a class="reference internal" href="#weight-norm"><span class="hidden-section">weight_norm</span></a></li>
<li><a class="reference internal" href="#remove-weight-norm"><span class="hidden-section">remove_weight_norm</span></a></li>
<li><a class="reference internal" href="#spectral-norm"><span class="hidden-section">spectral_norm</span></a></li>
<li><a class="reference internal" href="#remove-spectral-norm"><span class="hidden-section">remove_spectral_norm</span></a></li>
<li><a class="reference internal" href="#packedsequence"><span class="hidden-section">PackedSequence</span></a></li>
<li><a class="reference internal" href="#pack-padded-sequence"><span class="hidden-section">pack_padded_sequence</span></a></li>
<li><a class="reference internal" href="#pad-packed-sequence"><span class="hidden-section">pad_packed_sequence</span></a></li>
<li><a class="reference internal" href="#pad-sequence"><span class="hidden-section">pad_sequence</span></a></li>
<li><a class="reference internal" href="#pack-sequence"><span class="hidden-section">pack_sequence</span></a></li>
</ul>
</li>
</ul>
</li>
<li><a class="reference internal" href="#torch-nn-functional">torch.nn.functional</a><ul>
<li><a class="reference internal" href="#convolution-functions">Convolution functions</a><ul>
<li><a class="reference internal" href="#id21"><span class="hidden-section">conv1d</span></a></li>
<li><a class="reference internal" href="#id22"><span class="hidden-section">conv2d</span></a></li>
<li><a class="reference internal" href="#id23"><span class="hidden-section">conv3d</span></a></li>
<li><a class="reference internal" href="#conv-transpose1d"><span class="hidden-section">conv_transpose1d</span></a></li>
<li><a class="reference internal" href="#conv-transpose2d"><span class="hidden-section">conv_transpose2d</span></a></li>
<li><a class="reference internal" href="#conv-transpose3d"><span class="hidden-section">conv_transpose3d</span></a></li>
<li><a class="reference internal" href="#id24"><span class="hidden-section">unfold</span></a></li>
<li><a class="reference internal" href="#id25"><span class="hidden-section">fold</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#pooling-functions">Pooling functions</a><ul>
<li><a class="reference internal" href="#avg-pool1d"><span class="hidden-section">avg_pool1d</span></a></li>
<li><a class="reference internal" href="#avg-pool2d"><span class="hidden-section">avg_pool2d</span></a></li>
<li><a class="reference internal" href="#avg-pool3d"><span class="hidden-section">avg_pool3d</span></a></li>
<li><a class="reference internal" href="#max-pool1d"><span class="hidden-section">max_pool1d</span></a></li>
<li><a class="reference internal" href="#max-pool2d"><span class="hidden-section">max_pool2d</span></a></li>
<li><a class="reference internal" href="#max-pool3d"><span class="hidden-section">max_pool3d</span></a></li>
<li><a class="reference internal" href="#max-unpool1d"><span class="hidden-section">max_unpool1d</span></a></li>
<li><a class="reference internal" href="#max-unpool2d"><span class="hidden-section">max_unpool2d</span></a></li>
<li><a class="reference internal" href="#max-unpool3d"><span class="hidden-section">max_unpool3d</span></a></li>
<li><a class="reference internal" href="#lp-pool1d"><span class="hidden-section">lp_pool1d</span></a></li>
<li><a class="reference internal" href="#lp-pool2d"><span class="hidden-section">lp_pool2d</span></a></li>
<li><a class="reference internal" href="#adaptive-max-pool1d"><span class="hidden-section">adaptive_max_pool1d</span></a></li>
<li><a class="reference internal" href="#adaptive-max-pool2d"><span class="hidden-section">adaptive_max_pool2d</span></a></li>
<li><a class="reference internal" href="#adaptive-max-pool3d"><span class="hidden-section">adaptive_max_pool3d</span></a></li>
<li><a class="reference internal" href="#adaptive-avg-pool1d"><span class="hidden-section">adaptive_avg_pool1d</span></a></li>
<li><a class="reference internal" href="#adaptive-avg-pool2d"><span class="hidden-section">adaptive_avg_pool2d</span></a></li>
<li><a class="reference internal" href="#adaptive-avg-pool3d"><span class="hidden-section">adaptive_avg_pool3d</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#non-linear-activation-functions">Non-linear activation functions</a><ul>
<li><a class="reference internal" href="#id26"><span class="hidden-section">threshold</span></a></li>
<li><a class="reference internal" href="#id27"><span class="hidden-section">relu</span></a></li>
<li><a class="reference internal" href="#id28"><span class="hidden-section">hardtanh</span></a></li>
<li><a class="reference internal" href="#id29"><span class="hidden-section">relu6</span></a></li>
<li><a class="reference internal" href="#id30"><span class="hidden-section">elu</span></a></li>
<li><a class="reference internal" href="#id31"><span class="hidden-section">selu</span></a></li>
<li><a class="reference internal" href="#id32"><span class="hidden-section">celu</span></a></li>
<li><a class="reference internal" href="#leaky-relu"><span class="hidden-section">leaky_relu</span></a></li>
<li><a class="reference internal" href="#id33"><span class="hidden-section">prelu</span></a></li>
<li><a class="reference internal" href="#id34"><span class="hidden-section">rrelu</span></a></li>
<li><a class="reference internal" href="#glu"><span class="hidden-section">glu</span></a></li>
<li><a class="reference internal" href="#id35"><span class="hidden-section">logsigmoid</span></a></li>
<li><a class="reference internal" href="#id36"><span class="hidden-section">hardshrink</span></a></li>
<li><a class="reference internal" href="#id37"><span class="hidden-section">tanhshrink</span></a></li>
<li><a class="reference internal" href="#id38"><span class="hidden-section">softsign</span></a></li>
<li><a class="reference internal" href="#id39"><span class="hidden-section">softplus</span></a></li>
<li><a class="reference internal" href="#id40"><span class="hidden-section">softmin</span></a></li>
<li><a class="reference internal" href="#id41"><span class="hidden-section">softmax</span></a></li>
<li><a class="reference internal" href="#id42"><span class="hidden-section">softshrink</span></a></li>
<li><a class="reference internal" href="#gumbel-softmax"><span class="hidden-section">gumbel_softmax</span></a></li>
<li><a class="reference internal" href="#log-softmax"><span class="hidden-section">log_softmax</span></a></li>
<li><a class="reference internal" href="#id43"><span class="hidden-section">tanh</span></a></li>
<li><a class="reference internal" href="#id44"><span class="hidden-section">sigmoid</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#normalization-functions">Normalization functions</a><ul>
<li><a class="reference internal" href="#batch-norm"><span class="hidden-section">batch_norm</span></a></li>
<li><a class="reference internal" href="#instance-norm"><span class="hidden-section">instance_norm</span></a></li>
<li><a class="reference internal" href="#layer-norm"><span class="hidden-section">layer_norm</span></a></li>
<li><a class="reference internal" href="#local-response-norm"><span class="hidden-section">local_response_norm</span></a></li>
<li><a class="reference internal" href="#normalize"><span class="hidden-section">normalize</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#linear-functions">Linear functions</a><ul>
<li><a class="reference internal" href="#id45"><span class="hidden-section">linear</span></a></li>
<li><a class="reference internal" href="#id46"><span class="hidden-section">bilinear</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#dropout-functions">Dropout functions</a><ul>
<li><a class="reference internal" href="#id47"><span class="hidden-section">dropout</span></a></li>
<li><a class="reference internal" href="#alpha-dropout"><span class="hidden-section">alpha_dropout</span></a></li>
<li><a class="reference internal" href="#id48"><span class="hidden-section">dropout2d</span></a></li>
<li><a class="reference internal" href="#id49"><span class="hidden-section">dropout3d</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#sparse-functions">Sparse functions</a><ul>
<li><a class="reference internal" href="#id50"><span class="hidden-section">embedding</span></a></li>
<li><a class="reference internal" href="#embedding-bag"><span class="hidden-section">embedding_bag</span></a></li>
<li><a class="reference internal" href="#one-hot"><span class="hidden-section">one_hot</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#id51">Distance functions</a><ul>
<li><a class="reference internal" href="#pairwise-distance"><span class="hidden-section">pairwise_distance</span></a></li>
<li><a class="reference internal" href="#cosine-similarity"><span class="hidden-section">cosine_similarity</span></a></li>
<li><a class="reference internal" href="#pdist"><span class="hidden-section">pdist</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#id52">Loss functions</a><ul>
<li><a class="reference internal" href="#binary-cross-entropy"><span class="hidden-section">binary_cross_entropy</span></a></li>
<li><a class="reference internal" href="#binary-cross-entropy-with-logits"><span class="hidden-section">binary_cross_entropy_with_logits</span></a></li>
<li><a class="reference internal" href="#poisson-nll-loss"><span class="hidden-section">poisson_nll_loss</span></a></li>
<li><a class="reference internal" href="#cosine-embedding-loss"><span class="hidden-section">cosine_embedding_loss</span></a></li>
<li><a class="reference internal" href="#cross-entropy"><span class="hidden-section">cross_entropy</span></a></li>
<li><a class="reference internal" href="#ctc-loss"><span class="hidden-section">ctc_loss</span></a></li>
<li><a class="reference internal" href="#hinge-embedding-loss"><span class="hidden-section">hinge_embedding_loss</span></a></li>
<li><a class="reference internal" href="#kl-div"><span class="hidden-section">kl_div</span></a></li>
<li><a class="reference internal" href="#l1-loss"><span class="hidden-section">l1_loss</span></a></li>
<li><a class="reference internal" href="#mse-loss"><span class="hidden-section">mse_loss</span></a></li>
<li><a class="reference internal" href="#margin-ranking-loss"><span class="hidden-section">margin_ranking_loss</span></a></li>
<li><a class="reference internal" href="#multilabel-margin-loss"><span class="hidden-section">multilabel_margin_loss</span></a></li>
<li><a class="reference internal" href="#multilabel-soft-margin-loss"><span class="hidden-section">multilabel_soft_margin_loss</span></a></li>
<li><a class="reference internal" href="#multi-margin-loss"><span class="hidden-section">multi_margin_loss</span></a></li>
<li><a class="reference internal" href="#nll-loss"><span class="hidden-section">nll_loss</span></a></li>
<li><a class="reference internal" href="#smooth-l1-loss"><span class="hidden-section">smooth_l1_loss</span></a></li>
<li><a class="reference internal" href="#soft-margin-loss"><span class="hidden-section">soft_margin_loss</span></a></li>
<li><a class="reference internal" href="#triplet-margin-loss"><span class="hidden-section">triplet_margin_loss</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#vision-functions">Vision functions</a><ul>
<li><a class="reference internal" href="#pixel-shuffle"><span class="hidden-section">pixel_shuffle</span></a></li>
<li><a class="reference internal" href="#pad"><span class="hidden-section">pad</span></a></li>
<li><a class="reference internal" href="#interpolate"><span class="hidden-section">interpolate</span></a></li>
<li><a class="reference internal" href="#id53"><span class="hidden-section">upsample</span></a></li>
<li><a class="reference internal" href="#upsample-nearest"><span class="hidden-section">upsample_nearest</span></a></li>
<li><a class="reference internal" href="#upsample-bilinear"><span class="hidden-section">upsample_bilinear</span></a></li>
<li><a class="reference internal" href="#grid-sample"><span class="hidden-section">grid_sample</span></a></li>
<li><a class="reference internal" href="#affine-grid"><span class="hidden-section">affine_grid</span></a></li>
</ul>
</li>
<li><a class="reference internal" href="#dataparallel-functions-multi-gpu-distributed">DataParallel functions (multi-GPU, distributed)</a><ul>
<li><a class="reference internal" href="#data-parallel"><span class="hidden-section">data_parallel</span></a></li>
</ul>
</li>
</ul>
</li>
<li><a class="reference internal" href="#torch-nn-init">torch.nn.init</a></li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  
       <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
         <script type="text/javascript" src="_static/jquery.js"></script>
         <script type="text/javascript" src="_static/underscore.js"></script>
         <script type="text/javascript" src="_static/doctools.js"></script>
         <script type="text/javascript" src="_static/language_data.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/katex.min.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/contrib/auto-render.min.js"></script>
         <script type="text/javascript" src="_static/katex_autorenderer.js"></script>
     

  <script type="text/javascript" src="_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="_static/js/vendor/bootstrap.min.js"></script>
  <script type="text/javascript" src="_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script> 

  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://pytorch.org/resources">Resources</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/support">Support</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.slack.com" target="_blank">Slack</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md" target="_blank">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Follow Us</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="#">Get Started</a>
          </li>

          <li>
            <a href="#">Features</a>
          </li>

          <li>
            <a href="#">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>