2.0/autograd.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta name="robots" content="noindex">
  <meta charset="utf-8">
  <meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Automatic differentiation package - torch.autograd &mdash; PyTorch 2.0 documentation</title>
  

    <link rel="canonical" href="https://pytorch.org/docs/stable/autograd.html"/>
  

  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="_static/copybutton.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="_static/katex-math.css" type="text/css" />
  <link rel="stylesheet" href="_static/sphinx-dropdown.css" type="text/css" />
  <link rel="stylesheet" href="_static/panels-bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="_static/css/jit.css" type="text/css" />
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="torch.autograd.backward" href="generated/torch.autograd.backward.html" />
    <link rel="prev" title="Automatic Mixed Precision package - torch.amp" href="amp.html" />


  <!-- Google Analytics -->
  
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());

      gtag('config', 'UA-117752657-2');
    </script>
  
  <!-- End Google Analytics -->
  

  <script src="_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active docs-active">
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="resource-option with-down-orange-arrow">
                Docs
              </a>
              <div class="resources-dropdown-menu">
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/docs/stable/index.html">
                  <span class="dropdown-title">PyTorch</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/audio/stable/index.html">
                  <span class="dropdown-title">torchaudio</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/text/stable/index.html">
                  <span class="dropdown-title">torchtext</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/vision/stable/index.html">
                  <span class="dropdown-title">torchvision</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torcharrow">
                  <span class="dropdown-title">torcharrow</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/data">
                  <span class="dropdown-title">TorchData</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torchrec">
                  <span class="dropdown-title">TorchRec</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/serve/">
                  <span class="dropdown-title">TorchServe</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torchx/">
                  <span class="dropdown-title">TorchX</span>
                  <p></p>
                </a>
                <a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/xla">
                  <span class="dropdown-title">PyTorch on XLA Devices</span>
                  <p></p>
                </a>
            </div>
          </li>

          <li>
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="resource-option with-down-arrow">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class="dropdown-title">About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/foundation">
                  <span class="dropdown-title">PyTorch Foundation</span>
                  <p>Learn about the PyTorch foundation</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/#community-module">
                  <span class="dropdown-title">Community</span>
                  <p>Join the PyTorch developer community to contribute, learn, and get your questions answered.</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/community-stories">
                  <span class="dropdown-title">Community Stories</span>
                  <p>Learn how our community solves real, everyday machine learning problems with PyTorch.</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/resources">
                  <span class="dropdown-title">Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/events">
                  <span class="dropdown-title">Events</span>
                  <p>Find events, webinars, and podcasts</p>
                </a>
                <a class="nav-dropdown-item" href="https://discuss.pytorch.org/" target="_blank">
                  <span class="dropdown-title">Forums</span>
                  <p>A place to discuss PyTorch code, issues, install, research</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/hub">
                  <span class="dropdown-title">Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">GitHub</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">

   
    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            
    <div class="version">
      <a href='https://pytorch.org/docs/versions.html'>2.0 &#x25BC</a>
    </div>
    

<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          </div>

          
              <p class="caption" role="heading"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="community/build_ci_governance.html">PyTorch Governance | Build + CI</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/design.html">PyTorch Design Philosophy</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/governance.html">PyTorch Governance | Mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/persons_of_interest.html">PyTorch Governance | Maintainers</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Developer Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="notes/amp_examples.html">CUDA Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/extending.func.html">Extending torch.func with autograd.Function</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/gradcheck.html">Gradcheck mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/hip.html">HIP (ROCm) semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/modules.html">Modules</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/mps.html">MPS backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/numerical_accuracy.html">Numerical accuracy</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">torch.compile</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="dynamo/index.html">TorchDynamo Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamo/installation.html">Installing TorchDynamo</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamo/get-started.html">Getting Started</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamo/guards-overview.html">Guards Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamo/custom-backends.html">Custom Backends</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamo/deep-dive.html">TorchDynamo Deeper Dive</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamo/troubleshooting.html">TorchDynamo Troubleshooting</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamo/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="ir.html">IRs</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="cpp_index.html">C++</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/javadoc/">Javadoc</a></li>
<li class="toctree-l1"><a class="reference internal" href="deploy.html">torch::deploy</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Python API</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="amp.html">torch.amp</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="library.html">torch.library</a></li>
<li class="toctree-l1"><a class="reference internal" href="cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="mps.html">torch.mps</a></li>
<li class="toctree-l1"><a class="reference internal" href="backends.html">torch.backends</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.algorithms.join.html">torch.distributed.algorithms.join</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.elastic.html">torch.distributed.elastic</a></li>
<li class="toctree-l1"><a class="reference internal" href="fsdp.html">torch.distributed.fsdp</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.optim.html">torch.distributed.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.tensor.parallel.html">torch.distributed.tensor.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.checkpoint.html">torch.distributed.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="_dynamo.html">torch._dynamo</a></li>
<li class="toctree-l1"><a class="reference internal" href="fft.html">torch.fft</a></li>
<li class="toctree-l1"><a class="reference internal" href="func.html">torch.func</a></li>
<li class="toctree-l1"><a class="reference internal" href="futures.html">torch.futures</a></li>
<li class="toctree-l1"><a class="reference internal" href="fx.html">torch.fx</a></li>
<li class="toctree-l1"><a class="reference internal" href="hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="linalg.html">torch.linalg</a></li>
<li class="toctree-l1"><a class="reference internal" href="monitor.html">torch.monitor</a></li>
<li class="toctree-l1"><a class="reference internal" href="signal.html">torch.signal</a></li>
<li class="toctree-l1"><a class="reference internal" href="special.html">torch.special</a></li>
<li class="toctree-l1"><a class="reference internal" href="torch.overrides.html">torch.overrides</a></li>
<li class="toctree-l1"><a class="reference internal" href="package.html">torch.package</a></li>
<li class="toctree-l1"><a class="reference internal" href="profiler.html">torch.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="onnx_diagnostics.html">torch.onnx diagnostics</a></li>
<li class="toctree-l1"><a class="reference internal" href="optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="complex_numbers.html">Complex Numbers</a></li>
<li class="toctree-l1"><a class="reference internal" href="ddp_comm_hooks.html">DDP Communication Hooks</a></li>
<li class="toctree-l1"><a class="reference internal" href="pipeline.html">Pipeline Parallelism</a></li>
<li class="toctree-l1"><a class="reference internal" href="quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="rpc.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="masked.html">torch.masked</a></li>
<li class="toctree-l1"><a class="reference internal" href="nested.html">torch.nested</a></li>
<li class="toctree-l1"><a class="reference internal" href="sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="testing.html">torch.testing</a></li>
<li class="toctree-l1"><a class="reference internal" href="benchmark_utils.html">torch.utils.benchmark</a></li>
<li class="toctree-l1"><a class="reference internal" href="bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="jit_utils.html">torch.utils.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="mobile_optimizer.html">torch.utils.mobile_optimizer</a></li>
<li class="toctree-l1"><a class="reference internal" href="model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="config_mod.html">torch.__config__</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio/stable">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/data">TorchData</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/torchrec">TorchRec</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text/stable">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/vision/stable">torchvision</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          

<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
      <li>Automatic differentiation package - torch.autograd</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="_sources/autograd.rst.txt" rel="nofollow"><img src="_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <section id="module-torch.autograd">
<span id="automatic-differentiation-package-torch-autograd"></span><h1>Automatic differentiation package - torch.autograd<a class="headerlink" href="#module-torch.autograd" title="Permalink to this heading">¶</a></h1>
<p><code class="docutils literal notranslate"><span class="pre">torch.autograd</span></code> provides classes and functions implementing automatic
differentiation of arbitrary scalar valued functions. It requires minimal
changes to the existing code - you only need to declare <code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> s
for which gradients should be computed with the <code class="docutils literal notranslate"><span class="pre">requires_grad=True</span></code> keyword.
As of now, we only support autograd for floating point <code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> types (
half, float, double and bfloat16) and complex <code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> types (cfloat, cdouble).</p>
<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><p id="torch.autograd.backward"/><a class="reference internal" href="generated/torch.autograd.backward.html#torch.autograd.backward" title="torch.autograd.backward"><code class="xref py py-obj docutils literal notranslate"><span class="pre">backward</span></code></a></p></td>
<td><p>Computes the sum of gradients of given tensors with respect to graph leaves.</p></td>
</tr>
<tr class="row-even"><td><p><p id="torch.autograd.grad"/><a class="reference internal" href="generated/torch.autograd.grad.html#torch.autograd.grad" title="torch.autograd.grad"><code class="xref py py-obj docutils literal notranslate"><span class="pre">grad</span></code></a></p></td>
<td><p>Computes and returns the sum of gradients of outputs with respect to the inputs.</p></td>
</tr>
</tbody>
</table>
<section id="forward-mode-automatic-differentiation">
<span id="forward-mode-ad"></span><h2>Forward-mode Automatic Differentiation<a class="headerlink" href="#forward-mode-automatic-differentiation" title="Permalink to this heading">¶</a></h2>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This API is in beta. Even though the function signatures are very unlikely to change, improved
operator coverage is planned before we consider this stable.</p>
</div>
<p>Please see the <a class="reference external" href="https://pytorch.org/tutorials/intermediate/forward_ad_usage.html">forward-mode AD tutorial</a>
for detailed steps on how to use this API.</p>
<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.forward_ad.dual_level.html#torch.autograd.forward_ad.dual_level" title="torch.autograd.forward_ad.dual_level"><code class="xref py py-obj docutils literal notranslate"><span class="pre">forward_ad.dual_level</span></code></a></p></td>
<td><p>Context-manager that enables forward AD.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.forward_ad.make_dual.html#torch.autograd.forward_ad.make_dual" title="torch.autograd.forward_ad.make_dual"><code class="xref py py-obj docutils literal notranslate"><span class="pre">forward_ad.make_dual</span></code></a></p></td>
<td><p>Associates a tensor value with a forward gradient, the tangent, to create a &quot;dual tensor&quot;, which is used to compute forward AD gradients.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.forward_ad.unpack_dual.html#torch.autograd.forward_ad.unpack_dual" title="torch.autograd.forward_ad.unpack_dual"><code class="xref py py-obj docutils literal notranslate"><span class="pre">forward_ad.unpack_dual</span></code></a></p></td>
<td><p>Unpacks a &quot;dual tensor&quot; to get both its Tensor value and its forward AD gradient.</p></td>
</tr>
</tbody>
</table>
</section>
<section id="functional-higher-level-api">
<span id="functional-api"></span><h2>Functional higher level API<a class="headerlink" href="#functional-higher-level-api" title="Permalink to this heading">¶</a></h2>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This API is in beta. Even though the function signatures are very unlikely to change, major
improvements to performances are planned before we consider this stable.</p>
</div>
<p>This section contains the higher level API for the autograd that builds on the basic API above
and allows you to compute jacobians, hessians, etc.</p>
<p>This API works with user-provided functions that take only Tensors as input and return
only Tensors.
If your function takes other arguments that are not Tensors or Tensors that don’t have requires_grad set,
you can use a lambda to capture them.
For example, for a function <code class="docutils literal notranslate"><span class="pre">f</span></code> that takes three inputs, a Tensor for which we want the jacobian, another
tensor that should be considered constant and a boolean flag as <code class="docutils literal notranslate"><span class="pre">f(input,</span> <span class="pre">constant,</span> <span class="pre">flag=flag)</span></code>
you can use it as <code class="docutils literal notranslate"><span class="pre">functional.jacobian(lambda</span> <span class="pre">x:</span> <span class="pre">f(x,</span> <span class="pre">constant,</span> <span class="pre">flag=flag),</span> <span class="pre">input)</span></code>.</p>
<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.functional.jacobian.html#torch.autograd.functional.jacobian" title="torch.autograd.functional.jacobian"><code class="xref py py-obj docutils literal notranslate"><span class="pre">functional.jacobian</span></code></a></p></td>
<td><p>Function that computes the Jacobian of a given function.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.functional.hessian.html#torch.autograd.functional.hessian" title="torch.autograd.functional.hessian"><code class="xref py py-obj docutils literal notranslate"><span class="pre">functional.hessian</span></code></a></p></td>
<td><p>Function that computes the Hessian of a given scalar function.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.functional.vjp.html#torch.autograd.functional.vjp" title="torch.autograd.functional.vjp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">functional.vjp</span></code></a></p></td>
<td><p>Function that computes the dot product between a vector <code class="docutils literal notranslate"><span class="pre">v</span></code> and the Jacobian of the given function at the point given by the inputs.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.functional.jvp.html#torch.autograd.functional.jvp" title="torch.autograd.functional.jvp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">functional.jvp</span></code></a></p></td>
<td><p>Function that computes the dot product between  the Jacobian of the given function at the point given by the inputs and a vector <code class="docutils literal notranslate"><span class="pre">v</span></code>.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.functional.vhp.html#torch.autograd.functional.vhp" title="torch.autograd.functional.vhp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">functional.vhp</span></code></a></p></td>
<td><p>Function that computes the dot product between a vector <code class="docutils literal notranslate"><span class="pre">v</span></code> and the Hessian of a given scalar function at the point given by the inputs.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.functional.hvp.html#torch.autograd.functional.hvp" title="torch.autograd.functional.hvp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">functional.hvp</span></code></a></p></td>
<td><p>Function that computes the dot product between the Hessian of a given scalar function and a vector <code class="docutils literal notranslate"><span class="pre">v</span></code> at the point given by the inputs.</p></td>
</tr>
</tbody>
</table>
</section>
<section id="locally-disabling-gradient-computation">
<span id="locally-disable-grad"></span><h2>Locally disabling gradient computation<a class="headerlink" href="#locally-disabling-gradient-computation" title="Permalink to this heading">¶</a></h2>
<p>See <a class="reference internal" href="notes/autograd.html#locally-disable-grad-doc"><span class="std std-ref">Locally disabling gradient computation</span></a> for more information on the differences
between no-grad and inference mode as well as other related mechanisms that
may be confused with the two. Also see <a class="reference internal" href="torch.html#torch-rst-local-disable-grad"><span class="std std-ref">Locally disabling gradient computation</span></a>
for a list of functions that can be used to locally disable gradients.</p>
</section>
<section id="default-gradient-layouts">
<span id="default-grad-layouts"></span><h2>Default gradient layouts<a class="headerlink" href="#default-gradient-layouts" title="Permalink to this heading">¶</a></h2>
<p>When a non-sparse <code class="docutils literal notranslate"><span class="pre">param</span></code> receives a non-sparse gradient during
<a class="reference internal" href="generated/torch.autograd.backward.html#torch.autograd.backward" title="torch.autograd.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.autograd.backward()</span></code></a> or <a class="reference internal" href="generated/torch.Tensor.backward.html#torch.Tensor.backward" title="torch.Tensor.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.Tensor.backward()</span></code></a>
<code class="docutils literal notranslate"><span class="pre">param.grad</span></code> is accumulated as follows.</p>
<p>If <code class="docutils literal notranslate"><span class="pre">param.grad</span></code> is initially <code class="docutils literal notranslate"><span class="pre">None</span></code>:</p>
<ol class="arabic simple">
<li><p>If <code class="docutils literal notranslate"><span class="pre">param</span></code>’s memory is non-overlapping and dense, <code class="docutils literal notranslate"><span class="pre">.grad</span></code> is
created with strides matching <code class="docutils literal notranslate"><span class="pre">param</span></code> (thus matching <code class="docutils literal notranslate"><span class="pre">param</span></code>’s
layout).</p></li>
<li><p>Otherwise, <code class="docutils literal notranslate"><span class="pre">.grad</span></code> is created with rowmajor-contiguous strides.</p></li>
</ol>
<p>If <code class="docutils literal notranslate"><span class="pre">param</span></code> already has a non-sparse <code class="docutils literal notranslate"><span class="pre">.grad</span></code> attribute:</p>
<ol class="arabic simple" start="3">
<li><p>If <code class="docutils literal notranslate"><span class="pre">create_graph=False</span></code>, <code class="docutils literal notranslate"><span class="pre">backward()</span></code> accumulates into <code class="docutils literal notranslate"><span class="pre">.grad</span></code>
in-place, which preserves its strides.</p></li>
<li><p>If <code class="docutils literal notranslate"><span class="pre">create_graph=True</span></code>, <code class="docutils literal notranslate"><span class="pre">backward()</span></code> replaces <code class="docutils literal notranslate"><span class="pre">.grad</span></code> with a
new tensor <code class="docutils literal notranslate"><span class="pre">.grad</span> <span class="pre">+</span> <span class="pre">new</span> <span class="pre">grad</span></code>, which attempts (but does not guarantee)
matching the preexisting <code class="docutils literal notranslate"><span class="pre">.grad</span></code>’s strides.</p></li>
</ol>
<p>The default behavior (letting <code class="docutils literal notranslate"><span class="pre">.grad</span></code>s be <code class="docutils literal notranslate"><span class="pre">None</span></code> before the first
<code class="docutils literal notranslate"><span class="pre">backward()</span></code>, such that their layout is created according to 1 or 2,
and retained over time according to 3 or 4) is recommended for best performance.
Calls to <code class="docutils literal notranslate"><span class="pre">model.zero_grad()</span></code> or <code class="docutils literal notranslate"><span class="pre">optimizer.zero_grad()</span></code> will not affect <code class="docutils literal notranslate"><span class="pre">.grad</span></code>
layouts.</p>
<p>In fact, resetting all <code class="docutils literal notranslate"><span class="pre">.grad</span></code>s to <code class="docutils literal notranslate"><span class="pre">None</span></code> before each
accumulation phase, e.g.:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="n">iterations</span><span class="o">...</span>
    <span class="o">...</span>
    <span class="k">for</span> <span class="n">param</span> <span class="ow">in</span> <span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">():</span>
        <span class="n">param</span><span class="o">.</span><span class="n">grad</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</div>
<p>such that they’re recreated according to 1 or 2 every time,
is a valid alternative to <code class="docutils literal notranslate"><span class="pre">model.zero_grad()</span></code> or <code class="docutils literal notranslate"><span class="pre">optimizer.zero_grad()</span></code>
that may improve performance for some networks.</p>
<section id="manual-gradient-layouts">
<h3>Manual gradient layouts<a class="headerlink" href="#manual-gradient-layouts" title="Permalink to this heading">¶</a></h3>
<p>If you need manual control over <code class="docutils literal notranslate"><span class="pre">.grad</span></code>’s strides,
assign <code class="docutils literal notranslate"><span class="pre">param.grad</span> <span class="pre">=</span></code> a zeroed tensor with desired strides
before the first <code class="docutils literal notranslate"><span class="pre">backward()</span></code>, and never reset it to <code class="docutils literal notranslate"><span class="pre">None</span></code>.
3 guarantees your layout is preserved as long as <code class="docutils literal notranslate"><span class="pre">create_graph=False</span></code>.
4 indicates your layout is <em>likely</em> preserved even if <code class="docutils literal notranslate"><span class="pre">create_graph=True</span></code>.</p>
</section>
</section>
<section id="in-place-operations-on-tensors">
<h2>In-place operations on Tensors<a class="headerlink" href="#in-place-operations-on-tensors" title="Permalink to this heading">¶</a></h2>
<p>Supporting in-place operations in autograd is a hard matter, and we discourage
their use in most cases. Autograd’s aggressive buffer freeing and reuse makes
it very efficient and there are very few occasions when in-place operations
actually lower memory usage by any significant amount. Unless you’re operating
under heavy memory pressure, you might never need to use them.</p>
<section id="in-place-correctness-checks">
<h3>In-place correctness checks<a class="headerlink" href="#in-place-correctness-checks" title="Permalink to this heading">¶</a></h3>
<p>All <code class="xref py py-class docutils literal notranslate"><span class="pre">Tensor</span></code> s keep track of in-place operations applied to them, and
if the implementation detects that a tensor was saved for backward in one of
the functions, but it was modified in-place afterwards, an error will be raised
once backward pass is started. This ensures that if you’re using in-place
functions and not seeing any errors, you can be sure that the computed
gradients are correct.</p>
</section>
</section>
<section id="variable-deprecated">
<h2>Variable (deprecated)<a class="headerlink" href="#variable-deprecated" title="Permalink to this heading">¶</a></h2>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>The Variable API has been deprecated: Variables are no longer necessary to
use autograd with tensors. Autograd automatically supports Tensors with
<code class="docutils literal notranslate"><span class="pre">requires_grad</span></code> set to <code class="docutils literal notranslate"><span class="pre">True</span></code>. Below please find a quick guide on what
has changed:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">Variable(tensor)</span></code> and <code class="docutils literal notranslate"><span class="pre">Variable(tensor,</span> <span class="pre">requires_grad)</span></code> still work as expected,
but they return Tensors instead of Variables.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">var.data</span></code> is the same thing as <code class="docutils literal notranslate"><span class="pre">tensor.data</span></code>.</p></li>
<li><p>Methods such as <code class="docutils literal notranslate"><span class="pre">var.backward(),</span> <span class="pre">var.detach(),</span> <span class="pre">var.register_hook()</span></code> now work on tensors
with the same method names.</p></li>
</ul>
<p>In addition, one can now create tensors with <code class="docutils literal notranslate"><span class="pre">requires_grad=True</span></code> using factory
methods such as <a class="reference internal" href="generated/torch.randn.html#torch.randn" title="torch.randn"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.randn()</span></code></a>, <a class="reference internal" href="generated/torch.zeros.html#torch.zeros" title="torch.zeros"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.zeros()</span></code></a>, <a class="reference internal" href="generated/torch.ones.html#torch.ones" title="torch.ones"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.ones()</span></code></a>, and others
like the following:</p>
<p><code class="docutils literal notranslate"><span class="pre">autograd_tensor</span> <span class="pre">=</span> <span class="pre">torch.randn((2,</span> <span class="pre">3,</span> <span class="pre">4),</span> <span class="pre">requires_grad=True)</span></code></p>
</div>
</section>
<section id="tensor-autograd-functions">
<h2>Tensor autograd functions<a class="headerlink" href="#tensor-autograd-functions" title="Permalink to this heading">¶</a></h2>
<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.grad</span></code></p></td>
<td><p>This attribute is <code class="docutils literal notranslate"><span class="pre">None</span></code> by default and becomes a Tensor the first time a call to <a class="reference internal" href="generated/torch.autograd.backward.html#torch.autograd.backward" title="torch.autograd.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a> computes gradients for <code class="docutils literal notranslate"><span class="pre">self</span></code>.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.requires_grad</span></code></p></td>
<td><p>Is <code class="docutils literal notranslate"><span class="pre">True</span></code> if gradients need to be computed for this Tensor, <code class="docutils literal notranslate"><span class="pre">False</span></code> otherwise.</p></td>
</tr>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.is_leaf</span></code></p></td>
<td><p>All Tensors that have <code class="xref py py-attr docutils literal notranslate"><span class="pre">requires_grad</span></code> which is <code class="docutils literal notranslate"><span class="pre">False</span></code> will be leaf Tensors by convention.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.backward</span></code>([gradient, ...])</p></td>
<td><p>Computes the gradient of current tensor w.r.t.</p></td>
</tr>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.detach</span></code></p></td>
<td><p>Returns a new Tensor, detached from the current graph.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.detach_</span></code></p></td>
<td><p>Detaches the Tensor from the graph that created it, making it a leaf.</p></td>
</tr>
<tr class="row-odd"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.register_hook</span></code>(hook)</p></td>
<td><p>Registers a backward hook.</p></td>
</tr>
<tr class="row-even"><td><p><code class="xref py py-obj docutils literal notranslate"><span class="pre">torch.Tensor.retain_grad</span></code>()</p></td>
<td><p>Enables this Tensor to have their <a class="reference internal" href="generated/torch.autograd.grad.html#torch.autograd.grad" title="torch.autograd.grad"><code class="xref py py-attr docutils literal notranslate"><span class="pre">grad</span></code></a> populated during <a class="reference internal" href="generated/torch.autograd.backward.html#torch.autograd.backward" title="torch.autograd.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a>.</p></td>
</tr>
</tbody>
</table>
</section>
<section id="function">
<h2><span class="hidden-section">Function</span><a class="headerlink" href="#function" title="Permalink to this heading">¶</a></h2>
<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.Function">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.</span></span><span class="sig-name descname"><span class="pre">Function</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">args</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/function.html#Function"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.Function" title="Permalink to this definition">¶</a></dt>
<dd><p>Base class to create custom <cite>autograd.Function</cite></p>
<p>To create a custom <cite>autograd.Function</cite>, subclass this class and implement
the <a class="reference internal" href="generated/torch.autograd.Function.forward.html#torch.autograd.Function.forward" title="torch.autograd.Function.forward"><code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code></a> and <a class="reference internal" href="generated/torch.autograd.backward.html#torch.autograd.backward" title="torch.autograd.backward"><code class="xref py py-meth docutils literal notranslate"><span class="pre">backward()</span></code></a> static methods. Then, to use your custom
op in the forward pass, call the class method <code class="docutils literal notranslate"><span class="pre">apply</span></code>. Do not call
<a class="reference internal" href="generated/torch.autograd.Function.forward.html#torch.autograd.Function.forward" title="torch.autograd.Function.forward"><code class="xref py py-meth docutils literal notranslate"><span class="pre">forward()</span></code></a> directly.</p>
<p>To ensure correctness and best performance, make sure you are calling the
correct methods on <code class="docutils literal notranslate"><span class="pre">ctx</span></code> and validating your backward function using
<a class="reference internal" href="generated/torch.autograd.gradcheck.html#torch.autograd.gradcheck" title="torch.autograd.gradcheck"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.autograd.gradcheck()</span></code></a>.</p>
<p>See <a class="reference internal" href="notes/extending.html#extending-autograd"><span class="std std-ref">Extending torch.autograd</span></a> for more details on how to use this class.</p>
<p>Examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">class</span> <span class="nc">Exp</span><span class="p">(</span><span class="n">Function</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="nd">@staticmethod</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">i</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">result</span> <span class="o">=</span> <span class="n">i</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">ctx</span><span class="o">.</span><span class="n">save_for_backward</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="k">return</span> <span class="n">result</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="nd">@staticmethod</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">grad_output</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">result</span><span class="p">,</span> <span class="o">=</span> <span class="n">ctx</span><span class="o">.</span><span class="n">saved_tensors</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="k">return</span> <span class="n">grad_output</span> <span class="o">*</span> <span class="n">result</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Use it by calling the apply method:</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output</span> <span class="o">=</span> <span class="n">Exp</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.Function.forward.html#torch.autograd.Function.forward" title="torch.autograd.Function.forward"><code class="xref py py-obj docutils literal notranslate"><span class="pre">Function.forward</span></code></a></p></td>
<td><p>This function is to be overridden by all subclasses.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.Function.backward.html#torch.autograd.Function.backward" title="torch.autograd.Function.backward"><code class="xref py py-obj docutils literal notranslate"><span class="pre">Function.backward</span></code></a></p></td>
<td><p>Defines a formula for differentiating the operation with backward mode automatic differentiation (alias to the vjp function).</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.Function.jvp.html#torch.autograd.Function.jvp" title="torch.autograd.Function.jvp"><code class="xref py py-obj docutils literal notranslate"><span class="pre">Function.jvp</span></code></a></p></td>
<td><p>Defines a formula for differentiating the operation with forward mode automatic differentiation.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.Function.vmap.html#torch.autograd.Function.vmap" title="torch.autograd.Function.vmap"><code class="xref py py-obj docutils literal notranslate"><span class="pre">Function.vmap</span></code></a></p></td>
<td><p>Defines a rule for the behavior of this autograd.Function underneath <a class="reference internal" href="generated/torch.vmap.html#torch.vmap" title="torch.vmap"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.vmap()</span></code></a>.</p></td>
</tr>
</tbody>
</table>
</section>
<section id="context-method-mixins">
<h2>Context method mixins<a class="headerlink" href="#context-method-mixins" title="Permalink to this heading">¶</a></h2>
<p>When creating a new <a class="reference internal" href="#torch.autograd.Function" title="torch.autograd.Function"><code class="xref py py-class docutils literal notranslate"><span class="pre">Function</span></code></a>, the following methods are available to <cite>ctx</cite>.</p>
<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.function.FunctionCtx.mark_dirty.html#torch.autograd.function.FunctionCtx.mark_dirty" title="torch.autograd.function.FunctionCtx.mark_dirty"><code class="xref py py-obj docutils literal notranslate"><span class="pre">function.FunctionCtx.mark_dirty</span></code></a></p></td>
<td><p>Marks given tensors as modified in an in-place operation.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.function.FunctionCtx.mark_non_differentiable.html#torch.autograd.function.FunctionCtx.mark_non_differentiable" title="torch.autograd.function.FunctionCtx.mark_non_differentiable"><code class="xref py py-obj docutils literal notranslate"><span class="pre">function.FunctionCtx.mark_non_differentiable</span></code></a></p></td>
<td><p>Marks outputs as non-differentiable.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.function.FunctionCtx.save_for_backward.html#torch.autograd.function.FunctionCtx.save_for_backward" title="torch.autograd.function.FunctionCtx.save_for_backward"><code class="xref py py-obj docutils literal notranslate"><span class="pre">function.FunctionCtx.save_for_backward</span></code></a></p></td>
<td><p>Saves given tensors for a future call to <a class="reference internal" href="generated/torch.autograd.Function.backward.html#torch.autograd.Function.backward" title="torch.autograd.Function.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code></a>.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.function.FunctionCtx.set_materialize_grads.html#torch.autograd.function.FunctionCtx.set_materialize_grads" title="torch.autograd.function.FunctionCtx.set_materialize_grads"><code class="xref py py-obj docutils literal notranslate"><span class="pre">function.FunctionCtx.set_materialize_grads</span></code></a></p></td>
<td><p>Sets whether to materialize grad tensors.</p></td>
</tr>
</tbody>
</table>
</section>
<section id="numerical-gradient-checking">
<span id="grad-check"></span><h2>Numerical gradient checking<a class="headerlink" href="#numerical-gradient-checking" title="Permalink to this heading">¶</a></h2>
<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><p id="torch.autograd.gradcheck"/><a class="reference internal" href="generated/torch.autograd.gradcheck.html#torch.autograd.gradcheck" title="torch.autograd.gradcheck"><code class="xref py py-obj docutils literal notranslate"><span class="pre">gradcheck</span></code></a></p></td>
<td><p>Check gradients computed via small finite differences against analytical gradients w.r.t.</p></td>
</tr>
<tr class="row-even"><td><p><p id="torch.autograd.gradgradcheck"/><a class="reference internal" href="generated/torch.autograd.gradgradcheck.html#torch.autograd.gradgradcheck" title="torch.autograd.gradgradcheck"><code class="xref py py-obj docutils literal notranslate"><span class="pre">gradgradcheck</span></code></a></p></td>
<td><p>Check gradients of gradients computed via small finite differences against analytical gradients w.r.t.</p></td>
</tr>
</tbody>
</table>
</section>
<section id="profiler">
<h2>Profiler<a class="headerlink" href="#profiler" title="Permalink to this heading">¶</a></h2>
<p>Autograd includes a profiler that lets you inspect the cost of different
operators inside your model - both on the CPU and GPU. There are three modes
implemented at the moment - CPU-only using <a class="reference internal" href="#torch.autograd.profiler.profile" title="torch.autograd.profiler.profile"><code class="xref py py-class docutils literal notranslate"><span class="pre">profile</span></code></a>.
nvprof based (registers both CPU and GPU activity) using
<a class="reference internal" href="#torch.autograd.profiler.emit_nvtx" title="torch.autograd.profiler.emit_nvtx"><code class="xref py py-class docutils literal notranslate"><span class="pre">emit_nvtx</span></code></a>.
and vtune profiler based using
<a class="reference internal" href="#torch.autograd.profiler.emit_itt" title="torch.autograd.profiler.emit_itt"><code class="xref py py-class docutils literal notranslate"><span class="pre">emit_itt</span></code></a>.</p>
<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.profiler.profile">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.profiler.</span></span><span class="sig-name descname"><span class="pre">profile</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">enabled</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">*</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_cuda</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">record_shapes</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">with_flops</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">profile_memory</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">with_stack</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">with_modules</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_kineto</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">use_cpu</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">experimental_config</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#profile"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.profiler.profile" title="Permalink to this definition">¶</a></dt>
<dd><p>Context manager that manages autograd profiler state and holds a summary of results.
Under the hood it just records events of functions being executed in C++ and
exposes those events to Python. You can wrap any code into it and it will
only report runtime of PyTorch functions.
Note: profiler is thread local and is automatically propagated into the async tasks</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>enabled</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – Setting this to False makes this context manager a no-op.</p></li>
<li><p><strong>use_cuda</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – Enables timing of CUDA events as well using the cudaEvent API.
Adds approximately 4us of overhead to each tensor operation.</p></li>
<li><p><strong>record_shapes</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – If shapes recording is set, information
about input dimensions will be collected. This allows one to see which
dimensions have been used under the hood and further group by them
using prof.key_averages(group_by_input_shape=True). Please note that
shape recording might skew your profiling data. It is recommended to
use separate runs with and without shape recording to validate the timing.
Most likely the skew will be negligible for bottom most events (in a case
of nested function calls). But for higher level functions the total
self cpu time might be artificially increased because of the shape
collection.</p></li>
<li><p><strong>with_flops</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – If with_flops is set, the profiler will estimate
the FLOPs (floating point operations) value using the operator’s input shape.
This allows one to estimate the hardware performance. Currently,
this option only works for the matrix multiplication and 2D convolution operators.</p></li>
<li><p><strong>profile_memory</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – track tensor memory allocation/deallocation.</p></li>
<li><p><strong>with_stack</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – record source information (file and line number) for the ops.</p></li>
<li><p><strong>with_modules</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a>) – record module hierarchy (including function names)
corresponding to the callstack of the op. e.g. If module A’s forward call’s
module B’s forward which contains an aten::add op,
then aten::add’s module hierarchy is A.B
Note that this support exist, at the moment, only for TorchScript models
and not eager mode models.</p></li>
<li><p><strong>use_kineto</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – experimental, enable profiling with Kineto profiler.</p></li>
<li><p><strong>use_cpu</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – profile CPU events; setting to <code class="docutils literal notranslate"><span class="pre">False</span></code> requires
<code class="docutils literal notranslate"><span class="pre">use_kineto=True</span></code> and can be used to lower the overhead for GPU-only profiling.</p></li>
<li><p><strong>experimental_config</strong> (<em>_ExperimentalConfig</em>) – A set of experimental options
used by profiler libraries like Kineto. Note, backward compatibility is not guaranteed.</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">((</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">profile</span><span class="p">()</span> <span class="k">as</span> <span class="n">prof</span><span class="p">:</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">):</span>  <span class="c1"># any normal python code, really!</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">y</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">y</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># NOTE: some columns were removed for brevity</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">prof</span><span class="o">.</span><span class="n">key_averages</span><span class="p">()</span><span class="o">.</span><span class="n">table</span><span class="p">(</span><span class="n">sort_by</span><span class="o">=</span><span class="s2">&quot;self_cpu_time_total&quot;</span><span class="p">))</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
<span class="go">Name                                 Self CPU total   CPU time avg     Number of Calls</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
<span class="go">mul                                  32.048ms         32.048ms         200</span>
<span class="go">pow                                  27.041ms         27.041ms         200</span>
<span class="go">PowBackward0                         9.727ms          55.483ms         100</span>
<span class="go">torch::autograd::AccumulateGrad      9.148ms          9.148ms          100</span>
<span class="go">torch::autograd::GraphRoot           691.816us        691.816us        100</span>
<span class="go">-----------------------------------  ---------------  ---------------  ---------------</span>
</pre></div>
</div>
</dd></dl>

<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.profiler.profile.export_chrome_trace.html#torch.autograd.profiler.profile.export_chrome_trace" title="torch.autograd.profiler.profile.export_chrome_trace"><code class="xref py py-obj docutils literal notranslate"><span class="pre">profiler.profile.export_chrome_trace</span></code></a></p></td>
<td><p>Exports an EventList as a Chrome tracing tools file.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.profiler.profile.key_averages.html#torch.autograd.profiler.profile.key_averages" title="torch.autograd.profiler.profile.key_averages"><code class="xref py py-obj docutils literal notranslate"><span class="pre">profiler.profile.key_averages</span></code></a></p></td>
<td><p>Averages all function events over their keys.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.profiler.profile.self_cpu_time_total.html#torch.autograd.profiler.profile.self_cpu_time_total" title="torch.autograd.profiler.profile.self_cpu_time_total"><code class="xref py py-obj docutils literal notranslate"><span class="pre">profiler.profile.self_cpu_time_total</span></code></a></p></td>
<td><p>Returns total time spent on CPU obtained as a sum of all self times across all the events.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.profiler.profile.total_average.html#torch.autograd.profiler.profile.total_average" title="torch.autograd.profiler.profile.total_average"><code class="xref py py-obj docutils literal notranslate"><span class="pre">profiler.profile.total_average</span></code></a></p></td>
<td><p>Averages all events.</p></td>
</tr>
</tbody>
</table>
<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.profiler.emit_nvtx">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.profiler.</span></span><span class="sig-name descname"><span class="pre">emit_nvtx</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">enabled</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">record_shapes</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#emit_nvtx"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.profiler.emit_nvtx" title="Permalink to this definition">¶</a></dt>
<dd><p>Context manager that makes every autograd operation emit an NVTX range.</p>
<p>It is useful when running the program under nvprof:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">nvprof</span> <span class="o">--</span><span class="n">profile</span><span class="o">-</span><span class="n">from</span><span class="o">-</span><span class="n">start</span> <span class="n">off</span> <span class="o">-</span><span class="n">o</span> <span class="n">trace_name</span><span class="o">.</span><span class="n">prof</span> <span class="o">--</span> <span class="o">&lt;</span><span class="n">regular</span> <span class="n">command</span> <span class="n">here</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>Unfortunately, there’s no way to force nvprof to flush the data it collected
to disk, so for CUDA profiling one has to use this context manager to annotate
nvprof traces and wait for the process to exit before inspecting them.
Then, either NVIDIA Visual Profiler (nvvp) can be used to visualize the timeline, or
<a class="reference internal" href="generated/torch.autograd.profiler.load_nvprof.html#torch.autograd.profiler.load_nvprof" title="torch.autograd.profiler.load_nvprof"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.autograd.profiler.load_nvprof()</span></code></a> can load the results for inspection
e.g. in Python REPL.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>enabled</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – Setting <code class="docutils literal notranslate"><span class="pre">enabled=False</span></code> makes this context manager a no-op.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
<li><p><strong>record_shapes</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">record_shapes=True</span></code>, the nvtx range wrapping
each autograd op will append information about the sizes of Tensor arguments received
by that op, in the following format:
<code class="docutils literal notranslate"><span class="pre">[[arg0.size(0),</span> <span class="pre">arg0.size(1),</span> <span class="pre">...],</span> <span class="pre">[arg1.size(0),</span> <span class="pre">arg1.size(1),</span> <span class="pre">...],</span> <span class="pre">...]</span></code>
Non-tensor arguments will be represented by <code class="docutils literal notranslate"><span class="pre">[]</span></code>.
Arguments will be listed in the order they are received by the backend op.
Please note that this order may not match the order in which those arguments were passed
on the Python side.  Also note that shape recording may increase the overhead of nvtx range creation.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">profile</span><span class="p">():</span>
<span class="gp">... </span>    <span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>  <span class="c1"># Warmup CUDA memory allocator and profiler</span>
<span class="gp">... </span>    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">emit_nvtx</span><span class="p">():</span>
<span class="gp">... </span>        <span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
</pre></div>
</div>
<p><strong>Forward-backward correlation</strong></p>
<p>When viewing a profile created using <a class="reference internal" href="#torch.autograd.profiler.emit_nvtx" title="torch.autograd.profiler.emit_nvtx"><code class="xref py py-class docutils literal notranslate"><span class="pre">emit_nvtx</span></code></a> in the Nvidia Visual Profiler,
correlating each backward-pass op with the corresponding forward-pass op can be difficult.
To ease this task, <a class="reference internal" href="#torch.autograd.profiler.emit_nvtx" title="torch.autograd.profiler.emit_nvtx"><code class="xref py py-class docutils literal notranslate"><span class="pre">emit_nvtx</span></code></a> appends sequence number information to the ranges it
generates.</p>
<p>During the forward pass, each function range is decorated with <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code>.  <code class="docutils literal notranslate"><span class="pre">seq</span></code> is a running
counter, incremented each time a new backward Function object is created and stashed for backward.
Thus, the <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code> annotation associated with each forward function range tells you that
if a backward Function object is created by this forward function,
the backward object will receive sequence number N.
During the backward pass, the top-level range wrapping each C++ backward Function’s
<code class="docutils literal notranslate"><span class="pre">apply()</span></code> call is decorated with <code class="docutils literal notranslate"><span class="pre">stashed</span> <span class="pre">seq=&lt;M&gt;</span></code>.  <code class="docutils literal notranslate"><span class="pre">M</span></code> is the sequence number that
the backward object was created with.  By comparing <code class="docutils literal notranslate"><span class="pre">stashed</span> <span class="pre">seq</span></code> numbers in backward with <code class="docutils literal notranslate"><span class="pre">seq</span></code>
numbers in forward, you can track down which forward op created each backward Function.</p>
<p>Any functions executed during the backward pass are also decorated with <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code>.  During
default backward (with <code class="docutils literal notranslate"><span class="pre">create_graph=False</span></code>) this information is irrelevant, and in fact,
<code class="docutils literal notranslate"><span class="pre">N</span></code> may simply be 0 for all such functions.  Only the top-level ranges associated with
backward Function objects’ <code class="docutils literal notranslate"><span class="pre">apply()</span></code> methods are useful, as a way to correlate these Function
objects with the earlier forward pass.</p>
<p><strong>Double-backward</strong></p>
<p>If, on the other hand, a backward pass with <code class="docutils literal notranslate"><span class="pre">create_graph=True</span></code> is underway (in other words,
if you are setting up for a double-backward), each function’s execution during backward
is given a nonzero, useful <code class="docutils literal notranslate"><span class="pre">seq=&lt;N&gt;</span></code>.  Those functions may themselves create Function objects
to be executed later during double-backward, just as the original functions in the forward pass did.
The relationship between backward and double-backward is conceptually the same as the relationship
between forward and backward: The functions still emit current-sequence-number-tagged ranges,
the Function objects they create still stash those sequence numbers, and during the eventual
double-backward, the Function objects’ <code class="docutils literal notranslate"><span class="pre">apply()</span></code> ranges are still tagged with <code class="docutils literal notranslate"><span class="pre">stashed</span> <span class="pre">seq</span></code>
numbers, which can be compared to <cite>seq</cite> numbers from the backward pass.</p>
</dd></dl>

<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.profiler.emit_itt">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.profiler.</span></span><span class="sig-name descname"><span class="pre">emit_itt</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">enabled</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">record_shapes</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/profiler.html#emit_itt"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.profiler.emit_itt" title="Permalink to this definition">¶</a></dt>
<dd><p>Context manager that makes every autograd operation emit an ITT range.</p>
<p>It is useful when running the program under Intel(R) VTune Profiler:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">vtune</span> <span class="o">&lt;--</span><span class="n">vtune</span><span class="o">-</span><span class="n">flags</span><span class="o">&gt;</span> <span class="o">&lt;</span><span class="n">regular</span> <span class="n">command</span> <span class="n">here</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>The Instrumentation and Tracing Technology (ITT) API enables your application to generate and
control the collection of trace data during its execution across different Intel tools.
This context manager is to annotate Intel(R) VTune Profiling trace. With help of this context manager,
you will be able to see labled ranges in Intel(R) VTune Profiler GUI.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>enabled</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – Setting <code class="docutils literal notranslate"><span class="pre">enabled=False</span></code> makes this context manager a no-op.
Default: <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
<li><p><strong>record_shapes</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a><em>, </em><em>optional</em>) – If <code class="docutils literal notranslate"><span class="pre">record_shapes=True</span></code>, the itt range wrapping
each autograd op will append information about the sizes of Tensor arguments received
by that op, in the following format:
<code class="docutils literal notranslate"><span class="pre">[[arg0.size(0),</span> <span class="pre">arg0.size(1),</span> <span class="pre">...],</span> <span class="pre">[arg1.size(0),</span> <span class="pre">arg1.size(1),</span> <span class="pre">...],</span> <span class="pre">...]</span></code>
Non-tensor arguments will be represented by <code class="docutils literal notranslate"><span class="pre">[]</span></code>.
Arguments will be listed in the order they are received by the backend op.
Please note that this order may not match the order in which those arguments were passed
on the Python side.  Also note that shape recording may increase the overhead of itt range creation.
Default: <code class="docutils literal notranslate"><span class="pre">False</span></code></p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">emit_itt</span><span class="p">():</span>
<span class="gp">... </span>    <span class="n">model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.profiler.load_nvprof.html#torch.autograd.profiler.load_nvprof" title="torch.autograd.profiler.load_nvprof"><code class="xref py py-obj docutils literal notranslate"><span class="pre">profiler.load_nvprof</span></code></a></p></td>
<td><p>Opens an nvprof trace file and parses autograd annotations.</p></td>
</tr>
</tbody>
</table>
</section>
<section id="anomaly-detection">
<h2>Anomaly detection<a class="headerlink" href="#anomaly-detection" title="Permalink to this heading">¶</a></h2>
<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.detect_anomaly">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.</span></span><span class="sig-name descname"><span class="pre">detect_anomaly</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">check_nan</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/anomaly_mode.html#detect_anomaly"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.detect_anomaly" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that enable anomaly detection for the autograd engine.</p>
<p>This does two things:</p>
<ul class="simple">
<li><p>Running the forward pass with detection enabled will allow the backward
pass to print the traceback of the forward operation that created the failing
backward function.</p></li>
<li><p>If <code class="docutils literal notranslate"><span class="pre">check_nan</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, any backward computation that generate “nan”
value will raise an error. Default <code class="docutils literal notranslate"><span class="pre">True</span></code>.</p></li>
</ul>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This mode should be enabled only for debugging as the different tests
will slow down your program execution.</p>
</div>
<p class="rubric">Example</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">torch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">autograd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">class</span> <span class="nc">MyFunc</span><span class="p">(</span><span class="n">autograd</span><span class="o">.</span><span class="n">Function</span><span class="p">):</span>
<span class="gp">... </span>    <span class="nd">@staticmethod</span>
<span class="gp">... </span>    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">inp</span><span class="p">):</span>
<span class="gp">... </span>        <span class="k">return</span> <span class="n">inp</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
<span class="gp">... </span>    <span class="nd">@staticmethod</span>
<span class="gp">... </span>    <span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="n">ctx</span><span class="p">,</span> <span class="n">gO</span><span class="p">):</span>
<span class="gp">... </span>        <span class="c1"># Error during the backward pass</span>
<span class="gp">... </span>        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Some error in backward&quot;</span><span class="p">)</span>
<span class="gp">... </span>        <span class="k">return</span> <span class="n">gO</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">run_fn</span><span class="p">(</span><span class="n">a</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">out</span> <span class="o">=</span> <span class="n">MyFunc</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">out</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span> <span class="o">=</span> <span class="n">run_fn</span><span class="p">(</span><span class="n">inp</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">out</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="go">    Traceback (most recent call last):</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 1, in &lt;module&gt;</span>
<span class="go">      File &quot;/your/pytorch/install/torch/_tensor.py&quot;, line 93, in backward</span>
<span class="go">        torch.autograd.backward(self, gradient, retain_graph, create_graph)</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/__init__.py&quot;, line 90, in backward</span>
<span class="go">        allow_unreachable=True)  # allow_unreachable flag</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/function.py&quot;, line 76, in apply</span>
<span class="go">        return self._forward_cls.backward(self, *args)</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 8, in backward</span>
<span class="go">    RuntimeError: Some error in backward</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">autograd</span><span class="o">.</span><span class="n">detect_anomaly</span><span class="p">():</span>
<span class="gp">... </span>    <span class="n">inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">... </span>    <span class="n">out</span> <span class="o">=</span> <span class="n">run_fn</span><span class="p">(</span><span class="n">inp</span><span class="p">)</span>
<span class="gp">... </span>    <span class="n">out</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="go">    Traceback of forward call that caused the error:</span>
<span class="go">      File &quot;tmp.py&quot;, line 53, in &lt;module&gt;</span>
<span class="go">        out = run_fn(inp)</span>
<span class="go">      File &quot;tmp.py&quot;, line 44, in run_fn</span>
<span class="go">        out = MyFunc.apply(a)</span>
<span class="go">    Traceback (most recent call last):</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 4, in &lt;module&gt;</span>
<span class="go">      File &quot;/your/pytorch/install/torch/_tensor.py&quot;, line 93, in backward</span>
<span class="go">        torch.autograd.backward(self, gradient, retain_graph, create_graph)</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/__init__.py&quot;, line 90, in backward</span>
<span class="go">        allow_unreachable=True)  # allow_unreachable flag</span>
<span class="go">      File &quot;/your/pytorch/install/torch/autograd/function.py&quot;, line 76, in apply</span>
<span class="go">        return self._forward_cls.backward(self, *args)</span>
<span class="go">      File &quot;&lt;stdin&gt;&quot;, line 8, in backward</span>
<span class="go">    RuntimeError: Some error in backward</span>
</pre></div>
</div>
<dl class="field-list simple">
</dl>
</dd></dl>

<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.set_detect_anomaly">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.</span></span><span class="sig-name descname"><span class="pre">set_detect_anomaly</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">mode</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">check_nan</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/anomaly_mode.html#set_detect_anomaly"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.set_detect_anomaly" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that sets the anomaly detection for the autograd engine on or off.</p>
<p><code class="docutils literal notranslate"><span class="pre">set_detect_anomaly</span></code> will enable or disable the autograd anomaly detection
based on its argument <code class="xref py py-attr docutils literal notranslate"><span class="pre">mode</span></code>.
It can be used as a context-manager or as a function.</p>
<p>See <code class="docutils literal notranslate"><span class="pre">detect_anomaly</span></code> above for details of the anomaly detection behaviour.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a>) – Flag whether to enable anomaly detection (<code class="docutils literal notranslate"><span class="pre">True</span></code>),
or disable (<code class="docutils literal notranslate"><span class="pre">False</span></code>).</p></li>
<li><p><strong>check_nan</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a>) – Flag whether to raise an error when the backward
generate “nan”</p></li>
</ul>
</dd>
</dl>
</dd></dl>

</section>
<section id="autograd-graph">
<h2>Autograd graph<a class="headerlink" href="#autograd-graph" title="Permalink to this heading">¶</a></h2>
<p>Autograd exposes methods that allow one to inspect the graph and interpose behavior during
the backward pass.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">grad_fn</span></code> attribute of a <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.Tensor</span></code></a> holds a  <code class="xref py py-class docutils literal notranslate"><span class="pre">torch.autograd.graph.Node</span></code>
if the tensor is the output of a operation that was recorded by autograd (i.e., grad_mode is
enabled and at least one of the inputs required gradients), or <code class="docutils literal notranslate"><span class="pre">None</span></code> otherwise.</p>
<table class="autosummary longtable docutils align-default">
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.graph.Node.name.html#torch.autograd.graph.Node.name" title="torch.autograd.graph.Node.name"><code class="xref py py-obj docutils literal notranslate"><span class="pre">graph.Node.name</span></code></a></p></td>
<td><p>Returns the name.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.graph.Node.metadata.html#torch.autograd.graph.Node.metadata" title="torch.autograd.graph.Node.metadata"><code class="xref py py-obj docutils literal notranslate"><span class="pre">graph.Node.metadata</span></code></a></p></td>
<td><p>Returns the metadata.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.graph.Node.next_functions.html#torch.autograd.graph.Node.next_functions" title="torch.autograd.graph.Node.next_functions"><code class="xref py py-obj docutils literal notranslate"><span class="pre">graph.Node.next_functions</span></code></a></p></td>
<td><p></p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/torch.autograd.graph.Node.register_hook.html#torch.autograd.graph.Node.register_hook" title="torch.autograd.graph.Node.register_hook"><code class="xref py py-obj docutils literal notranslate"><span class="pre">graph.Node.register_hook</span></code></a></p></td>
<td><p>Registers a backward hook.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/torch.autograd.graph.Node.register_prehook.html#torch.autograd.graph.Node.register_prehook" title="torch.autograd.graph.Node.register_prehook"><code class="xref py py-obj docutils literal notranslate"><span class="pre">graph.Node.register_prehook</span></code></a></p></td>
<td><p>Registers a backward pre-hook.</p></td>
</tr>
</tbody>
</table>
<p>Some operations need intermediary results to be saved during the forward pass
in order to execute the backward pass.
These intermediary results are saved as attributes on the <code class="docutils literal notranslate"><span class="pre">grad_fn</span></code> and can be accessed.
For example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">a</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.</span><span class="p">,</span> <span class="mf">0.</span><span class="p">,</span> <span class="mf">0.</span><span class="p">],</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span> <span class="o">=</span> <span class="n">a</span><span class="o">.</span><span class="n">exp</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="n">b</span><span class="o">.</span><span class="n">grad_fn</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">graph</span><span class="o">.</span><span class="n">Node</span><span class="p">))</span>
<span class="go">True</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="nb">dir</span><span class="p">(</span><span class="n">b</span><span class="o">.</span><span class="n">grad_fn</span><span class="p">))</span>
<span class="go">[&#39;__call__&#39;, &#39;__class__&#39;, &#39;__delattr__&#39;, &#39;__dir__&#39;, &#39;__doc__&#39;, &#39;__eq__&#39;, &#39;__format__&#39;, &#39;__ge__&#39;, &#39;__getattribute__&#39;, &#39;__gt__&#39;, &#39;__hash__&#39;, &#39;__init__&#39;, &#39;__init_subclass__&#39;, &#39;__le__&#39;, &#39;__lt__&#39;, &#39;__ne__&#39;, &#39;__new__&#39;, &#39;__reduce__&#39;, &#39;__reduce_ex__&#39;, &#39;__repr__&#39;, &#39;__setattr__&#39;, &#39;__sizeof__&#39;, &#39;__str__&#39;, &#39;__subclasshook__&#39;, &#39;_raw_saved_result&#39;, &#39;_register_hook_dict&#39;, &#39;_saved_result&#39;, &#39;metadata&#39;, &#39;name&#39;, &#39;next_functions&#39;, &#39;register_hook&#39;, &#39;register_prehook&#39;, &#39;requires_grad&#39;]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">allclose</span><span class="p">(</span><span class="n">b</span><span class="o">.</span><span class="n">grad_fn</span><span class="o">.</span><span class="n">_saved_result</span><span class="p">,</span> <span class="n">b</span><span class="p">))</span>
<span class="go">True</span>
</pre></div>
</div>
<p>You can also define how these saved tensors should be packed / unpacked using hooks.
A common application is to trade compute for memory by saving those intermediary results
to disk or to CPU instead of leaving them on the GPU. This is especially useful if you
notice your model fits on GPU during evaluation, but not training.
Also see <a class="reference internal" href="notes/autograd.html#saved-tensors-hooks-doc"><span class="std std-ref">Hooks for saved tensors</span></a>.</p>
<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.graph.saved_tensors_hooks">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.graph.</span></span><span class="sig-name descname"><span class="pre">saved_tensors_hooks</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">pack_hook</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">unpack_hook</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/graph.html#saved_tensors_hooks"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.graph.saved_tensors_hooks" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that sets a pair of pack / unpack hooks for saved tensors.</p>
<p>Use this context-manager to define how intermediary results of an operation
should be packed before saving, and unpacked on retrieval.</p>
<p>In that context, the <code class="docutils literal notranslate"><span class="pre">pack_hook</span></code> function will be called everytime an
operation saves a tensor for backward (this includes intermediary results
saved using
<code class="xref py py-func docutils literal notranslate"><span class="pre">save_for_backward()</span></code> but
also those recorded by a PyTorch-defined operation). The output of
<code class="docutils literal notranslate"><span class="pre">pack_hook</span></code> is then stored in the computation graph instead of the
original tensor.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">unpack_hook</span></code> is called when the saved tensor needs to be accessed,
namely when executing <a class="reference internal" href="generated/torch.Tensor.backward.html#torch.Tensor.backward" title="torch.Tensor.backward"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.Tensor.backward()</span></code></a> or
<a class="reference internal" href="generated/torch.autograd.grad.html#torch.autograd.grad" title="torch.autograd.grad"><code class="xref py py-func docutils literal notranslate"><span class="pre">torch.autograd.grad()</span></code></a>. It takes as argument the <em>packed</em> object
returned by <code class="docutils literal notranslate"><span class="pre">pack_hook</span></code> and should return a tensor which has the same
content as the original tensor (passed as input to the corresponding
<code class="docutils literal notranslate"><span class="pre">pack_hook</span></code>).</p>
<p>The hooks should have the following signatures:</p>
<blockquote>
<div><p>pack_hook(tensor: Tensor) -&gt; Any</p>
<p>unpack_hook(Any) -&gt; Tensor</p>
</div></blockquote>
<p>where the return value of <code class="docutils literal notranslate"><span class="pre">pack_hook</span></code> is a valid input to <code class="docutils literal notranslate"><span class="pre">unpack_hook</span></code>.</p>
<p>In general, you want <code class="docutils literal notranslate"><span class="pre">unpack_hook(pack_hook(t))</span></code> to be equal to <code class="docutils literal notranslate"><span class="pre">t</span></code> in terms
of value, size, dtype and device.</p>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">pack_hook</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Packing&quot;</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">x</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">unpack_hook</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Unpacking&quot;</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">x</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">a</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">*</span> <span class="mi">2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">graph</span><span class="o">.</span><span class="n">saved_tensors_hooks</span><span class="p">(</span><span class="n">pack_hook</span><span class="p">,</span> <span class="n">unpack_hook</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">y</span> <span class="o">=</span> <span class="n">a</span> <span class="o">*</span> <span class="n">b</span>
<span class="go">Packing tensor([1., 1., 1., 1., 1.], requires_grad=True)</span>
<span class="go">Packing tensor([2., 2., 2., 2., 2.], grad_fn=&lt;MulBackward0&gt;)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="go">Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True)</span>
<span class="go">Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=&lt;MulBackward0&gt;)</span>
</pre></div>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Performing an inplace operation on the input to either hooks may lead
to undefined behavior.</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Only one pair of hooks is allowed at a time. When recursively nesting this
context-manager, only the inner-most pair of hooks will be applied.</p>
</div>
<dl class="field-list simple">
</dl>
</dd></dl>

<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.graph.save_on_cpu">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.graph.</span></span><span class="sig-name descname"><span class="pre">save_on_cpu</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">pin_memory</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/graph.html#save_on_cpu"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.graph.save_on_cpu" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager under which tensors saved by the forward pass will be
stored on cpu, then retrieved for backward.</p>
<p>When performing operations within this context manager, intermediary
results saved in the graph during the forward pass will be moved to CPU,
then copied back to the original device when needed for the backward pass.
If the graph was already on CPU, no tensor copy is performed.</p>
<p>Use this context-manager to trade compute for GPU memory usage (e.g.
when your model doesn’t fit in GPU memory during training).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>pin_memory</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.11)"><em>bool</em></a>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code> tensors will be saved to CPU pinned memory
during packing and copied to GPU asynchronously during unpacking.
Defaults to <code class="docutils literal notranslate"><span class="pre">False</span></code>.
Also see <a class="reference internal" href="notes/cuda.html#cuda-memory-pinning"><span class="std std-ref">Use pinned memory buffers</span></a>.</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">a</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s2">&quot;cuda&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">b</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s2">&quot;cuda&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">c</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s2">&quot;cuda&quot;</span><span class="p">)</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">f</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">c</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">prod_1</span> <span class="o">=</span> <span class="n">a</span> <span class="o">*</span> <span class="n">b</span>           <span class="c1"># a and b are saved on GPU</span>
<span class="gp">... </span>    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">graph</span><span class="o">.</span><span class="n">save_on_cpu</span><span class="p">():</span>
<span class="gp">... </span>        <span class="n">prod_2</span> <span class="o">=</span> <span class="n">prod_1</span> <span class="o">*</span> <span class="n">c</span>  <span class="c1"># prod_1 and c are saved on CPU</span>
<span class="gp">... </span>    <span class="n">y</span> <span class="o">=</span> <span class="n">prod_2</span> <span class="o">*</span> <span class="n">a</span>           <span class="c1"># prod_2 and a are saved on GPU</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">y</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="n">f</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">c</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">del</span> <span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">c</span>  <span class="c1"># for illustration only</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># the content of a, b, and prod_2 are still alive on GPU</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># the content of prod_1 and c only live on CPU</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>  <span class="c1"># all CPU tensors are moved back to GPU, for backward</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># all intermediary tensors are released (deleted) after the call to backward</span>
</pre></div>
</div>
</dd></dl>

<dl class="py class">
<dt class="sig sig-object py" id="torch.autograd.graph.disable_saved_tensors_hooks">
<em class="property"><span class="pre">class</span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">torch.autograd.graph.</span></span><span class="sig-name descname"><span class="pre">disable_saved_tensors_hooks</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">error_message</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/autograd/graph.html#disable_saved_tensors_hooks"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#torch.autograd.graph.disable_saved_tensors_hooks" title="Permalink to this definition">¶</a></dt>
<dd><p>Context-manager that disables the saved tensors default hooks feature.</p>
<p>Useful for if you are creating a feature that does not work with saved
tensors default hooks.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>error_message</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.11)"><em>str</em></a>) – When saved tensors default hooks are used when they
have been are disabled, a RuntimeError with this
error message gets raised.</p>
</dd>
</dl>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">message</span> <span class="o">=</span> <span class="s2">&quot;saved tensors default hooks are disabled&quot;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">graph</span><span class="o">.</span><span class="n">disable_saved_tensors_hooks</span><span class="p">(</span><span class="n">message</span><span class="p">):</span>
<span class="gp">... </span>    <span class="c1"># Raises RuntimeError: saved tensors default hooks are disabled</span>
<span class="gp">... </span>    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">graph</span><span class="o">.</span><span class="n">save_on_cpu</span><span class="p">():</span>
<span class="gp">... </span>        <span class="k">pass</span>
</pre></div>
</div>
</dd></dl>

</section>
</section>


             </article>
             
            </div>
            <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="generated/torch.autograd.backward.html" class="btn btn-neutral float-right" title="torch.autograd.backward" accesskey="n" rel="next">Next <img src="_static/images/chevron-right-orange.svg" class="next-page"></a>
      
      
        <a href="amp.html" class="btn btn-neutral" title="Automatic Mixed Precision package - torch.amp" accesskey="p" rel="prev"><img src="_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
      
    </div>
  

    <hr>

  
  <div role="contentinfo">
    <p>
        &copy; Copyright 2023, PyTorch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
<script>

var match = window.location.href.match(/\/_[a-zA-Z0-9_]*.html|_dynamo/gi);
var url = window.location.href.lastIndexOf(match[match.length-1]);

if (url)
  {
    var div = '<div class="admonition note"><p class="admonition-title">Note</p><p><i class="fa fa-exclamation-circle" aria-hidden="true">&nbsp</i> This page describes an internal API which is not intended to be used outside of the PyTorch codebase and can be modified or removed without notice.</p></div>'
    document.getElementById("pytorch-article").insertAdjacentHTML('afterBegin', div)
  }
</script>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Automatic differentiation package - torch.autograd</a><ul>
<li><a class="reference internal" href="#forward-mode-automatic-differentiation">Forward-mode Automatic Differentiation</a></li>
<li><a class="reference internal" href="#functional-higher-level-api">Functional higher level API</a></li>
<li><a class="reference internal" href="#locally-disabling-gradient-computation">Locally disabling gradient computation</a></li>
<li><a class="reference internal" href="#default-gradient-layouts">Default gradient layouts</a><ul>
<li><a class="reference internal" href="#manual-gradient-layouts">Manual gradient layouts</a></li>
</ul>
</li>
<li><a class="reference internal" href="#in-place-operations-on-tensors">In-place operations on Tensors</a><ul>
<li><a class="reference internal" href="#in-place-correctness-checks">In-place correctness checks</a></li>
</ul>
</li>
<li><a class="reference internal" href="#variable-deprecated">Variable (deprecated)</a></li>
<li><a class="reference internal" href="#tensor-autograd-functions">Tensor autograd functions</a></li>
<li><a class="reference internal" href="#function"><span class="hidden-section">Function</span></a></li>
<li><a class="reference internal" href="#context-method-mixins">Context method mixins</a></li>
<li><a class="reference internal" href="#numerical-gradient-checking">Numerical gradient checking</a></li>
<li><a class="reference internal" href="#profiler">Profiler</a><ul>
</ul>
</li>
<li><a class="reference internal" href="#anomaly-detection">Anomaly detection</a></li>
<li><a class="reference internal" href="#autograd-graph">Autograd graph</a></li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  
       <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
         <script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
         <script src="_static/jquery.js"></script>
         <script src="_static/underscore.js"></script>
         <script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
         <script src="_static/doctools.js"></script>
         <script src="_static/clipboard.min.js"></script>
         <script src="_static/copybutton.js"></script>
     

  <script type="text/javascript" src="_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script script type="text/javascript">
  var collapsedSections = ['Developer Notes', 'Language Bindings', 'Libraries', 'Community'];
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">Stay up to date</li>
            <li><a href="https://www.facebook.com/pytorch" target="_blank">Facebook</a></li>
            <li><a href="https://twitter.com/pytorch" target="_blank">Twitter</a></li>
            <li><a href="https://www.youtube.com/pytorch" target="_blank">YouTube</a></li>
            <li><a href="https://www.linkedin.com/company/pytorch" target="_blank">LinkedIn</a></li>
          </ul>  
          </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">PyTorch Podcasts</li>
            <li><a href="https://open.spotify.com/show/6UzHKeiy368jKfQMKKvJY5" target="_blank">Spotify</a></li>
            <li><a href="https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008" target="_blank">Apple</a></li>
            <li><a href="https://www.google.com/podcasts?feed=aHR0cHM6Ly9mZWVkcy5zaW1wbGVjYXN0LmNvbS9PQjVGa0lsOA%3D%3D" target="_blank">Google</a></li>
            <li><a href="https://music.amazon.com/podcasts/7a4e6f0e-26c2-49e9-a478-41bd244197d0/PyTorch-Developer-Podcast?" target="_blank">Amazon</a></li>
          </ul>
         </div>
        </div>
        
        <div class="privacy-policy">
          <ul>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/terms/" target="_blank">Terms</a></li>
            <li class="privacy-policy-links">|</li>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/privacy-policy/" target="_blank">Privacy</a></li>
          </ul>
        </div>
        <div class="copyright">
        <p>© Copyright The Linux Foundation. The PyTorch Foundation is a project of The Linux Foundation.
          For web site terms of use, trademark policy and other policies applicable to The PyTorch Foundation please see
          <a href="www.linuxfoundation.org/policies/">www.linuxfoundation.org/policies/</a>. The PyTorch Foundation supports the PyTorch open source
          project, which has been established as PyTorch Project a Series of LF Projects, LLC. For policies applicable to the PyTorch Project a Series of LF Projects, LLC,
          please see <a href="www.lfprojects.org/policies/">www.lfprojects.org/policies/</a>.</p>
      </div>
     </div>

  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>
            
          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="resources-mobile-menu-title" class="active">
            Docs
          </li>

          <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/docs/stable/index.html">PyTorch</a>
            </li>

            <li>
              <a href="https://pytorch.org/audio/stable/index.html">torchaudio</a>
            </li>

            <li>
              <a href="https://pytorch.org/text/stable/index.html">torchtext</a>
            </li>

            <li>
              <a href="https://pytorch.org/vision/stable/index.html">torchvision</a>
            </li>

            <li>
              <a href="https://pytorch.org/torcharrow">torcharrow</a>
            </li>

            <li>
              <a href="https://pytorch.org/data">TorchData</a>
            </li>

            <li>
              <a href="https://pytorch.org/torchrec">TorchRec</a>
            </li>

            <li>
              <a href="https://pytorch.org/serve/">TorchServe</a>
            </li>

            <li>
              <a href="https://pytorch.org/torchx/">TorchX</a>
            </li>

            <li>
              <a href="https://pytorch.org/xla">PyTorch on XLA Devices</a>
            </li>
          </ul>

          <li class="resources-mobile-menu-title">
            Resources
          </li>
            
           <ul class="resources-mobile-menu-items">

            <li>
              <a href="https://pytorch.org/features">About</a>
            </li>

            <li>
              <a href="https://pytorch.org/foundation">PyTorch Foundation</a>
            </li>

            <li>
              <a href="https://pytorch.org/#community-module">Community</a>
            </li>

            <li>
              <a href="https://pytorch.org/community-stories">Community Stories</a>
            </li>

            <li>
              <a href="https://pytorch.org/resources">Developer Resources</a>
            </li>

            <li>
              <a href="https://pytorch.org/events">Events</a>
            </li>

            <li>
              <a href="https://discuss.pytorch.org/">Forums</a>
            </li>

            <li>
              <a href="https://pytorch.org/hub">Models (Beta)</a>
            </li>
          </ul>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>