1.4.0/multiprocessing.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta name="robots" content="noindex">
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Multiprocessing package - torch.multiprocessing &mdash; PyTorch master documentation</title>
  

    <link rel="canonical" href="https://pytorch.org/docs/stable/multiprocessing.html"/>
  

  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" /> 

  
  <script src="_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   
    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

                <div class="version">
                  <a href='http://pytorch.org/docs/versions.html'>1.4.0 &#x25BC</a>
                </div>
              
            
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/distributed_autograd.html">Distributed Autograd Design</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/rref.html">Remote Reference Protocol</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/windows.html">Windows FAQ</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/javadoc/">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="rpc.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">torchvision Reference</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="torchvision/index.html">torchvision</a></li>
</ul>
<p class="caption"><span class="caption-text">torchaudio Reference</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
</ul>
<p class="caption"><span class="caption-text">torchtext Reference</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          

<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
      <li>Multiprocessing package - torch.multiprocessing</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="_sources/multiprocessing.rst.txt" rel="nofollow"><img src="_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <div class="section" id="module-torch.multiprocessing">
<span id="multiprocessing-package-torch-multiprocessing"></span><span id="multiprocessing-doc"></span><h1>Multiprocessing package - torch.multiprocessing<a class="headerlink" href="#module-torch.multiprocessing" title="Permalink to this headline">¶</a></h1>
<p>torch.multiprocessing is a wrapper around the native <code class="xref py py-mod docutils literal notranslate"><span class="pre">multiprocessing</span></code>
module. It registers custom reducers, that use shared memory to provide shared
views on the same data in different processes. Once the tensor/storage is moved
to shared_memory (see <a class="reference internal" href="tensors.html#torch.Tensor.share_memory_" title="torch.Tensor.share_memory_"><code class="xref py py-func docutils literal notranslate"><span class="pre">share_memory_()</span></code></a>), it will be possible
to send it to other processes without making any copies.</p>
<p>The API is 100% compatible with the original module - it’s enough to change
<code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">multiprocessing</span></code> to <code class="docutils literal notranslate"><span class="pre">import</span> <span class="pre">torch.multiprocessing</span></code> to have all the
tensors sent through the queues or shared via other mechanisms, moved to shared
memory.</p>
<p>Because of the similarity of APIs we do not document most of this package
contents, and we recommend referring to very good docs of the original module.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>If the main process exits abruptly (e.g. because of an incoming signal),
Python’s <code class="docutils literal notranslate"><span class="pre">multiprocessing</span></code> sometimes fails to clean up its children.
It’s a known caveat, so if you’re seeing any resource leaks after
interrupting the interpreter, it probably means that this has just happened
to you.</p>
</div>
<div class="section" id="strategy-management">
<h2>Strategy management<a class="headerlink" href="#strategy-management" title="Permalink to this headline">¶</a></h2>
<dl class="function">
<dt id="torch.multiprocessing.get_all_sharing_strategies">
<code class="sig-prename descclassname">torch.multiprocessing.</code><code class="sig-name descname">get_all_sharing_strategies</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/multiprocessing.html#get_all_sharing_strategies"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.multiprocessing.get_all_sharing_strategies" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns a set of sharing strategies supported on a current system.</p>
</dd></dl>

<dl class="function">
<dt id="torch.multiprocessing.get_sharing_strategy">
<code class="sig-prename descclassname">torch.multiprocessing.</code><code class="sig-name descname">get_sharing_strategy</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/multiprocessing.html#get_sharing_strategy"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.multiprocessing.get_sharing_strategy" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the current strategy for sharing CPU tensors.</p>
</dd></dl>

<dl class="function">
<dt id="torch.multiprocessing.set_sharing_strategy">
<code class="sig-prename descclassname">torch.multiprocessing.</code><code class="sig-name descname">set_sharing_strategy</code><span class="sig-paren">(</span><em class="sig-param">new_strategy</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/multiprocessing.html#set_sharing_strategy"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.multiprocessing.set_sharing_strategy" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets the strategy for sharing CPU tensors.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>new_strategy</strong> (<em>str</em>) – Name of the selected strategy. Should be one of
the values returned by <a class="reference internal" href="#torch.multiprocessing.get_all_sharing_strategies" title="torch.multiprocessing.get_all_sharing_strategies"><code class="xref py py-func docutils literal notranslate"><span class="pre">get_all_sharing_strategies()</span></code></a>.</p>
</dd>
</dl>
</dd></dl>

</div>
<div class="section" id="sharing-cuda-tensors">
<span id="multiprocessing-cuda-sharing-details"></span><h2>Sharing CUDA tensors<a class="headerlink" href="#sharing-cuda-tensors" title="Permalink to this headline">¶</a></h2>
<p>Sharing CUDA tensors between processes is supported only in Python 3, using
a <code class="docutils literal notranslate"><span class="pre">spawn</span></code> or <code class="docutils literal notranslate"><span class="pre">forkserver</span></code> start methods. <code class="xref py py-mod docutils literal notranslate"><span class="pre">python:multiprocessing</span></code> in
Python 2 can only create subprocesses using <code class="docutils literal notranslate"><span class="pre">fork</span></code>, and it’s not supported
by the CUDA runtime.</p>
<p>Unlike CPU tensors, the sending process is required to keep the original tensor
as long as the receiving process retains a copy of the tensor. The refcounting is
implemented under the hood but requires users to follow the next best practices.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>If the consumer process dies abnormally to a fatal signal, the shared tensor
could be forever kept in memory as long as the sending process is running.</p>
</div>
<ol class="arabic simple">
<li><p>Release memory ASAP in the consumer.</p></li>
</ol>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1">## Good</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<span class="c1"># do somethings with x</span>
<span class="k">del</span> <span class="n">x</span>
</pre></div>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1">## Bad</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<span class="c1"># do somethings with x</span>
<span class="c1"># do everything else (producer have to keep x in memory)</span>
</pre></div>
</div>
<p>2. Keep producer process running until all consumers exits. This will prevent
the situation when the producer process releasing memory which is still in use
by the consumer.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1">## producer</span>
<span class="c1"># send tensors, do something</span>
<span class="n">event</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
</pre></div>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1">## consumer</span>
<span class="c1"># receive tensors and use them</span>
<span class="n">event</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
</pre></div>
</div>
<ol class="arabic simple" start="3">
<li><p>Don’t pass received tensors.</p></li>
</ol>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># not going to work</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<span class="n">queue_2</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
</pre></div>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># you need to create a process-local copy</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<span class="n">x_clone</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">clone</span><span class="p">()</span>
<span class="n">queue_2</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">x_clone</span><span class="p">)</span>
</pre></div>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># putting and getting from the same queue in the same process will likely end up with segfault</span>
<span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">tensor</span><span class="p">)</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="sharing-strategies">
<h2>Sharing strategies<a class="headerlink" href="#sharing-strategies" title="Permalink to this headline">¶</a></h2>
<p>This section provides a brief overview into how different sharing strategies
work. Note that it applies only to CPU tensor - CUDA tensors will always use
the CUDA API, as that’s the only way they can be shared.</p>
<div class="section" id="file-descriptor-file-descriptor">
<h3>File descriptor - <code class="docutils literal notranslate"><span class="pre">file_descriptor</span></code><a class="headerlink" href="#file-descriptor-file-descriptor" title="Permalink to this headline">¶</a></h3>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This is the default strategy (except for macOS and OS X where it’s not
supported).</p>
</div>
<p>This strategy will use file descriptors as shared memory handles. Whenever a
storage is moved to shared memory, a file descriptor obtained from <code class="docutils literal notranslate"><span class="pre">shm_open</span></code>
is cached with the object, and when it’s going to be sent to other processes,
the file descriptor will be transferred (e.g. via UNIX sockets) to it. The
receiver will also cache the file descriptor and <code class="docutils literal notranslate"><span class="pre">mmap</span></code> it, to obtain a shared
view onto the storage data.</p>
<p>Note that if there will be a lot of tensors shared, this strategy will keep a
large number of file descriptors open most of the time. If your system has low
limits for the number of open file descriptors, and you can’t raise them, you
should use the <code class="docutils literal notranslate"><span class="pre">file_system</span></code> strategy.</p>
</div>
<div class="section" id="file-system-file-system">
<h3>File system - <code class="docutils literal notranslate"><span class="pre">file_system</span></code><a class="headerlink" href="#file-system-file-system" title="Permalink to this headline">¶</a></h3>
<p>This strategy will use file names given to <code class="docutils literal notranslate"><span class="pre">shm_open</span></code> to identify the shared
memory regions. This has a benefit of not requiring the implementation to cache
the file descriptors obtained from it, but at the same time is prone to shared
memory leaks. The file can’t be deleted right after its creation, because other
processes need to access it to open their views. If the processes fatally
crash, or are killed, and don’t call the storage destructors, the files will
remain in the system. This is very serious, because they keep using up the
memory until the system is restarted, or they’re freed manually.</p>
<p>To counter the problem of shared memory file leaks, <a class="reference internal" href="#module-torch.multiprocessing" title="torch.multiprocessing"><code class="xref py py-mod docutils literal notranslate"><span class="pre">torch.multiprocessing</span></code></a>
will spawn a daemon named <code class="docutils literal notranslate"><span class="pre">torch_shm_manager</span></code> that will isolate itself from
the current process group, and will keep track of all shared memory allocations.
Once all processes connected to it exit, it will wait a moment to ensure there
will be no new connections, and will iterate over all shared memory files
allocated by the group. If it finds that any of them still exist, they will be
deallocated. We’ve tested this method and it proved to be robust to various
failures. Still, if your system has high enough limits, and <code class="docutils literal notranslate"><span class="pre">file_descriptor</span></code>
is a supported strategy, we do not recommend switching to this one.</p>
</div>
</div>
<div class="section" id="spawning-subprocesses">
<h2>Spawning subprocesses<a class="headerlink" href="#spawning-subprocesses" title="Permalink to this headline">¶</a></h2>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Available for Python &gt;= 3.4.</p>
<p>This depends on the <code class="docutils literal notranslate"><span class="pre">spawn</span></code> start method in Python’s
<code class="docutils literal notranslate"><span class="pre">multiprocessing</span></code> package.</p>
</div>
<p>Spawning a number of subprocesses to perform some function can be done
by creating <code class="docutils literal notranslate"><span class="pre">Process</span></code> instances and calling <code class="docutils literal notranslate"><span class="pre">join</span></code> to wait for
their completion. This approach works fine when dealing with a single
subprocess but presents potential issues when dealing with multiple
processes.</p>
<p>Namely, joining processes sequentially implies they will terminate
sequentially. If they don’t, and the first process does not terminate,
the process termination will go unnoticed. Also, there are no native
facilities for error propagation.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">spawn</span></code> function below addresses these concerns and takes care
of error propagation, out of order termination, and will actively
terminate processes upon detecting an error in one of them.</p>
<dl class="function">
<dt id="torch.multiprocessing.spawn">
<code class="sig-prename descclassname">torch.multiprocessing.</code><code class="sig-name descname">spawn</code><span class="sig-paren">(</span><em class="sig-param">fn</em>, <em class="sig-param">args=()</em>, <em class="sig-param">nprocs=1</em>, <em class="sig-param">join=True</em>, <em class="sig-param">daemon=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/multiprocessing/spawn.html#spawn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.multiprocessing.spawn" title="Permalink to this definition">¶</a></dt>
<dd><p>Spawns <code class="docutils literal notranslate"><span class="pre">nprocs</span></code> processes that run <code class="docutils literal notranslate"><span class="pre">fn</span></code> with <code class="docutils literal notranslate"><span class="pre">args</span></code>.</p>
<p>If one of the processes exits with a non-zero exit status, the
remaining processes are killed and an exception is raised with the
cause of termination. In the case an exception was caught in the
child process, it is forwarded and its traceback is included in
the exception raised in the parent process.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>fn</strong> (<em>function</em>) – <p>Function is called as the entrypoint of the
spawned process. This function must be defined at the top
level of a module so it can be pickled and spawned. This
is a requirement imposed by multiprocessing.</p>
<p>The function is called as <code class="docutils literal notranslate"><span class="pre">fn(i,</span> <span class="pre">*args)</span></code>, where <code class="docutils literal notranslate"><span class="pre">i</span></code> is
the process index and <code class="docutils literal notranslate"><span class="pre">args</span></code> is the passed through tuple
of arguments.</p>
</p></li>
<li><p><strong>args</strong> (<em>tuple</em>) – Arguments passed to <code class="docutils literal notranslate"><span class="pre">fn</span></code>.</p></li>
<li><p><strong>nprocs</strong> (<em>python:int</em>) – Number of processes to spawn.</p></li>
<li><p><strong>join</strong> (<em>bool</em>) – Perform a blocking join on all processes.</p></li>
<li><p><strong>daemon</strong> (<em>bool</em>) – The spawned processes’ daemon flag. If set to True,
daemonic processes will be created.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>None if <code class="docutils literal notranslate"><span class="pre">join</span></code> is <code class="docutils literal notranslate"><span class="pre">True</span></code>,
<a class="reference internal" href="#torch.multiprocessing.SpawnContext" title="torch.multiprocessing.SpawnContext"><code class="xref py py-class docutils literal notranslate"><span class="pre">SpawnContext</span></code></a> if <code class="docutils literal notranslate"><span class="pre">join</span></code> is <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
</dd>
</dl>
</dd></dl>

<dl class="class">
<dt id="torch.multiprocessing.SpawnContext">
<em class="property">class </em><code class="sig-prename descclassname">torch.multiprocessing.</code><code class="sig-name descname">SpawnContext</code><a class="reference internal" href="_modules/torch/multiprocessing/spawn.html#SpawnContext"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.multiprocessing.SpawnContext" title="Permalink to this definition">¶</a></dt>
<dd><p>Returned by <a class="reference internal" href="#torch.multiprocessing.spawn" title="torch.multiprocessing.spawn"><code class="xref py py-func docutils literal notranslate"><span class="pre">spawn()</span></code></a> when called with <code class="docutils literal notranslate"><span class="pre">join=False</span></code>.</p>
<dl class="method">
<dt id="torch.multiprocessing.SpawnContext.join">
<code class="sig-name descname">join</code><span class="sig-paren">(</span><em class="sig-param">timeout=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/multiprocessing/spawn.html#SpawnContext.join"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.multiprocessing.SpawnContext.join" title="Permalink to this definition">¶</a></dt>
<dd><p>Tries to join one or more processes in this spawn context.
If one of them exited with a non-zero exit status, this function
kills the remaining processes and raises an exception with the cause
of the first process exiting.</p>
<p>Returns <code class="docutils literal notranslate"><span class="pre">True</span></code> if all processes have been joined successfully,
<code class="docutils literal notranslate"><span class="pre">False</span></code> if there are more processes that need to be joined.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>timeout</strong> (<em>python:float</em>) – Wait this long before giving up on waiting.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
</div>


             </article>
             
            </div>
            <footer>
  

    <hr>

  
  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Multiprocessing package - torch.multiprocessing</a><ul>
<li><a class="reference internal" href="#strategy-management">Strategy management</a></li>
<li><a class="reference internal" href="#sharing-cuda-tensors">Sharing CUDA tensors</a></li>
<li><a class="reference internal" href="#sharing-strategies">Sharing strategies</a><ul>
<li><a class="reference internal" href="#file-descriptor-file-descriptor">File descriptor - <code class="docutils literal notranslate"><span class="pre">file_descriptor</span></code></a></li>
<li><a class="reference internal" href="#file-system-file-system">File system - <code class="docutils literal notranslate"><span class="pre">file_system</span></code></a></li>
</ul>
</li>
<li><a class="reference internal" href="#spawning-subprocesses">Spawning subprocesses</a></li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  
       <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
         <script src="_static/jquery.js"></script>
         <script src="_static/underscore.js"></script>
         <script src="_static/doctools.js"></script>
         <script src="_static/language_data.js"></script>
     

  <script type="text/javascript" src="_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="_static/js/vendor/bootstrap.min.js"></script>
  <script type="text/javascript" src="_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://pytorch.org/resources">Resources</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/support">Support</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.slack.com" target="_blank">Slack</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md" target="_blank">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Follow Us</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>