docs/intermediate/mario_rl_tutorial.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Train a Mario-playing RL Agent &mdash; PyTorch Tutorials 1.10.2+cu102 documentation</title>
  

  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="../_static/copybutton.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="Flask를 사용하여 Python에서 PyTorch를 REST API로 배포하기" href="flask_rest_api_tutorial.html" />
    <link rel="prev" title="강화 학습 (DQN) 튜토리얼" href="reinforcement_q_learning.html" /> 

  
  <script src="../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.kr/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.kr/get-started">시작하기</a>
          </li>

          <li class="active">
            <a href="https://tutorials.pytorch.kr">튜토리얼</a>
          </li>

          <li>
            <a href="https://pytorch.kr/hub">허브</a>
          </li>

          <li>
            <a href="https://discuss.pytorch.kr">커뮤니티</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">

   
    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

                <div class="version">
                  1.10.2+cu102
                </div>
              
            
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search Tutorials" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
              <p class="caption"><span class="caption-text">파이토치(PyTorch) 레시피</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../recipes/recipes_index.html">모든 레시피 보기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../prototype/prototype_index.html">모든 프로토타입 레시피 보기</a></li>
</ul>
<p class="caption"><span class="caption-text">파이토치(PyTorch) 시작하기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/intro.html">파이토치(PyTorch) 기본 익히기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/quickstart_tutorial.html">빠른 시작(Quickstart)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/tensorqs_tutorial.html">텐서(Tensor)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/data_tutorial.html">Dataset과 DataLoader</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/transforms_tutorial.html">변형(Transform)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/buildmodel_tutorial.html">신경망 모델 구성하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/autogradqs_tutorial.html"><code class="docutils literal notranslate"><span class="pre">torch.autograd</span></code>를 사용한 자동 미분</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/optimization_tutorial.html">모델 매개변수 최적화하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/basics/saveloadrun_tutorial.html">모델 저장하고 불러오기</a></li>
</ul>
<p class="caption"><span class="caption-text">Introduction to PyTorch on YouTube</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt.html">Introduction to PyTorch - YouTube Series</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt/introyt1_tutorial.html">Introduction to PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt/tensors_deeper_tutorial.html">Introduction to PyTorch Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt/autogradyt_tutorial.html">The Fundamentals of Autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt/modelsyt_tutorial.html">Building Models with PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt/tensorboardyt_tutorial.html">PyTorch TensorBoard Support</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt/trainingyt.html">Training with PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/introyt/captumyt.html">Model Understanding with Captum</a></li>
</ul>
<p class="caption"><span class="caption-text">파이토치(PyTorch) 배우기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/deep_learning_60min_blitz.html">PyTorch로 딥러닝하기: 60분만에 끝장내기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/pytorch_with_examples.html">예제로 배우는 파이토치(PyTorch)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/nn_tutorial.html"><cite>torch.nn</cite> 이 <em>실제로</em> 무엇인가요?</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensorboard_tutorial.html">TensorBoard로 모델, 데이터, 학습 시각화하기</a></li>
</ul>
<p class="caption"><span class="caption-text">이미지/비디오</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="torchvision_tutorial.html">TorchVision 객체 검출 미세조정(Finetuning) 튜토리얼</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/transfer_learning_tutorial.html">컴퓨터 비전(Vision)을 위한 전이학습(Transfer Learning)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/fgsm_tutorial.html">적대적 예제 생성(Adversarial Example Generation)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/dcgan_faces_tutorial.html">DCGAN 튜토리얼</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/vt_tutorial.html">배포를 위한 비전 트랜스포머(Vision Transformer) 모델 최적화하기</a></li>
</ul>
<p class="caption"><span class="caption-text">오디오</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/audio_io_tutorial.html">Audio I/O</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/audio_resampling_tutorial.html">Audio Resampling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/audio_data_augmentation_tutorial.html">Audio Data Augmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/audio_feature_extractions_tutorial.html">Audio Feature Extractions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/audio_feature_augmentation_tutorial.html">Audio Feature Augmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/audio_datasets_tutorial.html">Audio Datasets</a></li>
<li class="toctree-l1"><a class="reference internal" href="speech_recognition_pipeline_tutorial.html">Speech Recognition with Wav2Vec2</a></li>
<li class="toctree-l1"><a class="reference internal" href="speech_command_classification_with_torchaudio_tutorial.html">Speech Command Classification with torchaudio</a></li>
<li class="toctree-l1"><a class="reference internal" href="text_to_speech_with_torchaudio.html">Text-to-speech with torchaudio</a></li>
<li class="toctree-l1"><a class="reference internal" href="forced_alignment_with_torchaudio_tutorial.html">Forced Alignment with Wav2Vec2</a></li>
</ul>
<p class="caption"><span class="caption-text">텍스트</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/transformer_tutorial.html">nn.Transformer 와 TorchText 로 시퀀스-투-시퀀스(Sequence-to-Sequence) 모델링하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="char_rnn_classification_tutorial.html">기초부터 시작하는 NLP: 문자-단위 RNN으로 이름 분류하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="char_rnn_generation_tutorial.html">기초부터 시작하는 NLP:  문자-단위 RNN으로 이름 생성하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="seq2seq_translation_tutorial.html">기초부터 시작하는 NLP: Sequence to Sequence 네트워크와 Attention을 이용한 번역</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/text_sentiment_ngrams_tutorial.html">torchtext 라이브러리로 텍스트 분류하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/translation_transformer.html">nn.Transformer와 torchtext로 언어 번역하기</a></li>
</ul>
<p class="caption"><span class="caption-text">강화학습</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="reinforcement_q_learning.html">강화 학습 (DQN) 튜토리얼</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Train a Mario-playing RL Agent</a></li>
</ul>
<p class="caption"><span class="caption-text">PyTorch 모델을 프로덕션 환경에 배포하기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="flask_rest_api_tutorial.html">Flask를 사용하여 Python에서 PyTorch를 REST API로 배포하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/Intro_to_TorchScript_tutorial.html">TorchScript 소개</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_export.html">C++에서 TorchScript 모델 로딩하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/super_resolution_with_onnxruntime.html">(선택) PyTorch 모델을 ONNX으로 변환하고 ONNX 런타임에서 실행하기</a></li>
</ul>
<p class="caption"><span class="caption-text">Code Transforms with FX</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="fx_conv_bn_fuser.html">(베타) FX에서 합성곱/배치 정규화(Convolution/Batch Norm) 결합기(Fuser) 만들기</a></li>
<li class="toctree-l1"><a class="reference internal" href="fx_profiling_tutorial.html">(beta) Building a Simple CPU Performance Profiler with FX</a></li>
</ul>
<p class="caption"><span class="caption-text">프론트엔드 API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="memory_format_tutorial.html">(베타) PyTorch를 사용한 Channels Last 메모리 형식</a></li>
<li class="toctree-l1"><a class="reference internal" href="forward_ad_usage.html">Forward-mode Automatic Differentiation (Beta)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_frontend.html">PyTorch C++ 프론트엔드 사용하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/torch-script-parallelism.html">TorchScript의 동적 병렬 처리(Dynamic Parallelism)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_autograd.html">C++ 프론트엔드의 자동 미분 (autograd)</a></li>
</ul>
<p class="caption"><span class="caption-text">PyTorch 확장하기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="custom_function_double_backward_tutorial.html">Double Backward with Custom Functions</a></li>
<li class="toctree-l1"><a class="reference internal" href="custom_function_conv_bn_tutorial.html">Fusing Convolution and Batch Norm using Custom Function</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_extension.html">Custom C++ and CUDA Extensions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/torch_script_custom_ops.html">Extending TorchScript with Custom C++ Operators</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/torch_script_custom_classes.html">커스텀 C++ 클래스로 TorchScript 확장하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/dispatcher.html">Registering a Dispatched Operator in C++</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/extend_dispatcher.html">Extending dispatcher for a new backend in C++</a></li>
</ul>
<p class="caption"><span class="caption-text">모델 최적화</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/profiler.html">PyTorch 모듈 프로파일링 하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensorboard_profiler_tutorial.html">PyTorch Profiler With TensorBoard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/hyperparameter_tuning_tutorial.html">Hyperparameter tuning with Ray Tune</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/vt_tutorial.html">배포를 위한 비전 트랜스포머(Vision Transformer) 모델 최적화하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="parametrizations.html">Parametrizations Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="pruning_tutorial.html">가지치기 기법(Pruning) 튜토리얼</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/dynamic_quantization_tutorial.html">(베타) LSTM 기반 단어 단위 언어 모델의 동적 양자화</a></li>
<li class="toctree-l1"><a class="reference internal" href="dynamic_quantization_bert_tutorial.html">(베타) BERT 모델 동적 양자화하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="quantized_transfer_learning_tutorial.html">(베타) 컴퓨터 비전 튜토리얼을 위한 양자화된 전이학습(Quantized Transfer Learning)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/static_quantization_tutorial.html">(beta) Static Quantization with Eager Mode in PyTorch</a></li>
</ul>
<p class="caption"><span class="caption-text">병렬 및 분산 학습</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/dist_overview.html">PyTorch Distributed Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="model_parallel_tutorial.html">단일 머신을 사용한 모델 병렬화 모범 사례</a></li>
<li class="toctree-l1"><a class="reference internal" href="ddp_tutorial.html">분산 데이터 병렬 처리 시작하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="dist_tuto.html">PyTorch로 분산 어플리케이션 개발하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="rpc_tutorial.html">Getting Started with Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="rpc_param_server_tutorial.html">Implementing a Parameter Server Using Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="dist_pipeline_parallel_tutorial.html">Distributed Pipeline Parallelism Using RPC</a></li>
<li class="toctree-l1"><a class="reference internal" href="rpc_async_execution.html">Implementing Batch RPC Processing Using Asynchronous Executions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/rpc_ddp_tutorial.html">분산 데이터 병렬(DDP)과 분산 RPC 프레임워크 결합</a></li>
<li class="toctree-l1"><a class="reference internal" href="pipeline_tutorial.html">파이프라인 병렬화로 트랜스포머 모델 학습시키기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/ddp_pipeline.html">분산 데이터 병렬 처리와 병렬 처리 파이프라인을 사용한 트랜스포머 모델 학습</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/generic_join.html">Distributed Training with Uneven Inputs Using the Join Context Manager</a></li>
</ul>
<p class="caption"><span class="caption-text">Mobile</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../beginner/deeplabv3_on_ios.html">iOS에서의 이미지 분할 DeepLapV3</a></li>
<li class="toctree-l1"><a class="reference internal" href="../beginner/deeplabv3_on_android.html">안드로이드에서의 이미지 분할 DeepLapV3</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          

<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../index.html">
          
            Tutorials
          
        </a> &gt;
      </li>

        
      <li>Train a Mario-playing RL Agent</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="../_sources/intermediate/mario_rl_tutorial.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <div class="pytorch-call-to-action-links">
            <div id="tutorial-type">intermediate/mario_rl_tutorial</div>

            <div id="google-colab-link">
              <img class="call-to-action-img" src="../_static/images/pytorch-colab.svg"/>
              <div class="call-to-action-desktop-view">Run in Google Colab</div>
              <div class="call-to-action-mobile-view">Colab</div>
            </div>
            <div id="download-notebook-link">
              <img class="call-to-action-notebook-img" src="../_static/images/pytorch-download.svg"/>
              <div class="call-to-action-desktop-view">Download Notebook</div>
              <div class="call-to-action-mobile-view">Notebook</div>
            </div>
            <div id="github-view-link">
              <img class="call-to-action-img" src="../_static/images/pytorch-github.svg"/>
              <div class="call-to-action-desktop-view">View on GitHub</div>
              <div class="call-to-action-mobile-view">GitHub</div>
            </div>
          </div>

        
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <div class="sphx-glr-download-link-note admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Click <a class="reference internal" href="#sphx-glr-download-intermediate-mario-rl-tutorial-py"><span class="std std-ref">here</span></a> to download the full example code</p>
</div>
<div class="sphx-glr-example-title section" id="train-a-mario-playing-rl-agent">
<span id="sphx-glr-intermediate-mario-rl-tutorial-py"></span><h1>Train a Mario-playing RL Agent<a class="headerlink" href="#train-a-mario-playing-rl-agent" title="Permalink to this headline">¶</a></h1>
<p>Authors: <a class="reference external" href="https://github.com/YuansongFeng">Yuansong Feng</a>, <a class="reference external" href="https://github.com/suraj813">Suraj
Subramanian</a>, <a class="reference external" href="https://github.com/hw26">Howard
Wang</a>, <a class="reference external" href="https://github.com/GuoYuzhang">Steven
Guo</a>.</p>
<p>This tutorial walks you through the fundamentals of Deep Reinforcement
Learning. At the end, you will implement an AI-powered Mario (using
<a class="reference external" href="https://arxiv.org/pdf/1509.06461.pdf">Double Deep Q-Networks</a>) that
can play the game by itself.</p>
<p>Although no prior knowledge of RL is necessary for this tutorial, you
can familiarize yourself with these RL
<a class="reference external" href="https://spinningup.openai.com/en/latest/spinningup/rl_intro.html">concepts</a>,
and have this handy
<a class="reference external" href="https://colab.research.google.com/drive/1eN33dPVtdPViiS1njTW_-r-IYCDTFU7N">cheatsheet</a>
as your companion. The full code is available
<a class="reference external" href="https://github.com/yuansongFeng/MadMario/">here</a>.</p>
<div class="figure">
<img alt="mario" src="../_images/mario.gif" />
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># !pip install gym-super-mario-bros==7.3.0</span>

<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">nn</span>
<span class="kn">from</span> <span class="nn">torchvision</span> <span class="kn">import</span> <span class="n">transforms</span> <span class="k">as</span> <span class="n">T</span>
<span class="kn">from</span> <span class="nn">PIL</span> <span class="kn">import</span> <span class="n">Image</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">pathlib</span> <span class="kn">import</span> <span class="n">Path</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">deque</span>
<span class="kn">import</span> <span class="nn">random</span><span class="o">,</span> <span class="nn">datetime</span><span class="o">,</span> <span class="nn">os</span><span class="o">,</span> <span class="nn">copy</span>

<span class="c1"># Gym is an OpenAI toolkit for RL</span>
<span class="kn">import</span> <span class="nn">gym</span>
<span class="kn">from</span> <span class="nn">gym.spaces</span> <span class="kn">import</span> <span class="n">Box</span>
<span class="kn">from</span> <span class="nn">gym.wrappers</span> <span class="kn">import</span> <span class="n">FrameStack</span>

<span class="c1"># NES Emulator for OpenAI Gym</span>
<span class="kn">from</span> <span class="nn">nes_py.wrappers</span> <span class="kn">import</span> <span class="n">JoypadSpace</span>

<span class="c1"># Super Mario environment for OpenAI Gym</span>
<span class="kn">import</span> <span class="nn">gym_super_mario_bros</span>
</pre></div>
</div>
<div class="section" id="rl-definitions">
<h2>RL Definitions<a class="headerlink" href="#rl-definitions" title="Permalink to this headline">¶</a></h2>
<p><strong>Environment</strong> The world that an agent interacts with and learns from.</p>
<p><strong>Action</strong> <span class="math">\(a\)</span> : How the Agent responds to the Environment. The
set of all possible Actions is called <em>action-space</em>.</p>
<p><strong>State</strong> <span class="math">\(s\)</span> : The current characteristic of the Environment. The
set of all possible States the Environment can be in is called
<em>state-space</em>.</p>
<p><strong>Reward</strong> <span class="math">\(r\)</span> : Reward is the key feedback from Environment to
Agent. It is what drives the Agent to learn and to change its future
action. An aggregation of rewards over multiple time steps is called
<strong>Return</strong>.</p>
<p><strong>Optimal Action-Value function</strong> <span class="math">\(Q^*(s,a)\)</span> : Gives the expected
return if you start in state <span class="math">\(s\)</span>, take an arbitrary action
<span class="math">\(a\)</span>, and then for each future time step take the action that
maximizes returns. <span class="math">\(Q\)</span> can be said to stand for the “quality” of
the action in a state. We try to approximate this function.</p>
</div>
<div class="section" id="environment">
<h2>Environment<a class="headerlink" href="#environment" title="Permalink to this headline">¶</a></h2>
<div class="section" id="initialize-environment">
<h3>Initialize Environment<a class="headerlink" href="#initialize-environment" title="Permalink to this headline">¶</a></h3>
<p>In Mario, the environment consists of tubes, mushrooms and other
components.</p>
<p>When Mario makes an action, the environment responds with the changed
(next) state, reward and other info.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># Initialize Super Mario environment</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">gym_super_mario_bros</span><span class="o">.</span><span class="n">make</span><span class="p">(</span><span class="s2">&quot;SuperMarioBros-1-1-v0&quot;</span><span class="p">)</span>

<span class="c1"># Limit the action-space to</span>
<span class="c1">#   0. walk right</span>
<span class="c1">#   1. jump right</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">JoypadSpace</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="p">[[</span><span class="s2">&quot;right&quot;</span><span class="p">],</span> <span class="p">[</span><span class="s2">&quot;right&quot;</span><span class="p">,</span> <span class="s2">&quot;A&quot;</span><span class="p">]])</span>

<span class="n">env</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>
<span class="n">next_state</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span><span class="p">,</span> <span class="n">info</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">action</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">next_state</span><span class="o">.</span><span class="n">shape</span><span class="si">}</span><span class="s2">,</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">reward</span><span class="si">}</span><span class="s2">,</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">done</span><span class="si">}</span><span class="s2">,</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">info</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>(240, 256, 3),
 0,
 False,
 {&#39;coins&#39;: 0, &#39;flag_get&#39;: False, &#39;life&#39;: 2, &#39;score&#39;: 0, &#39;stage&#39;: 1, &#39;status&#39;: &#39;small&#39;, &#39;time&#39;: 400, &#39;world&#39;: 1, &#39;x_pos&#39;: 40, &#39;x_pos_screen&#39;: 40, &#39;y_pos&#39;: 79}
</pre></div>
</div>
</div>
<div class="section" id="preprocess-environment">
<h3>Preprocess Environment<a class="headerlink" href="#preprocess-environment" title="Permalink to this headline">¶</a></h3>
<p>Environment data is returned to the agent in <code class="docutils literal notranslate"><span class="pre">next_state</span></code>. As you saw
above, each state is represented by a <code class="docutils literal notranslate"><span class="pre">[3,</span> <span class="pre">240,</span> <span class="pre">256]</span></code> size array.
Often that is more information than our agent needs; for instance,
Mario’s actions do not depend on the color of the pipes or the sky!</p>
<p>We use <strong>Wrappers</strong> to preprocess environment data before sending it to
the agent.</p>
<p><code class="docutils literal notranslate"><span class="pre">GrayScaleObservation</span></code> is a common wrapper to transform an RGB image
to grayscale; doing so reduces the size of the state representation
without losing useful information. Now the size of each state:
<code class="docutils literal notranslate"><span class="pre">[1,</span> <span class="pre">240,</span> <span class="pre">256]</span></code></p>
<p><code class="docutils literal notranslate"><span class="pre">ResizeObservation</span></code> downsamples each observation into a square image.
New size: <code class="docutils literal notranslate"><span class="pre">[1,</span> <span class="pre">84,</span> <span class="pre">84]</span></code></p>
<p><code class="docutils literal notranslate"><span class="pre">SkipFrame</span></code> is a custom wrapper that inherits from <code class="docutils literal notranslate"><span class="pre">gym.Wrapper</span></code> and
implements the <code class="docutils literal notranslate"><span class="pre">step()</span></code> function. Because consecutive frames don’t
vary much, we can skip n-intermediate frames without losing much
information. The n-th frame aggregates rewards accumulated over each
skipped frame.</p>
<p><code class="docutils literal notranslate"><span class="pre">FrameStack</span></code> is a wrapper that allows us to squash consecutive frames
of the environment into a single observation point to feed to our
learning model. This way, we can identify if Mario was landing or
jumping based on the direction of his movement in the previous several
frames.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">SkipFrame</span><span class="p">(</span><span class="n">gym</span><span class="o">.</span><span class="n">Wrapper</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">env</span><span class="p">,</span> <span class="n">skip</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Return only every `skip`-th frame&quot;&quot;&quot;</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">env</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_skip</span> <span class="o">=</span> <span class="n">skip</span>

    <span class="k">def</span> <span class="nf">step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">action</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Repeat action, and sum reward&quot;&quot;&quot;</span>
        <span class="n">total_reward</span> <span class="o">=</span> <span class="mf">0.0</span>
        <span class="n">done</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_skip</span><span class="p">):</span>
            <span class="c1"># Accumulate reward and repeat the same action</span>
            <span class="n">obs</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span><span class="p">,</span> <span class="n">info</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">env</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">action</span><span class="p">)</span>
            <span class="n">total_reward</span> <span class="o">+=</span> <span class="n">reward</span>
            <span class="k">if</span> <span class="n">done</span><span class="p">:</span>
                <span class="k">break</span>
        <span class="k">return</span> <span class="n">obs</span><span class="p">,</span> <span class="n">total_reward</span><span class="p">,</span> <span class="n">done</span><span class="p">,</span> <span class="n">info</span>


<span class="k">class</span> <span class="nc">GrayScaleObservation</span><span class="p">(</span><span class="n">gym</span><span class="o">.</span><span class="n">ObservationWrapper</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">env</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">env</span><span class="p">)</span>
        <span class="n">obs_shape</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">observation_space</span><span class="o">.</span><span class="n">shape</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">observation_space</span> <span class="o">=</span> <span class="n">Box</span><span class="p">(</span><span class="n">low</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="mi">255</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="n">obs_shape</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">permute_orientation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">observation</span><span class="p">):</span>
        <span class="c1"># permute [H, W, C] array to [C, H, W] tensor</span>
        <span class="n">observation</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">observation</span><span class="p">,</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
        <span class="n">observation</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">observation</span><span class="o">.</span><span class="n">copy</span><span class="p">(),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">observation</span>

    <span class="k">def</span> <span class="nf">observation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">observation</span><span class="p">):</span>
        <span class="n">observation</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">permute_orientation</span><span class="p">(</span><span class="n">observation</span><span class="p">)</span>
        <span class="n">transform</span> <span class="o">=</span> <span class="n">T</span><span class="o">.</span><span class="n">Grayscale</span><span class="p">()</span>
        <span class="n">observation</span> <span class="o">=</span> <span class="n">transform</span><span class="p">(</span><span class="n">observation</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">observation</span>


<span class="k">class</span> <span class="nc">ResizeObservation</span><span class="p">(</span><span class="n">gym</span><span class="o">.</span><span class="n">ObservationWrapper</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">env</span><span class="p">,</span> <span class="n">shape</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">env</span><span class="p">)</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">shape</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">shape</span> <span class="o">=</span> <span class="p">(</span><span class="n">shape</span><span class="p">,</span> <span class="n">shape</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">shape</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">shape</span><span class="p">)</span>

        <span class="n">obs_shape</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">shape</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">observation_space</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">2</span><span class="p">:]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">observation_space</span> <span class="o">=</span> <span class="n">Box</span><span class="p">(</span><span class="n">low</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">high</span><span class="o">=</span><span class="mi">255</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="n">obs_shape</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">uint8</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">observation</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">observation</span><span class="p">):</span>
        <span class="n">transforms</span> <span class="o">=</span> <span class="n">T</span><span class="o">.</span><span class="n">Compose</span><span class="p">(</span>
            <span class="p">[</span><span class="n">T</span><span class="o">.</span><span class="n">Resize</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">shape</span><span class="p">),</span> <span class="n">T</span><span class="o">.</span><span class="n">Normalize</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">255</span><span class="p">)]</span>
        <span class="p">)</span>
        <span class="n">observation</span> <span class="o">=</span> <span class="n">transforms</span><span class="p">(</span><span class="n">observation</span><span class="p">)</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">observation</span>


<span class="c1"># Apply Wrappers to environment</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">SkipFrame</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">skip</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">GrayScaleObservation</span><span class="p">(</span><span class="n">env</span><span class="p">)</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">ResizeObservation</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">shape</span><span class="o">=</span><span class="mi">84</span><span class="p">)</span>
<span class="n">env</span> <span class="o">=</span> <span class="n">FrameStack</span><span class="p">(</span><span class="n">env</span><span class="p">,</span> <span class="n">num_stack</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
</pre></div>
</div>
<p>After applying the above wrappers to the environment, the final wrapped
state consists of 4 gray-scaled consecutive frames stacked together, as
shown above in the image on the left. Each time Mario makes an action,
the environment responds with a state of this structure. The structure
is represented by a 3-D array of size <code class="docutils literal notranslate"><span class="pre">[4,</span> <span class="pre">84,</span> <span class="pre">84]</span></code>.</p>
<div class="figure">
<img alt="picture" src="../_images/mario_env.png" />
</div>
</div>
</div>
<div class="section" id="agent">
<h2>Agent<a class="headerlink" href="#agent" title="Permalink to this headline">¶</a></h2>
<p>We create a class <code class="docutils literal notranslate"><span class="pre">Mario</span></code> to represent our agent in the game. Mario
should be able to:</p>
<ul class="simple">
<li><strong>Act</strong> according to the optimal action policy based on the current
state (of the environment).</li>
<li><strong>Remember</strong> experiences. Experience = (current state, current
action, reward, next state). Mario <em>caches</em> and later <em>recalls</em> his
experiences to update his action policy.</li>
<li><strong>Learn</strong> a better action policy over time</li>
</ul>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Mario</span><span class="p">:</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">():</span>
        <span class="k">pass</span>

    <span class="k">def</span> <span class="nf">act</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Given a state, choose an epsilon-greedy action&quot;&quot;&quot;</span>
        <span class="k">pass</span>

    <span class="k">def</span> <span class="nf">cache</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">experience</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Add the experience to memory&quot;&quot;&quot;</span>
        <span class="k">pass</span>

    <span class="k">def</span> <span class="nf">recall</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Sample experiences from memory&quot;&quot;&quot;</span>
        <span class="k">pass</span>

    <span class="k">def</span> <span class="nf">learn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Update online action value (Q) function with a batch of experiences&quot;&quot;&quot;</span>
        <span class="k">pass</span>
</pre></div>
</div>
<p>In the following sections, we will populate Mario’s parameters and
define his functions.</p>
<div class="section" id="act">
<h3>Act<a class="headerlink" href="#act" title="Permalink to this headline">¶</a></h3>
<p>For any given state, an agent can choose to do the most optimal action
(<strong>exploit</strong>) or a random action (<strong>explore</strong>).</p>
<p>Mario randomly explores with a chance of <code class="docutils literal notranslate"><span class="pre">self.exploration_rate</span></code>; when
he chooses to exploit, he relies on <code class="docutils literal notranslate"><span class="pre">MarioNet</span></code> (implemented in
<code class="docutils literal notranslate"><span class="pre">Learn</span></code> section) to provide the most optimal action.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Mario</span><span class="p">:</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">state_dim</span> <span class="o">=</span> <span class="n">state_dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">action_dim</span> <span class="o">=</span> <span class="n">action_dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">save_dir</span> <span class="o">=</span> <span class="n">save_dir</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">use_cuda</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">()</span>

        <span class="c1"># Mario&#39;s DNN to predict the most optimal action - we implement this in the Learn section</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">net</span> <span class="o">=</span> <span class="n">MarioNet</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">state_dim</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_dim</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cuda</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">net</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="s2">&quot;cuda&quot;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate</span> <span class="o">=</span> <span class="mi">1</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate_decay</span> <span class="o">=</span> <span class="mf">0.99999975</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate_min</span> <span class="o">=</span> <span class="mf">0.1</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">save_every</span> <span class="o">=</span> <span class="mf">5e5</span>  <span class="c1"># no. of experiences between saving Mario Net</span>

    <span class="k">def</span> <span class="nf">act</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Given a state, choose an epsilon-greedy action and update value of step.</span>

<span class="sd">    Inputs:</span>
<span class="sd">    state(LazyFrame): A single observation of the current state, dimension is (state_dim)</span>
<span class="sd">    Outputs:</span>
<span class="sd">    action_idx (int): An integer representing which action Mario will perform</span>
<span class="sd">    &quot;&quot;&quot;</span>
        <span class="c1"># EXPLORE</span>
        <span class="k">if</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">()</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate</span><span class="p">:</span>
            <span class="n">action_idx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">action_dim</span><span class="p">)</span>

        <span class="c1"># EXPLOIT</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">state</span> <span class="o">=</span> <span class="n">state</span><span class="o">.</span><span class="n">__array__</span><span class="p">()</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cuda</span><span class="p">:</span>
                <span class="n">state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">state</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">state</span><span class="p">)</span>
            <span class="n">state</span> <span class="o">=</span> <span class="n">state</span><span class="o">.</span><span class="n">unsqueeze</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
            <span class="n">action_values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">model</span><span class="o">=</span><span class="s2">&quot;online&quot;</span><span class="p">)</span>
            <span class="n">action_idx</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">action_values</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>

        <span class="c1"># decrease exploration_rate</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate</span> <span class="o">*=</span> <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate_decay</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate_min</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate</span><span class="p">)</span>

        <span class="c1"># increment step</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="k">return</span> <span class="n">action_idx</span>
</pre></div>
</div>
</div>
<div class="section" id="cache-and-recall">
<h3>Cache and Recall<a class="headerlink" href="#cache-and-recall" title="Permalink to this headline">¶</a></h3>
<p>These two functions serve as Mario’s “memory” process.</p>
<p><code class="docutils literal notranslate"><span class="pre">cache()</span></code>: Each time Mario performs an action, he stores the
<code class="docutils literal notranslate"><span class="pre">experience</span></code> to his memory. His experience includes the current
<em>state</em>, <em>action</em> performed, <em>reward</em> from the action, the <em>next state</em>,
and whether the game is <em>done</em>.</p>
<p><code class="docutils literal notranslate"><span class="pre">recall()</span></code>: Mario randomly samples a batch of experiences from his
memory, and uses that to learn the game.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Mario</span><span class="p">(</span><span class="n">Mario</span><span class="p">):</span>  <span class="c1"># subclassing for continuity</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">memory</span> <span class="o">=</span> <span class="n">deque</span><span class="p">(</span><span class="n">maxlen</span><span class="o">=</span><span class="mi">100000</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">32</span>

    <span class="k">def</span> <span class="nf">cache</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">action</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Store the experience to self.memory (replay buffer)</span>

<span class="sd">        Inputs:</span>
<span class="sd">        state (LazyFrame),</span>
<span class="sd">        next_state (LazyFrame),</span>
<span class="sd">        action (int),</span>
<span class="sd">        reward (float),</span>
<span class="sd">        done(bool))</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">state</span> <span class="o">=</span> <span class="n">state</span><span class="o">.</span><span class="n">__array__</span><span class="p">()</span>
        <span class="n">next_state</span> <span class="o">=</span> <span class="n">next_state</span><span class="o">.</span><span class="n">__array__</span><span class="p">()</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_cuda</span><span class="p">:</span>
            <span class="n">state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">state</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
            <span class="n">next_state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">next_state</span><span class="p">)</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
            <span class="n">action</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">action</span><span class="p">])</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
            <span class="n">reward</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">reward</span><span class="p">])</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
            <span class="n">done</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">done</span><span class="p">])</span><span class="o">.</span><span class="n">cuda</span><span class="p">()</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">state</span><span class="p">)</span>
            <span class="n">next_state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="n">next_state</span><span class="p">)</span>
            <span class="n">action</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">action</span><span class="p">])</span>
            <span class="n">reward</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">reward</span><span class="p">])</span>
            <span class="n">done</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="n">done</span><span class="p">])</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">memory</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">state</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">action</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span><span class="p">,))</span>

    <span class="k">def</span> <span class="nf">recall</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Retrieve a batch of experiences from memory</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">batch</span> <span class="o">=</span> <span class="n">random</span><span class="o">.</span><span class="n">sample</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">memory</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span><span class="p">)</span>
        <span class="n">state</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">action</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">stack</span><span class="p">,</span> <span class="nb">zip</span><span class="p">(</span><span class="o">*</span><span class="n">batch</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">state</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">action</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(),</span> <span class="n">reward</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(),</span> <span class="n">done</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="learn">
<h3>Learn<a class="headerlink" href="#learn" title="Permalink to this headline">¶</a></h3>
<p>Mario uses the <a class="reference external" href="https://arxiv.org/pdf/1509.06461">DDQN algorithm</a>
under the hood. DDQN uses two ConvNets - <span class="math">\(Q_{online}\)</span> and
<span class="math">\(Q_{target}\)</span> - that independently approximate the optimal
action-value function.</p>
<p>In our implementation, we share feature generator <code class="docutils literal notranslate"><span class="pre">features</span></code> across
<span class="math">\(Q_{online}\)</span> and <span class="math">\(Q_{target}\)</span>, but maintain separate FC
classifiers for each. <span class="math">\(\theta_{target}\)</span> (the parameters of
<span class="math">\(Q_{target}\)</span>) is frozen to prevent updation by backprop. Instead,
it is periodically synced with <span class="math">\(\theta_{online}\)</span> (more on this
later).</p>
<div class="section" id="neural-network">
<h4>Neural Network<a class="headerlink" href="#neural-network" title="Permalink to this headline">¶</a></h4>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">MarioNet</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;mini cnn structure</span>
<span class="sd">  input -&gt; (conv2d + relu) x 3 -&gt; flatten -&gt; (dense + relu) x 2 -&gt; output</span>
<span class="sd">  &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_dim</span><span class="p">,</span> <span class="n">output_dim</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="n">c</span><span class="p">,</span> <span class="n">h</span><span class="p">,</span> <span class="n">w</span> <span class="o">=</span> <span class="n">input_dim</span>

        <span class="k">if</span> <span class="n">h</span> <span class="o">!=</span> <span class="mi">84</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Expecting input height: 84, got: </span><span class="si">{</span><span class="n">h</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">w</span> <span class="o">!=</span> <span class="mi">84</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Expecting input width: 84, got: </span><span class="si">{</span><span class="n">w</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">online</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Sequential</span><span class="p">(</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="n">in_channels</span><span class="o">=</span><span class="n">c</span><span class="p">,</span> <span class="n">out_channels</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">kernel_size</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">4</span><span class="p">),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="n">in_channels</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">out_channels</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span> <span class="n">kernel_size</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="n">in_channels</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span> <span class="n">out_channels</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span> <span class="n">kernel_size</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">Flatten</span><span class="p">(),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">3136</span><span class="p">,</span> <span class="mi">512</span><span class="p">),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">ReLU</span><span class="p">(),</span>
            <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">512</span><span class="p">,</span> <span class="n">output_dim</span><span class="p">),</span>
        <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">target</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">deepcopy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">online</span><span class="p">)</span>

        <span class="c1"># Q_target parameters are frozen.</span>
        <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">parameters</span><span class="p">():</span>
            <span class="n">p</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">input</span><span class="p">,</span> <span class="n">model</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">model</span> <span class="o">==</span> <span class="s2">&quot;online&quot;</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">online</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">model</span> <span class="o">==</span> <span class="s2">&quot;target&quot;</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">target</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="td-estimate-td-target">
<h4>TD Estimate &amp; TD Target<a class="headerlink" href="#td-estimate-td-target" title="Permalink to this headline">¶</a></h4>
<p>Two values are involved in learning:</p>
<p><strong>TD Estimate</strong> - the predicted optimal <span class="math">\(Q^*\)</span> for a given state
<span class="math">\(s\)</span></p>
<div class="math">
\[{TD}_e = Q_{online}^*(s,a)\]</div>
<p><strong>TD Target</strong> - aggregation of current reward and the estimated
<span class="math">\(Q^*\)</span> in the next state <span class="math">\(s'\)</span></p>
<div class="math">
\[a' = argmax_{a} Q_{online}(s', a)\]</div>
<div class="math">
\[{TD}_t = r + \gamma Q_{target}^*(s',a')\]</div>
<p>Because we don’t know what next action <span class="math">\(a'\)</span> will be, we use the
action <span class="math">\(a'\)</span> maximizes <span class="math">\(Q_{online}\)</span> in the next state
<span class="math">\(s'\)</span>.</p>
<p>Notice we use the
<a class="reference external" href="https://pytorch.org/docs/stable/generated/torch.no_grad.html#no-grad">&#64;torch.no_grad()</a>
decorator on <code class="docutils literal notranslate"><span class="pre">td_target()</span></code> to disable gradient calculations here
(because we don’t need to backpropagate on <span class="math">\(\theta_{target}\)</span>).</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Mario</span><span class="p">(</span><span class="n">Mario</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="mf">0.9</span>

    <span class="k">def</span> <span class="nf">td_estimate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">action</span><span class="p">):</span>
        <span class="n">current_Q</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">model</span><span class="o">=</span><span class="s2">&quot;online&quot;</span><span class="p">)[</span>
            <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span><span class="p">),</span> <span class="n">action</span>
        <span class="p">]</span>  <span class="c1"># Q_online(s,a)</span>
        <span class="k">return</span> <span class="n">current_Q</span>

    <span class="nd">@torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">()</span>
    <span class="k">def</span> <span class="nf">td_target</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">done</span><span class="p">):</span>
        <span class="n">next_state_Q</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="p">(</span><span class="n">next_state</span><span class="p">,</span> <span class="n">model</span><span class="o">=</span><span class="s2">&quot;online&quot;</span><span class="p">)</span>
        <span class="n">best_action</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">next_state_Q</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">next_Q</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="p">(</span><span class="n">next_state</span><span class="p">,</span> <span class="n">model</span><span class="o">=</span><span class="s2">&quot;target&quot;</span><span class="p">)[</span>
            <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">batch_size</span><span class="p">),</span> <span class="n">best_action</span>
        <span class="p">]</span>
        <span class="k">return</span> <span class="p">(</span><span class="n">reward</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">done</span><span class="o">.</span><span class="n">float</span><span class="p">())</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">*</span> <span class="n">next_Q</span><span class="p">)</span><span class="o">.</span><span class="n">float</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="updating-the-model">
<h4>Updating the model<a class="headerlink" href="#updating-the-model" title="Permalink to this headline">¶</a></h4>
<p>As Mario samples inputs from his replay buffer, we compute <span class="math">\(TD_t\)</span>
and <span class="math">\(TD_e\)</span> and backpropagate this loss down <span class="math">\(Q_{online}\)</span> to
update its parameters <span class="math">\(\theta_{online}\)</span> (<span class="math">\(\alpha\)</span> is the
learning rate <code class="docutils literal notranslate"><span class="pre">lr</span></code> passed to the <code class="docutils literal notranslate"><span class="pre">optimizer</span></code>)</p>
<div class="math">
\[\theta_{online} \leftarrow \theta_{online} + \alpha \nabla(TD_e - TD_t)\]</div>
<p><span class="math">\(\theta_{target}\)</span> does not update through backpropagation.
Instead, we periodically copy <span class="math">\(\theta_{online}\)</span> to
<span class="math">\(\theta_{target}\)</span></p>
<div class="math">
\[\theta_{target} \leftarrow \theta_{online}\]</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Mario</span><span class="p">(</span><span class="n">Mario</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">Adam</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.00025</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">SmoothL1Loss</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">update_Q_online</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">td_estimate</span><span class="p">,</span> <span class="n">td_target</span><span class="p">):</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">loss_fn</span><span class="p">(</span><span class="n">td_estimate</span><span class="p">,</span> <span class="n">td_target</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
        <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
        <span class="k">return</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">sync_Q_target</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="o">.</span><span class="n">target</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="o">.</span><span class="n">online</span><span class="o">.</span><span class="n">state_dict</span><span class="p">())</span>
</pre></div>
</div>
</div>
<div class="section" id="save-checkpoint">
<h4>Save checkpoint<a class="headerlink" href="#save-checkpoint" title="Permalink to this headline">¶</a></h4>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Mario</span><span class="p">(</span><span class="n">Mario</span><span class="p">):</span>
    <span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">save_path</span> <span class="o">=</span> <span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">save_dir</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">&quot;mario_net_</span><span class="si">{</span><span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span> <span class="o">//</span> <span class="bp">self</span><span class="o">.</span><span class="n">save_every</span><span class="p">)</span><span class="si">}</span><span class="s2">.chkpt&quot;</span>
        <span class="p">)</span>
        <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">(</span>
            <span class="nb">dict</span><span class="p">(</span><span class="n">model</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">net</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">exploration_rate</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">exploration_rate</span><span class="p">),</span>
            <span class="n">save_path</span><span class="p">,</span>
        <span class="p">)</span>
        <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;MarioNet saved to </span><span class="si">{</span><span class="n">save_path</span><span class="si">}</span><span class="s2"> at step </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="putting-it-all-together">
<h4>Putting it all together<a class="headerlink" href="#putting-it-all-together" title="Permalink to this headline">¶</a></h4>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Mario</span><span class="p">(</span><span class="n">Mario</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">state_dim</span><span class="p">,</span> <span class="n">action_dim</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">burnin</span> <span class="o">=</span> <span class="mf">1e4</span>  <span class="c1"># min. experiences before training</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">learn_every</span> <span class="o">=</span> <span class="mi">3</span>  <span class="c1"># no. of experiences between updates to Q_online</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">sync_every</span> <span class="o">=</span> <span class="mf">1e4</span>  <span class="c1"># no. of experiences between Q_target &amp; Q_online sync</span>

    <span class="k">def</span> <span class="nf">learn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">sync_every</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">sync_Q_target</span><span class="p">()</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">save_every</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">burnin</span><span class="p">:</span>
            <span class="k">return</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">curr_step</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">learn_every</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">return</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span>

        <span class="c1"># Sample from memory</span>
        <span class="n">state</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">action</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">recall</span><span class="p">()</span>

        <span class="c1"># Get TD Estimate</span>
        <span class="n">td_est</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_estimate</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">action</span><span class="p">)</span>

        <span class="c1"># Get TD Target</span>
        <span class="n">td_tgt</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_target</span><span class="p">(</span><span class="n">reward</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">done</span><span class="p">)</span>

        <span class="c1"># Backpropagate loss through Q_online</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">update_Q_online</span><span class="p">(</span><span class="n">td_est</span><span class="p">,</span> <span class="n">td_tgt</span><span class="p">)</span>

        <span class="k">return</span> <span class="p">(</span><span class="n">td_est</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="n">loss</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="logging">
<h3>Logging<a class="headerlink" href="#logging" title="Permalink to this headline">¶</a></h3>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">time</span><span class="o">,</span> <span class="nn">datetime</span>
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>


<span class="k">class</span> <span class="nc">MetricLogger</span><span class="p">:</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">save_dir</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">save_log</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;log&quot;</span>
        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">save_log</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span>
                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="s1">&#39;Episode&#39;</span><span class="si">:</span><span class="s2">&gt;8</span><span class="si">}{</span><span class="s1">&#39;Step&#39;</span><span class="si">:</span><span class="s2">&gt;8</span><span class="si">}{</span><span class="s1">&#39;Epsilon&#39;</span><span class="si">:</span><span class="s2">&gt;10</span><span class="si">}{</span><span class="s1">&#39;MeanReward&#39;</span><span class="si">:</span><span class="s2">&gt;15</span><span class="si">}</span><span class="s2">&quot;</span>
                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="s1">&#39;MeanLength&#39;</span><span class="si">:</span><span class="s2">&gt;15</span><span class="si">}{</span><span class="s1">&#39;MeanLoss&#39;</span><span class="si">:</span><span class="s2">&gt;15</span><span class="si">}{</span><span class="s1">&#39;MeanQValue&#39;</span><span class="si">:</span><span class="s2">&gt;15</span><span class="si">}</span><span class="s2">&quot;</span>
                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="s1">&#39;TimeDelta&#39;</span><span class="si">:</span><span class="s2">&gt;15</span><span class="si">}{</span><span class="s1">&#39;Time&#39;</span><span class="si">:</span><span class="s2">&gt;20</span><span class="si">}</span><span class="se">\n</span><span class="s2">&quot;</span>
            <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_rewards_plot</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;reward_plot.jpg&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_lengths_plot</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;length_plot.jpg&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_losses_plot</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;loss_plot.jpg&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_qs_plot</span> <span class="o">=</span> <span class="n">save_dir</span> <span class="o">/</span> <span class="s2">&quot;q_plot.jpg&quot;</span>

        <span class="c1"># History metrics</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_rewards</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_lengths</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_losses</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_qs</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="c1"># Moving averages, added for every call to record()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_rewards</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_lengths</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_avg_losses</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_avg_qs</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="c1"># Current episode metric</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">init_episode</span><span class="p">()</span>

        <span class="c1"># Timing</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">record_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">log_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">loss</span><span class="p">,</span> <span class="n">q</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_reward</span> <span class="o">+=</span> <span class="n">reward</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_length</span> <span class="o">+=</span> <span class="mi">1</span>
        <span class="k">if</span> <span class="n">loss</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss</span> <span class="o">+=</span> <span class="n">loss</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_q</span> <span class="o">+=</span> <span class="n">q</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss_length</span> <span class="o">+=</span> <span class="mi">1</span>

    <span class="k">def</span> <span class="nf">log_episode</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="s2">&quot;Mark end of episode&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_rewards</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_reward</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_lengths</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_length</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss_length</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">ep_avg_loss</span> <span class="o">=</span> <span class="mi">0</span>
            <span class="n">ep_avg_q</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">ep_avg_loss</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss_length</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
            <span class="n">ep_avg_q</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_q</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss_length</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_losses</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ep_avg_loss</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_qs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ep_avg_q</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">init_episode</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">init_episode</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_reward</span> <span class="o">=</span> <span class="mf">0.0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_length</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_q</span> <span class="o">=</span> <span class="mf">0.0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">curr_ep_loss_length</span> <span class="o">=</span> <span class="mi">0</span>

    <span class="k">def</span> <span class="nf">record</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">episode</span><span class="p">,</span> <span class="n">epsilon</span><span class="p">,</span> <span class="n">step</span><span class="p">):</span>
        <span class="n">mean_ep_reward</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ep_rewards</span><span class="p">[</span><span class="o">-</span><span class="mi">100</span><span class="p">:]),</span> <span class="mi">3</span><span class="p">)</span>
        <span class="n">mean_ep_length</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ep_lengths</span><span class="p">[</span><span class="o">-</span><span class="mi">100</span><span class="p">:]),</span> <span class="mi">3</span><span class="p">)</span>
        <span class="n">mean_ep_loss</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_losses</span><span class="p">[</span><span class="o">-</span><span class="mi">100</span><span class="p">:]),</span> <span class="mi">3</span><span class="p">)</span>
        <span class="n">mean_ep_q</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ep_avg_qs</span><span class="p">[</span><span class="o">-</span><span class="mi">100</span><span class="p">:]),</span> <span class="mi">3</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_rewards</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_ep_reward</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_lengths</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_ep_length</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_avg_losses</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_ep_loss</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">moving_avg_ep_avg_qs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_ep_q</span><span class="p">)</span>

        <span class="n">last_record_time</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">record_time</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">record_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
        <span class="n">time_since_last_record</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">record_time</span> <span class="o">-</span> <span class="n">last_record_time</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>

        <span class="nb">print</span><span class="p">(</span>
            <span class="sa">f</span><span class="s2">&quot;Episode </span><span class="si">{</span><span class="n">episode</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Step </span><span class="si">{</span><span class="n">step</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Epsilon </span><span class="si">{</span><span class="n">epsilon</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Mean Reward </span><span class="si">{</span><span class="n">mean_ep_reward</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Mean Length </span><span class="si">{</span><span class="n">mean_ep_length</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Mean Loss </span><span class="si">{</span><span class="n">mean_ep_loss</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Mean Q Value </span><span class="si">{</span><span class="n">mean_ep_q</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Time Delta </span><span class="si">{</span><span class="n">time_since_last_record</span><span class="si">}</span><span class="s2"> - &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;Time </span><span class="si">{</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">T%H:%M:%S&#39;</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span>
        <span class="p">)</span>

        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">save_log</span><span class="p">,</span> <span class="s2">&quot;a&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
            <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span>
                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">episode</span><span class="si">:</span><span class="s2">8d</span><span class="si">}{</span><span class="n">step</span><span class="si">:</span><span class="s2">8d</span><span class="si">}{</span><span class="n">epsilon</span><span class="si">:</span><span class="s2">10.3f</span><span class="si">}</span><span class="s2">&quot;</span>
                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">mean_ep_reward</span><span class="si">:</span><span class="s2">15.3f</span><span class="si">}{</span><span class="n">mean_ep_length</span><span class="si">:</span><span class="s2">15.3f</span><span class="si">}{</span><span class="n">mean_ep_loss</span><span class="si">:</span><span class="s2">15.3f</span><span class="si">}{</span><span class="n">mean_ep_q</span><span class="si">:</span><span class="s2">15.3f</span><span class="si">}</span><span class="s2">&quot;</span>
                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">time_since_last_record</span><span class="si">:</span><span class="s2">15.3f</span><span class="si">}</span><span class="s2">&quot;</span>
                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s1">&#39;%Y-%m-</span><span class="si">%d</span><span class="s1">T%H:%M:%S&#39;</span><span class="p">)</span><span class="si">:</span><span class="s2">&gt;20</span><span class="si">}</span><span class="se">\n</span><span class="s2">&quot;</span>
            <span class="p">)</span>

        <span class="k">for</span> <span class="n">metric</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;ep_rewards&quot;</span><span class="p">,</span> <span class="s2">&quot;ep_lengths&quot;</span><span class="p">,</span> <span class="s2">&quot;ep_avg_losses&quot;</span><span class="p">,</span> <span class="s2">&quot;ep_avg_qs&quot;</span><span class="p">]:</span>
            <span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;moving_avg_</span><span class="si">{</span><span class="n">metric</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">))</span>
            <span class="n">plt</span><span class="o">.</span><span class="n">savefig</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">metric</span><span class="si">}</span><span class="s2">_plot&quot;</span><span class="p">))</span>
            <span class="n">plt</span><span class="o">.</span><span class="n">clf</span><span class="p">()</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="lets-play">
<h2>Let’s play!<a class="headerlink" href="#lets-play" title="Permalink to this headline">¶</a></h2>
<p>In this example we run the training loop for 10 episodes, but for Mario to truly learn the ways of
his world, we suggest running the loop for at least 40,000 episodes!</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">use_cuda</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Using CUDA: </span><span class="si">{</span><span class="n">use_cuda</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">()</span>

<span class="n">save_dir</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">&quot;checkpoints&quot;</span><span class="p">)</span> <span class="o">/</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">T%H-%M-%S&quot;</span><span class="p">)</span>
<span class="n">save_dir</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">mario</span> <span class="o">=</span> <span class="n">Mario</span><span class="p">(</span><span class="n">state_dim</span><span class="o">=</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">84</span><span class="p">,</span> <span class="mi">84</span><span class="p">),</span> <span class="n">action_dim</span><span class="o">=</span><span class="n">env</span><span class="o">.</span><span class="n">action_space</span><span class="o">.</span><span class="n">n</span><span class="p">,</span> <span class="n">save_dir</span><span class="o">=</span><span class="n">save_dir</span><span class="p">)</span>

<span class="n">logger</span> <span class="o">=</span> <span class="n">MetricLogger</span><span class="p">(</span><span class="n">save_dir</span><span class="p">)</span>

<span class="n">episodes</span> <span class="o">=</span> <span class="mi">10</span>
<span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">episodes</span><span class="p">):</span>

    <span class="n">state</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">reset</span><span class="p">()</span>

    <span class="c1"># Play the game!</span>
    <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>

        <span class="c1"># Run agent on the state</span>
        <span class="n">action</span> <span class="o">=</span> <span class="n">mario</span><span class="o">.</span><span class="n">act</span><span class="p">(</span><span class="n">state</span><span class="p">)</span>

        <span class="c1"># Agent performs action</span>
        <span class="n">next_state</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span><span class="p">,</span> <span class="n">info</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">action</span><span class="p">)</span>

        <span class="c1"># Remember</span>
        <span class="n">mario</span><span class="o">.</span><span class="n">cache</span><span class="p">(</span><span class="n">state</span><span class="p">,</span> <span class="n">next_state</span><span class="p">,</span> <span class="n">action</span><span class="p">,</span> <span class="n">reward</span><span class="p">,</span> <span class="n">done</span><span class="p">)</span>

        <span class="c1"># Learn</span>
        <span class="n">q</span><span class="p">,</span> <span class="n">loss</span> <span class="o">=</span> <span class="n">mario</span><span class="o">.</span><span class="n">learn</span><span class="p">()</span>

        <span class="c1"># Logging</span>
        <span class="n">logger</span><span class="o">.</span><span class="n">log_step</span><span class="p">(</span><span class="n">reward</span><span class="p">,</span> <span class="n">loss</span><span class="p">,</span> <span class="n">q</span><span class="p">)</span>

        <span class="c1"># Update state</span>
        <span class="n">state</span> <span class="o">=</span> <span class="n">next_state</span>

        <span class="c1"># Check if end of game</span>
        <span class="k">if</span> <span class="n">done</span> <span class="ow">or</span> <span class="n">info</span><span class="p">[</span><span class="s2">&quot;flag_get&quot;</span><span class="p">]:</span>
            <span class="k">break</span>

    <span class="n">logger</span><span class="o">.</span><span class="n">log_episode</span><span class="p">()</span>

    <span class="k">if</span> <span class="n">e</span> <span class="o">%</span> <span class="mi">20</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="n">logger</span><span class="o">.</span><span class="n">record</span><span class="p">(</span><span class="n">episode</span><span class="o">=</span><span class="n">e</span><span class="p">,</span> <span class="n">epsilon</span><span class="o">=</span><span class="n">mario</span><span class="o">.</span><span class="n">exploration_rate</span><span class="p">,</span> <span class="n">step</span><span class="o">=</span><span class="n">mario</span><span class="o">.</span><span class="n">curr_step</span><span class="p">)</span>
</pre></div>
</div>
<img alt="../_images/sphx_glr_mario_rl_tutorial_001.png" class="sphx-glr-single-img" src="../_images/sphx_glr_mario_rl_tutorial_001.png" />
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Using CUDA: True

Episode 0 - Step 289 - Epsilon 0.9999277526009276 - Mean Reward 1016.0 - Mean Length 289.0 - Mean Loss 0.0 - Mean Q Value 0.0 - Time Delta 10.814 - Time 2021-07-29T18:36:54
</pre></div>
</div>
</div>
<div class="section" id="conclusion">
<h2>Conclusion<a class="headerlink" href="#conclusion" title="Permalink to this headline">¶</a></h2>
<p>In this tutorial, we saw how we can use PyTorch to train a game-playing AI. You can use the same methods
to train an AI to play any of the games at the <a class="reference external" href="https://gym.openai.com/">OpenAI gym</a>. Hope you enjoyed this tutorial, feel free to reach us at
<a class="reference external" href="https://github.com/yuansongFeng/MadMario/">our github</a>!</p>
<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 1 minutes  10.504 seconds)</p>
<div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-intermediate-mario-rl-tutorial-py">
<div class="sphx-glr-download docutils container">
<a class="reference download internal" download="" href="../_downloads/934f425a64afc04a1466c778c945fb84/mario_rl_tutorial.py"><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Python</span> <span class="pre">source</span> <span class="pre">code:</span> <span class="pre">mario_rl_tutorial.py</span></code></a></div>
<div class="sphx-glr-download docutils container">
<a class="reference download internal" download="" href="../_downloads/1ceade89aedc4c99a944f752a51c6d35/mario_rl_tutorial.ipynb"><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Jupyter</span> <span class="pre">notebook:</span> <span class="pre">mario_rl_tutorial.ipynb</span></code></a></div>
</div>
<p class="sphx-glr-signature"><a class="reference external" href="https://sphinx-gallery.readthedocs.io">Gallery generated by Sphinx-Gallery</a></p>
</div>
</div>


             </article>
             
            </div>
            <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="flask_rest_api_tutorial.html" class="btn btn-neutral float-right" title="Flask를 사용하여 Python에서 PyTorch를 REST API로 배포하기" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
      
      
        <a href="reinforcement_q_learning.html" class="btn btn-neutral" title="강화 학습 (DQN) 튜토리얼" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
      
    </div>
  

    <hr class="rating-hr hr-top">
      <div class="rating-container">
        <div class="rating-prompt">Rate this Tutorial</div>
        <div class="stars-outer">
          <i class="far fa-star" title="1 Star" data-behavior="tutorial-rating" data-count="1"></i>
          <i class="far fa-star" title="2 Stars" data-behavior="tutorial-rating" data-count="2"></i>
          <i class="far fa-star" title="3 Stars" data-behavior="tutorial-rating" data-count="3"></i>
          <i class="far fa-star" title="4 Stars" data-behavior="tutorial-rating" data-count="4"></i>
          <i class="far fa-star" title="5 Stars" data-behavior="tutorial-rating" data-count="5"></i>
        </div>
      </div>
    <hr class="rating-hr hr-bottom"/>

  
  <div role="contentinfo">
    <p>
        &copy; Copyright 2021, PyTorch &amp; PyTorch Korea Community.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Train a Mario-playing RL Agent</a><ul>
<li><a class="reference internal" href="#rl-definitions">RL Definitions</a></li>
<li><a class="reference internal" href="#environment">Environment</a><ul>
<li><a class="reference internal" href="#initialize-environment">Initialize Environment</a></li>
<li><a class="reference internal" href="#preprocess-environment">Preprocess Environment</a></li>
</ul>
</li>
<li><a class="reference internal" href="#agent">Agent</a><ul>
<li><a class="reference internal" href="#act">Act</a></li>
<li><a class="reference internal" href="#cache-and-recall">Cache and Recall</a></li>
<li><a class="reference internal" href="#learn">Learn</a><ul>
<li><a class="reference internal" href="#neural-network">Neural Network</a></li>
<li><a class="reference internal" href="#td-estimate-td-target">TD Estimate &amp; TD Target</a></li>
<li><a class="reference internal" href="#updating-the-model">Updating the model</a></li>
<li><a class="reference internal" href="#save-checkpoint">Save checkpoint</a></li>
<li><a class="reference internal" href="#putting-it-all-together">Putting it all together</a></li>
</ul>
</li>
<li><a class="reference internal" href="#logging">Logging</a></li>
</ul>
</li>
<li><a class="reference internal" href="#lets-play">Let’s play!</a></li>
<li><a class="reference internal" href="#conclusion">Conclusion</a></li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  
       <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
         <script type="text/javascript" src="../_static/jquery.js"></script>
         <script type="text/javascript" src="../_static/underscore.js"></script>
         <script type="text/javascript" src="../_static/doctools.js"></script>
         <script type="text/javascript" src="../_static/clipboard.min.js"></script>
         <script type="text/javascript" src="../_static/copybutton.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/contrib/auto-render.min.js"></script>
         <script type="text/javascript" src="../_static/katex_autorenderer.js"></script>
     

  <script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 

<script>
//add microsoft link

if(window.location.href.indexOf("/beginner/basics/")!= -1)
{
  var url="https://docs.microsoft.com/learn/paths/pytorch-fundamentals/?wt.mc_id=aiml-7486-cxa";
  switch(window.location.pathname.split("/").pop().replace('.html',''))
  {
    case"quickstart_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/9-quickstart?WT.mc_id=aiml-7486-cxa";
      break;
    case"tensorqs_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/2-tensors?WT.mc_id=aiml-7486-cxa";
      break;
    case"data_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/3-data?WT.mc_id=aiml-7486-cxa";
      break;
    case"transforms_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/4-transforms?WT.mc_id=aiml-7486-cxa";
      break;
    case"buildmodel_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/5-model?WT.mc_id=aiml-7486-cxa";
      break;
    case"autogradqs_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/6-autograd?WT.mc_id=aiml-7486-cxa";
      break;
    case"optimization_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/7-optimization?WT.mc_id=aiml-7486-cxa";
      break;
    case"saveloadrun_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/8-inference?WT.mc_id=aiml-7486-cxa";
    }

    $(".pytorch-call-to-action-links").children().first().before("<a href="+url+' data-behavior="call-to-action-event" data-response="Run in Microsoft Learn" target="_blank"><div id="microsoft-learn-link" style="padding-bottom: 0.625rem;border-bottom: 1px solid #f3f4f7;padding-right: 2.5rem;display: -webkit-box;  display: -ms-flexbox; isplay: flex; -webkit-box-align: center;-ms-flex-align: center;align-items: center;"><img class="call-to-action-img" src="../../_static/images/microsoft-logo.svg"/><div class="call-to-action-desktop-view">Run in Microsoft Learn</div><div class="call-to-action-mobile-view">Learn</div></div></a>')
  }
</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-71919972-3"></script>
<script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());
  gtag('config', 'UA-71919972-3');
</script>


<script>
  $("[data-behavior='call-to-action-event']").on('click', function(){
    ga('send', {
      hitType: 'event',
      eventCategory: $(this).attr("data-response"),
      eventAction: 'click',
      eventLabel: window.location.href
    });

    gtag('event', 'click', {
      'event_category': $(this).attr("data-response"),
      'event_label': $("h1").first().text(),
      'tutorial_link': window.location.href
    });
   });

   $("[data-behavior='tutorial-rating']").on('click', function(){
    gtag('event', 'click', {
      'event_category': 'Tutorial Rating',
      'event_label': $("h1").first().text(),
      'value': $(this).attr("data-count")
    });
   });

   if (location.pathname == "/") {
     $(".rating-container").hide();
     $(".hr-bottom").hide();
   }
</script>

<script type="text/javascript">
  var collapsedSections = ['파이토치(PyTorch) 레시피', '파이토치(PyTorch) 배우기', '이미지/비디오', '오디오', '텍스트', '강화학습', 'PyTorch 모델을 프로덕션 환경에 배포하기', 'Code Transforms with FX', '프론트엔드 API', 'PyTorch 확장하기', '모델 최적화', '병렬 및 분산 학습', 'Mobile'];
</script>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>공식 문서 (영어)</h2>
          <p>PyTorch 공식 문서입니다.</p>
          <a id="orgTutorialLink" class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html" target="_blank">공식 문서로 이동</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>한국어 튜토리얼</h2>
          <p>한국어로 번역 중인 PyTorch 튜토리얼입니다.</p>
          <a class="with-right-arrow" href="https://tutorials.pytorch.kr">튜토리얼로 이동</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>커뮤니티</h2>
          <p>다른 사용자들과 의견을 나눠보세요!</p>
          <a class="with-right-arrow" href="https://discuss.pytorch.kr">커뮤니티로 이동</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.kr/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org">PyTorch 홈페이지 (공식)</a></li>
            <li><a href="https://pytorch.org" target="_blank">공식 홈페이지</a></li>
            <li><a href="https://pytorch.org/tutorials" target="_blank">공식 튜토리얼</a></li>
            <li><a href="https://pytorch.org/docs" target="_blank">공식 문서</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="">한국 사용자 모임</a></li>
            <li><a href="https://pytorch.kr/about">사이트 소개</a></li>
            <li><a href="https://tutorials.pytorch.kr/">한국어 튜토리얼</a></li>
            <li><a href="https://github.com/9bow/PyTorch-tutorials-kr" target="_blank">한국어 튜토리얼 저장소</a></li>
          </ul>
        </div>

      </div>
      <div class="trademark-disclaimer">
        <ul>
          <li>이 사이트는 PyTorch 한국 사용자 커뮤니티로 Facebook, Inc에서 운영하는 사이트가 아닙니다. PyTorch, PyTorch 로고 및 모든 관련 표기는 Facebook, Inc의 상표입니다.</li>
          <li>This site is a user community and is not operated by Facebook, Inc. PyTorch, the PyTorch logo and any related marks are trademarks of Facebook, Inc.</li>
        </ul>
      </div>
    </div>
  </footer>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.kr/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.kr/get-started">시작하기</a>
          </li>

          <li class="active">
            <a href="https://tutorials.pytorch.kr">튜토리얼</a>
          </li>

          <li>
            <a href="https://pytorch.kr/hub">허브</a>
          </li>

          <li>
            <a href="https://discuss.pytorch.kr">커뮤니티</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>