docs/beginner/hyperparameter_tuning_tutorial.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Hyperparameter tuning with Ray Tune &mdash; PyTorch Tutorials 1.10.2+cu102 documentation</title>
  

  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="../_static/copybutton.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="Parametrizations Tutorial" href="../intermediate/parametrizations.html" />
    <link rel="prev" title="PyTorch Profiler With TensorBoard" href="../intermediate/tensorboard_profiler_tutorial.html" /> 

  
  <script src="../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.kr/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.kr/get-started">시작하기</a>
          </li>

          <li class="active">
            <a href="https://tutorials.pytorch.kr">튜토리얼</a>
          </li>

          <li>
            <a href="https://pytorch.kr/hub">허브</a>
          </li>

          <li>
            <a href="https://discuss.pytorch.kr">커뮤니티</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">

   
    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

                <div class="version">
                  1.10.2+cu102
                </div>
              
            
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search Tutorials" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
              <p class="caption"><span class="caption-text">파이토치(PyTorch) 레시피</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../recipes/recipes_index.html">모든 레시피 보기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../prototype/prototype_index.html">모든 프로토타입 레시피 보기</a></li>
</ul>
<p class="caption"><span class="caption-text">파이토치(PyTorch) 시작하기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="basics/intro.html">파이토치(PyTorch) 기본 익히기</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/quickstart_tutorial.html">빠른 시작(Quickstart)</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/tensorqs_tutorial.html">텐서(Tensor)</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/data_tutorial.html">Dataset과 DataLoader</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/transforms_tutorial.html">변형(Transform)</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/buildmodel_tutorial.html">신경망 모델 구성하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/autogradqs_tutorial.html"><code class="docutils literal notranslate"><span class="pre">torch.autograd</span></code>를 사용한 자동 미분</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/optimization_tutorial.html">모델 매개변수 최적화하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="basics/saveloadrun_tutorial.html">모델 저장하고 불러오기</a></li>
</ul>
<p class="caption"><span class="caption-text">Introduction to PyTorch on YouTube</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="introyt.html">Introduction to PyTorch - YouTube Series</a></li>
<li class="toctree-l1"><a class="reference internal" href="introyt/introyt1_tutorial.html">Introduction to PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="introyt/tensors_deeper_tutorial.html">Introduction to PyTorch Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="introyt/autogradyt_tutorial.html">The Fundamentals of Autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="introyt/modelsyt_tutorial.html">Building Models with PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="introyt/tensorboardyt_tutorial.html">PyTorch TensorBoard Support</a></li>
<li class="toctree-l1"><a class="reference internal" href="introyt/trainingyt.html">Training with PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="introyt/captumyt.html">Model Understanding with Captum</a></li>
</ul>
<p class="caption"><span class="caption-text">파이토치(PyTorch) 배우기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="deep_learning_60min_blitz.html">PyTorch로 딥러닝하기: 60분만에 끝장내기</a></li>
<li class="toctree-l1"><a class="reference internal" href="pytorch_with_examples.html">예제로 배우는 파이토치(PyTorch)</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn_tutorial.html"><cite>torch.nn</cite> 이 <em>실제로</em> 무엇인가요?</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/tensorboard_tutorial.html">TensorBoard로 모델, 데이터, 학습 시각화하기</a></li>
</ul>
<p class="caption"><span class="caption-text">이미지/비디오</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/torchvision_tutorial.html">TorchVision 객체 검출 미세조정(Finetuning) 튜토리얼</a></li>
<li class="toctree-l1"><a class="reference internal" href="transfer_learning_tutorial.html">컴퓨터 비전(Vision)을 위한 전이학습(Transfer Learning)</a></li>
<li class="toctree-l1"><a class="reference internal" href="fgsm_tutorial.html">적대적 예제 생성(Adversarial Example Generation)</a></li>
<li class="toctree-l1"><a class="reference internal" href="dcgan_faces_tutorial.html">DCGAN 튜토리얼</a></li>
<li class="toctree-l1"><a class="reference internal" href="vt_tutorial.html">배포를 위한 비전 트랜스포머(Vision Transformer) 모델 최적화하기</a></li>
</ul>
<p class="caption"><span class="caption-text">오디오</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="audio_io_tutorial.html">Audio I/O</a></li>
<li class="toctree-l1"><a class="reference internal" href="audio_resampling_tutorial.html">Audio Resampling</a></li>
<li class="toctree-l1"><a class="reference internal" href="audio_data_augmentation_tutorial.html">Audio Data Augmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="audio_feature_extractions_tutorial.html">Audio Feature Extractions</a></li>
<li class="toctree-l1"><a class="reference internal" href="audio_feature_augmentation_tutorial.html">Audio Feature Augmentation</a></li>
<li class="toctree-l1"><a class="reference internal" href="audio_datasets_tutorial.html">Audio Datasets</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/speech_recognition_pipeline_tutorial.html">Speech Recognition with Wav2Vec2</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/speech_command_classification_with_torchaudio_tutorial.html">Speech Command Classification with torchaudio</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/text_to_speech_with_torchaudio.html">Text-to-speech with torchaudio</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/forced_alignment_with_torchaudio_tutorial.html">Forced Alignment with Wav2Vec2</a></li>
</ul>
<p class="caption"><span class="caption-text">텍스트</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="transformer_tutorial.html">nn.Transformer 와 TorchText 로 시퀀스-투-시퀀스(Sequence-to-Sequence) 모델링하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/char_rnn_classification_tutorial.html">기초부터 시작하는 NLP: 문자-단위 RNN으로 이름 분류하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/char_rnn_generation_tutorial.html">기초부터 시작하는 NLP:  문자-단위 RNN으로 이름 생성하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/seq2seq_translation_tutorial.html">기초부터 시작하는 NLP: Sequence to Sequence 네트워크와 Attention을 이용한 번역</a></li>
<li class="toctree-l1"><a class="reference internal" href="text_sentiment_ngrams_tutorial.html">torchtext 라이브러리로 텍스트 분류하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="translation_transformer.html">nn.Transformer와 torchtext로 언어 번역하기</a></li>
</ul>
<p class="caption"><span class="caption-text">강화학습</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/reinforcement_q_learning.html">강화 학습 (DQN) 튜토리얼</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/mario_rl_tutorial.html">Train a Mario-playing RL Agent</a></li>
</ul>
<p class="caption"><span class="caption-text">PyTorch 모델을 프로덕션 환경에 배포하기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/flask_rest_api_tutorial.html">Flask를 사용하여 Python에서 PyTorch를 REST API로 배포하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="Intro_to_TorchScript_tutorial.html">TorchScript 소개</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_export.html">C++에서 TorchScript 모델 로딩하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/super_resolution_with_onnxruntime.html">(선택) PyTorch 모델을 ONNX으로 변환하고 ONNX 런타임에서 실행하기</a></li>
</ul>
<p class="caption"><span class="caption-text">Code Transforms with FX</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/fx_conv_bn_fuser.html">(베타) FX에서 합성곱/배치 정규화(Convolution/Batch Norm) 결합기(Fuser) 만들기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/fx_profiling_tutorial.html">(beta) Building a Simple CPU Performance Profiler with FX</a></li>
</ul>
<p class="caption"><span class="caption-text">프론트엔드 API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/memory_format_tutorial.html">(베타) PyTorch를 사용한 Channels Last 메모리 형식</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/forward_ad_usage.html">Forward-mode Automatic Differentiation (Beta)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_frontend.html">PyTorch C++ 프론트엔드 사용하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/torch-script-parallelism.html">TorchScript의 동적 병렬 처리(Dynamic Parallelism)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_autograd.html">C++ 프론트엔드의 자동 미분 (autograd)</a></li>
</ul>
<p class="caption"><span class="caption-text">PyTorch 확장하기</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/custom_function_double_backward_tutorial.html">Double Backward with Custom Functions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/custom_function_conv_bn_tutorial.html">Fusing Convolution and Batch Norm using Custom Function</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/cpp_extension.html">Custom C++ and CUDA Extensions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/torch_script_custom_ops.html">Extending TorchScript with Custom C++ Operators</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/torch_script_custom_classes.html">커스텀 C++ 클래스로 TorchScript 확장하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/dispatcher.html">Registering a Dispatched Operator in C++</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/extend_dispatcher.html">Extending dispatcher for a new backend in C++</a></li>
</ul>
<p class="caption"><span class="caption-text">모델 최적화</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="profiler.html">PyTorch 모듈 프로파일링 하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/tensorboard_profiler_tutorial.html">PyTorch Profiler With TensorBoard</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Hyperparameter tuning with Ray Tune</a></li>
<li class="toctree-l1"><a class="reference internal" href="vt_tutorial.html">배포를 위한 비전 트랜스포머(Vision Transformer) 모델 최적화하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/parametrizations.html">Parametrizations Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/pruning_tutorial.html">가지치기 기법(Pruning) 튜토리얼</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/dynamic_quantization_tutorial.html">(베타) LSTM 기반 단어 단위 언어 모델의 동적 양자화</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/dynamic_quantization_bert_tutorial.html">(베타) BERT 모델 동적 양자화하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/quantized_transfer_learning_tutorial.html">(베타) 컴퓨터 비전 튜토리얼을 위한 양자화된 전이학습(Quantized Transfer Learning)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/static_quantization_tutorial.html">(beta) Static Quantization with Eager Mode in PyTorch</a></li>
</ul>
<p class="caption"><span class="caption-text">병렬 및 분산 학습</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="dist_overview.html">PyTorch Distributed Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/model_parallel_tutorial.html">단일 머신을 사용한 모델 병렬화 모범 사례</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/ddp_tutorial.html">분산 데이터 병렬 처리 시작하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/dist_tuto.html">PyTorch로 분산 어플리케이션 개발하기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/rpc_tutorial.html">Getting Started with Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/rpc_param_server_tutorial.html">Implementing a Parameter Server Using Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/dist_pipeline_parallel_tutorial.html">Distributed Pipeline Parallelism Using RPC</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/rpc_async_execution.html">Implementing Batch RPC Processing Using Asynchronous Executions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/rpc_ddp_tutorial.html">분산 데이터 병렬(DDP)과 분산 RPC 프레임워크 결합</a></li>
<li class="toctree-l1"><a class="reference internal" href="../intermediate/pipeline_tutorial.html">파이프라인 병렬화로 트랜스포머 모델 학습시키기</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/ddp_pipeline.html">분산 데이터 병렬 처리와 병렬 처리 파이프라인을 사용한 트랜스포머 모델 학습</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/generic_join.html">Distributed Training with Uneven Inputs Using the Join Context Manager</a></li>
</ul>
<p class="caption"><span class="caption-text">Mobile</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="deeplabv3_on_ios.html">iOS에서의 이미지 분할 DeepLapV3</a></li>
<li class="toctree-l1"><a class="reference internal" href="deeplabv3_on_android.html">안드로이드에서의 이미지 분할 DeepLapV3</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          

<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../index.html">
          
            Tutorials
          
        </a> &gt;
      </li>

        
      <li>Hyperparameter tuning with Ray Tune</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="../_sources/beginner/hyperparameter_tuning_tutorial.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <div class="pytorch-call-to-action-links">
            <div id="tutorial-type">beginner/hyperparameter_tuning_tutorial</div>

            <div id="google-colab-link">
              <img class="call-to-action-img" src="../_static/images/pytorch-colab.svg"/>
              <div class="call-to-action-desktop-view">Run in Google Colab</div>
              <div class="call-to-action-mobile-view">Colab</div>
            </div>
            <div id="download-notebook-link">
              <img class="call-to-action-notebook-img" src="../_static/images/pytorch-download.svg"/>
              <div class="call-to-action-desktop-view">Download Notebook</div>
              <div class="call-to-action-mobile-view">Notebook</div>
            </div>
            <div id="github-view-link">
              <img class="call-to-action-img" src="../_static/images/pytorch-github.svg"/>
              <div class="call-to-action-desktop-view">View on GitHub</div>
              <div class="call-to-action-mobile-view">GitHub</div>
            </div>
          </div>

        
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <div class="sphx-glr-download-link-note admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Click <a class="reference internal" href="#sphx-glr-download-beginner-hyperparameter-tuning-tutorial-py"><span class="std std-ref">here</span></a> to download the full example code</p>
</div>
<div class="sphx-glr-example-title section" id="hyperparameter-tuning-with-ray-tune">
<span id="sphx-glr-beginner-hyperparameter-tuning-tutorial-py"></span><h1>Hyperparameter tuning with Ray Tune<a class="headerlink" href="#hyperparameter-tuning-with-ray-tune" title="Permalink to this headline">¶</a></h1>
<p>Hyperparameter tuning can make the difference between an average model and a highly
accurate one. Often simple things like choosing a different learning rate or changing
a network layer size can have a dramatic impact on your model performance.</p>
<p>Fortunately, there are tools that help with finding the best combination of parameters.
<a class="reference external" href="https://docs.ray.io/en/latest/tune.html">Ray Tune</a> is an industry standard tool for
distributed hyperparameter tuning. Ray Tune includes the latest hyperparameter search
algorithms, integrates with TensorBoard and other analysis libraries, and natively
supports distributed training through <a class="reference external" href="https://ray.io/">Ray’s distributed machine learning engine</a>.</p>
<p>In this tutorial, we will show you how to integrate Ray Tune into your PyTorch
training workflow. We will extend <a class="reference external" href="https://tutorials.pytorch.kr/beginner/blitz/cifar10_tutorial.html">this tutorial from the PyTorch documentation</a> for training
a CIFAR10 image classifier.</p>
<p>As you will see, we only need to add some slight modifications. In particular, we
need to</p>
<ol class="arabic simple">
<li>wrap data loading and training in functions,</li>
<li>make some network parameters configurable,</li>
<li>add checkpointing (optional),</li>
<li>and define the search space for the model tuning</li>
</ol>
<div class="line-block">
<div class="line"><br /></div>
</div>
<p>To run this tutorial, please make sure the following packages are
installed:</p>
<ul class="simple">
<li><code class="docutils literal notranslate"><span class="pre">ray[tune]</span></code>: Distributed hyperparameter tuning library</li>
<li><code class="docutils literal notranslate"><span class="pre">torchvision</span></code>: For the data transformers</li>
</ul>
<div class="section" id="setup-imports">
<h2>Setup / Imports<a class="headerlink" href="#setup-imports" title="Permalink to this headline">¶</a></h2>
<p>Let’s start with the imports:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">functools</span> <span class="kn">import</span> <span class="n">partial</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">import</span> <span class="nn">torch.nn</span> <span class="k">as</span> <span class="nn">nn</span>
<span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>
<span class="kn">import</span> <span class="nn">torch.optim</span> <span class="k">as</span> <span class="nn">optim</span>
<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="kn">import</span> <span class="n">random_split</span>
<span class="kn">import</span> <span class="nn">torchvision</span>
<span class="kn">import</span> <span class="nn">torchvision.transforms</span> <span class="k">as</span> <span class="nn">transforms</span>
<span class="kn">from</span> <span class="nn">ray</span> <span class="kn">import</span> <span class="n">tune</span>
<span class="kn">from</span> <span class="nn">ray.tune</span> <span class="kn">import</span> <span class="n">CLIReporter</span>
<span class="kn">from</span> <span class="nn">ray.tune.schedulers</span> <span class="kn">import</span> <span class="n">ASHAScheduler</span>
</pre></div>
</div>
<p>Most of the imports are needed for building the PyTorch model. Only the last three
imports are for Ray Tune.</p>
</div>
<div class="section" id="data-loaders">
<h2>Data loaders<a class="headerlink" href="#data-loaders" title="Permalink to this headline">¶</a></h2>
<p>We wrap the data loaders in their own function and pass a global data directory.
This way we can share a data directory between different trials.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">load_data</span><span class="p">(</span><span class="n">data_dir</span><span class="o">=</span><span class="s2">&quot;./data&quot;</span><span class="p">):</span>
    <span class="n">transform</span> <span class="o">=</span> <span class="n">transforms</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span>
        <span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(),</span>
        <span class="n">transforms</span><span class="o">.</span><span class="n">Normalize</span><span class="p">((</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">),</span> <span class="p">(</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">))</span>
    <span class="p">])</span>

    <span class="n">trainset</span> <span class="o">=</span> <span class="n">torchvision</span><span class="o">.</span><span class="n">datasets</span><span class="o">.</span><span class="n">CIFAR10</span><span class="p">(</span>
        <span class="n">root</span><span class="o">=</span><span class="n">data_dir</span><span class="p">,</span> <span class="n">train</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">download</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">transform</span><span class="o">=</span><span class="n">transform</span><span class="p">)</span>

    <span class="n">testset</span> <span class="o">=</span> <span class="n">torchvision</span><span class="o">.</span><span class="n">datasets</span><span class="o">.</span><span class="n">CIFAR10</span><span class="p">(</span>
        <span class="n">root</span><span class="o">=</span><span class="n">data_dir</span><span class="p">,</span> <span class="n">train</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">download</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">transform</span><span class="o">=</span><span class="n">transform</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">trainset</span><span class="p">,</span> <span class="n">testset</span>
</pre></div>
</div>
</div>
<div class="section" id="configurable-neural-network">
<h2>Configurable neural network<a class="headerlink" href="#configurable-neural-network" title="Permalink to this headline">¶</a></h2>
<p>We can only tune those parameters that are configurable. In this example, we can specify
the layer sizes of the fully connected layers:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">Net</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">l1</span><span class="o">=</span><span class="mi">120</span><span class="p">,</span> <span class="n">l2</span><span class="o">=</span><span class="mi">84</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">Net</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">MaxPool2d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">16</span> <span class="o">*</span> <span class="mi">5</span> <span class="o">*</span> <span class="mi">5</span><span class="p">,</span> <span class="n">l1</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">l1</span><span class="p">,</span> <span class="n">l2</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc3</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="n">l2</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv1</span><span class="p">(</span><span class="n">x</span><span class="p">)))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv2</span><span class="p">(</span><span class="n">x</span><span class="p">)))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">x</span><span class="o">.</span><span class="n">view</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">16</span> <span class="o">*</span> <span class="mi">5</span> <span class="o">*</span> <span class="mi">5</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fc1</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fc2</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fc3</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>
</pre></div>
</div>
</div>
<div class="section" id="the-train-function">
<h2>The train function<a class="headerlink" href="#the-train-function" title="Permalink to this headline">¶</a></h2>
<p>Now it gets interesting, because we introduce some changes to the example <a class="reference external" href="https://tutorials.pytorch.kr/beginner/blitz/cifar10_tutorial.html">from the PyTorch
documentation</a>.</p>
<p>We wrap the training script in a function <code class="docutils literal notranslate"><span class="pre">train_cifar(config,</span> <span class="pre">checkpoint_dir=None,</span> <span class="pre">data_dir=None)</span></code>.
As you can guess, the <code class="docutils literal notranslate"><span class="pre">config</span></code> parameter will receive the hyperparameters we would like to
train with. The <code class="docutils literal notranslate"><span class="pre">checkpoint_dir</span></code> parameter is used to restore checkpoints. The <code class="docutils literal notranslate"><span class="pre">data_dir</span></code> specifies
the directory where we load and store the data, so multiple runs can share the same data source.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">net</span> <span class="o">=</span> <span class="n">Net</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;l1&quot;</span><span class="p">],</span> <span class="n">config</span><span class="p">[</span><span class="s2">&quot;l2&quot;</span><span class="p">])</span>

<span class="k">if</span> <span class="n">checkpoint_dir</span><span class="p">:</span>
    <span class="n">model_state</span><span class="p">,</span> <span class="n">optimizer_state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">load</span><span class="p">(</span>
        <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">checkpoint_dir</span><span class="p">,</span> <span class="s2">&quot;checkpoint&quot;</span><span class="p">))</span>
    <span class="n">net</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">model_state</span><span class="p">)</span>
    <span class="n">optimizer</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">optimizer_state</span><span class="p">)</span>
</pre></div>
</div>
<p>The learning rate of the optimizer is made configurable, too:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">net</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;lr&quot;</span><span class="p">],</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
</pre></div>
</div>
<p>We also split the training data into a training and validation subset. We thus train on
80% of the data and calculate the validation loss on the remaining 20%. The batch sizes
with which we iterate through the training and test sets are configurable as well.</p>
<div class="section" id="adding-multi-gpu-support-with-dataparallel">
<h3>Adding (multi) GPU support with DataParallel<a class="headerlink" href="#adding-multi-gpu-support-with-dataparallel" title="Permalink to this headline">¶</a></h3>
<p>Image classification benefits largely from GPUs. Luckily, we can continue to use
PyTorch’s abstractions in Ray Tune. Thus, we can wrap our model in <code class="docutils literal notranslate"><span class="pre">nn.DataParallel</span></code>
to support data parallel training on multiple GPUs:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">device</span> <span class="o">=</span> <span class="s2">&quot;cpu&quot;</span>
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
    <span class="n">device</span> <span class="o">=</span> <span class="s2">&quot;cuda:0&quot;</span>
    <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">device_count</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
        <span class="n">net</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">DataParallel</span><span class="p">(</span><span class="n">net</span><span class="p">)</span>
<span class="n">net</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
</pre></div>
</div>
<p>By using a <code class="docutils literal notranslate"><span class="pre">device</span></code> variable we make sure that training also works when we have
no GPUs available. PyTorch requires us to send our data to the GPU memory explicitly,
like this:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">data</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">trainloader</span><span class="p">,</span> <span class="mi">0</span><span class="p">):</span>
    <span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">data</span>
    <span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">inputs</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">),</span> <span class="n">labels</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
</pre></div>
</div>
<p>The code now supports training on CPUs, on a single GPU, and on multiple GPUs. Notably, Ray
also supports <a class="reference external" href="https://docs.ray.io/en/master/using-ray-with-gpus.html#fractional-gpus">fractional GPUs</a>
so we can share GPUs among trials, as long as the model still fits on the GPU memory. We’ll come back
to that later.</p>
</div>
<div class="section" id="communicating-with-ray-tune">
<h3>Communicating with Ray Tune<a class="headerlink" href="#communicating-with-ray-tune" title="Permalink to this headline">¶</a></h3>
<p>The most interesting part is the communication with Ray Tune:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">with</span> <span class="n">tune</span><span class="o">.</span><span class="n">checkpoint_dir</span><span class="p">(</span><span class="n">epoch</span><span class="p">)</span> <span class="k">as</span> <span class="n">checkpoint_dir</span><span class="p">:</span>
    <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">checkpoint_dir</span><span class="p">,</span> <span class="s2">&quot;checkpoint&quot;</span><span class="p">)</span>
    <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">((</span><span class="n">net</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">state_dict</span><span class="p">()),</span> <span class="n">path</span><span class="p">)</span>

<span class="n">tune</span><span class="o">.</span><span class="n">report</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="p">(</span><span class="n">val_loss</span> <span class="o">/</span> <span class="n">val_steps</span><span class="p">),</span> <span class="n">accuracy</span><span class="o">=</span><span class="n">correct</span> <span class="o">/</span> <span class="n">total</span><span class="p">)</span>
</pre></div>
</div>
<p>Here we first save a checkpoint and then report some metrics back to Ray Tune. Specifically,
we send the validation loss and accuracy back to Ray Tune. Ray Tune can then use these metrics
to decide which hyperparameter configuration lead to the best results. These metrics
can also be used to stop bad performing trials early in order to avoid wasting
resources on those trials.</p>
<p>The checkpoint saving is optional, however, it is necessary if we wanted to use advanced
schedulers like
<a class="reference external" href="https://docs.ray.io/en/master/tune/tutorials/tune-advanced-tutorial.html">Population Based Training</a>.
Also, by saving the checkpoint we can later load the trained models and validate them
on a test set.</p>
</div>
<div class="section" id="full-training-function">
<h3>Full training function<a class="headerlink" href="#full-training-function" title="Permalink to this headline">¶</a></h3>
<p>The full code example looks like this:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">train_cifar</span><span class="p">(</span><span class="n">config</span><span class="p">,</span> <span class="n">checkpoint_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">data_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="n">net</span> <span class="o">=</span> <span class="n">Net</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;l1&quot;</span><span class="p">],</span> <span class="n">config</span><span class="p">[</span><span class="s2">&quot;l2&quot;</span><span class="p">])</span>

    <span class="n">device</span> <span class="o">=</span> <span class="s2">&quot;cpu&quot;</span>
    <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
        <span class="n">device</span> <span class="o">=</span> <span class="s2">&quot;cuda:0&quot;</span>
        <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">device_count</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
            <span class="n">net</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">DataParallel</span><span class="p">(</span><span class="n">net</span><span class="p">)</span>
    <span class="n">net</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>

    <span class="n">criterion</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">CrossEntropyLoss</span><span class="p">()</span>
    <span class="n">optimizer</span> <span class="o">=</span> <span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">net</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;lr&quot;</span><span class="p">],</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>

    <span class="k">if</span> <span class="n">checkpoint_dir</span><span class="p">:</span>
        <span class="n">model_state</span><span class="p">,</span> <span class="n">optimizer_state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">load</span><span class="p">(</span>
            <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">checkpoint_dir</span><span class="p">,</span> <span class="s2">&quot;checkpoint&quot;</span><span class="p">))</span>
        <span class="n">net</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">model_state</span><span class="p">)</span>
        <span class="n">optimizer</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">optimizer_state</span><span class="p">)</span>

    <span class="n">trainset</span><span class="p">,</span> <span class="n">testset</span> <span class="o">=</span> <span class="n">load_data</span><span class="p">(</span><span class="n">data_dir</span><span class="p">)</span>

    <span class="n">test_abs</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">trainset</span><span class="p">)</span> <span class="o">*</span> <span class="mf">0.8</span><span class="p">)</span>
    <span class="n">train_subset</span><span class="p">,</span> <span class="n">val_subset</span> <span class="o">=</span> <span class="n">random_split</span><span class="p">(</span>
        <span class="n">trainset</span><span class="p">,</span> <span class="p">[</span><span class="n">test_abs</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">trainset</span><span class="p">)</span> <span class="o">-</span> <span class="n">test_abs</span><span class="p">])</span>

    <span class="n">trainloader</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span>
        <span class="n">train_subset</span><span class="p">,</span>
        <span class="n">batch_size</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;batch_size&quot;</span><span class="p">]),</span>
        <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
        <span class="n">num_workers</span><span class="o">=</span><span class="mi">8</span><span class="p">)</span>
    <span class="n">valloader</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span>
        <span class="n">val_subset</span><span class="p">,</span>
        <span class="n">batch_size</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;batch_size&quot;</span><span class="p">]),</span>
        <span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
        <span class="n">num_workers</span><span class="o">=</span><span class="mi">8</span><span class="p">)</span>

    <span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>  <span class="c1"># loop over the dataset multiple times</span>
        <span class="n">running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
        <span class="n">epoch_steps</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">data</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">trainloader</span><span class="p">,</span> <span class="mi">0</span><span class="p">):</span>
            <span class="c1"># get the inputs; data is a list of [inputs, labels]</span>
            <span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">data</span>
            <span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">inputs</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">),</span> <span class="n">labels</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>

            <span class="c1"># zero the parameter gradients</span>
            <span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>

            <span class="c1"># forward + backward + optimize</span>
            <span class="n">outputs</span> <span class="o">=</span> <span class="n">net</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
            <span class="n">loss</span> <span class="o">=</span> <span class="n">criterion</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="n">labels</span><span class="p">)</span>
            <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
            <span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>

            <span class="c1"># print statistics</span>
            <span class="n">running_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>
            <span class="n">epoch_steps</span> <span class="o">+=</span> <span class="mi">1</span>
            <span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="mi">2000</span> <span class="o">==</span> <span class="mi">1999</span><span class="p">:</span>  <span class="c1"># print every 2000 mini-batches</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;[</span><span class="si">%d</span><span class="s2">, </span><span class="si">%5d</span><span class="s2">] loss: </span><span class="si">%.3f</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
                                                <span class="n">running_loss</span> <span class="o">/</span> <span class="n">epoch_steps</span><span class="p">))</span>
                <span class="n">running_loss</span> <span class="o">=</span> <span class="mf">0.0</span>

        <span class="c1"># Validation loss</span>
        <span class="n">val_loss</span> <span class="o">=</span> <span class="mf">0.0</span>
        <span class="n">val_steps</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="n">total</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="n">correct</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">data</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">valloader</span><span class="p">,</span> <span class="mi">0</span><span class="p">):</span>
            <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
                <span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">data</span>
                <span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">inputs</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">),</span> <span class="n">labels</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>

                <span class="n">outputs</span> <span class="o">=</span> <span class="n">net</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
                <span class="n">_</span><span class="p">,</span> <span class="n">predicted</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">outputs</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
                <span class="n">total</span> <span class="o">+=</span> <span class="n">labels</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
                <span class="n">correct</span> <span class="o">+=</span> <span class="p">(</span><span class="n">predicted</span> <span class="o">==</span> <span class="n">labels</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>

                <span class="n">loss</span> <span class="o">=</span> <span class="n">criterion</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="n">labels</span><span class="p">)</span>
                <span class="n">val_loss</span> <span class="o">+=</span> <span class="n">loss</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">()</span>
                <span class="n">val_steps</span> <span class="o">+=</span> <span class="mi">1</span>

        <span class="k">with</span> <span class="n">tune</span><span class="o">.</span><span class="n">checkpoint_dir</span><span class="p">(</span><span class="n">epoch</span><span class="p">)</span> <span class="k">as</span> <span class="n">checkpoint_dir</span><span class="p">:</span>
            <span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">checkpoint_dir</span><span class="p">,</span> <span class="s2">&quot;checkpoint&quot;</span><span class="p">)</span>
            <span class="n">torch</span><span class="o">.</span><span class="n">save</span><span class="p">((</span><span class="n">net</span><span class="o">.</span><span class="n">state_dict</span><span class="p">(),</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">state_dict</span><span class="p">()),</span> <span class="n">path</span><span class="p">)</span>

        <span class="n">tune</span><span class="o">.</span><span class="n">report</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="p">(</span><span class="n">val_loss</span> <span class="o">/</span> <span class="n">val_steps</span><span class="p">),</span> <span class="n">accuracy</span><span class="o">=</span><span class="n">correct</span> <span class="o">/</span> <span class="n">total</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Finished Training&quot;</span><span class="p">)</span>
</pre></div>
</div>
<p>As you can see, most of the code is adapted directly from the original example.</p>
</div>
</div>
<div class="section" id="test-set-accuracy">
<h2>Test set accuracy<a class="headerlink" href="#test-set-accuracy" title="Permalink to this headline">¶</a></h2>
<p>Commonly the performance of a machine learning model is tested on a hold-out test
set with data that has not been used for training the model. We also wrap this in a
function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">test_accuracy</span><span class="p">(</span><span class="n">net</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="s2">&quot;cpu&quot;</span><span class="p">):</span>
    <span class="n">trainset</span><span class="p">,</span> <span class="n">testset</span> <span class="o">=</span> <span class="n">load_data</span><span class="p">()</span>

    <span class="n">testloader</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span>
        <span class="n">testset</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">num_workers</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>

    <span class="n">correct</span> <span class="o">=</span> <span class="mi">0</span>
    <span class="n">total</span> <span class="o">=</span> <span class="mi">0</span>
    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
        <span class="k">for</span> <span class="n">data</span> <span class="ow">in</span> <span class="n">testloader</span><span class="p">:</span>
            <span class="n">images</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">data</span>
            <span class="n">images</span><span class="p">,</span> <span class="n">labels</span> <span class="o">=</span> <span class="n">images</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">),</span> <span class="n">labels</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
            <span class="n">outputs</span> <span class="o">=</span> <span class="n">net</span><span class="p">(</span><span class="n">images</span><span class="p">)</span>
            <span class="n">_</span><span class="p">,</span> <span class="n">predicted</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">outputs</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
            <span class="n">total</span> <span class="o">+=</span> <span class="n">labels</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
            <span class="n">correct</span> <span class="o">+=</span> <span class="p">(</span><span class="n">predicted</span> <span class="o">==</span> <span class="n">labels</span><span class="p">)</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span><span class="o">.</span><span class="n">item</span><span class="p">()</span>

    <span class="k">return</span> <span class="n">correct</span> <span class="o">/</span> <span class="n">total</span>
</pre></div>
</div>
<p>The function also expects a <code class="docutils literal notranslate"><span class="pre">device</span></code> parameter, so we can do the
test set validation on a GPU.</p>
</div>
<div class="section" id="configuring-the-search-space">
<h2>Configuring the search space<a class="headerlink" href="#configuring-the-search-space" title="Permalink to this headline">¶</a></h2>
<p>Lastly, we need to define Ray Tune’s search space. Here is an example:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">config</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s2">&quot;l1&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">sample_from</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="mi">2</span><span class="o">**</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">9</span><span class="p">)),</span>
    <span class="s2">&quot;l2&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">sample_from</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="mi">2</span><span class="o">**</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">9</span><span class="p">)),</span>
    <span class="s2">&quot;lr&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">loguniform</span><span class="p">(</span><span class="mf">1e-4</span><span class="p">,</span> <span class="mf">1e-1</span><span class="p">),</span>
    <span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">choice</span><span class="p">([</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">16</span><span class="p">])</span>
<span class="p">}</span>
</pre></div>
</div>
<p>The <code class="docutils literal notranslate"><span class="pre">tune.sample_from()</span></code> function makes it possible to define your own sample
methods to obtain hyperparameters. In this example, the <code class="docutils literal notranslate"><span class="pre">l1</span></code> and <code class="docutils literal notranslate"><span class="pre">l2</span></code> parameters
should be powers of 2 between 4 and 256, so either 4, 8, 16, 32, 64, 128, or 256.
The <code class="docutils literal notranslate"><span class="pre">lr</span></code> (learning rate) should be uniformly sampled between 0.0001 and 0.1. Lastly,
the batch size is a choice between 2, 4, 8, and 16.</p>
<p>At each trial, Ray Tune will now randomly sample a combination of parameters from these
search spaces. It will then train a number of models in parallel and find the best
performing one among these. We also use the <code class="docutils literal notranslate"><span class="pre">ASHAScheduler</span></code> which will terminate bad
performing trials early.</p>
<p>We wrap the <code class="docutils literal notranslate"><span class="pre">train_cifar</span></code> function with <code class="docutils literal notranslate"><span class="pre">functools.partial</span></code> to set the constant
<code class="docutils literal notranslate"><span class="pre">data_dir</span></code> parameter. We can also tell Ray Tune what resources should be
available for each trial:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">gpus_per_trial</span> <span class="o">=</span> <span class="mi">2</span>
<span class="c1"># ...</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">tune</span><span class="o">.</span><span class="n">run</span><span class="p">(</span>
    <span class="n">partial</span><span class="p">(</span><span class="n">train_cifar</span><span class="p">,</span> <span class="n">data_dir</span><span class="o">=</span><span class="n">data_dir</span><span class="p">),</span>
    <span class="n">resources_per_trial</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;cpu&quot;</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span> <span class="s2">&quot;gpu&quot;</span><span class="p">:</span> <span class="n">gpus_per_trial</span><span class="p">},</span>
    <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
    <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
    <span class="n">scheduler</span><span class="o">=</span><span class="n">scheduler</span><span class="p">,</span>
    <span class="n">progress_reporter</span><span class="o">=</span><span class="n">reporter</span><span class="p">,</span>
    <span class="n">checkpoint_at_end</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</pre></div>
</div>
<p>You can specify the number of CPUs, which are then available e.g.
to increase the <code class="docutils literal notranslate"><span class="pre">num_workers</span></code> of the PyTorch <code class="docutils literal notranslate"><span class="pre">DataLoader</span></code> instances. The selected
number of GPUs are made visible to PyTorch in each trial. Trials do not have access to
GPUs that haven’t been requested for them - so you don’t have to care about two trials
using the same set of resources.</p>
<p>Here we can also specify fractional GPUs, so something like <code class="docutils literal notranslate"><span class="pre">gpus_per_trial=0.5</span></code> is
completely valid. The trials will then share GPUs among each other.
You just have to make sure that the models still fit in the GPU memory.</p>
<p>After training the models, we will find the best performing one and load the trained
network from the checkpoint file. We then obtain the test set accuracy and report
everything by printing.</p>
<p>The full main function looks like this:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">num_samples</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">max_num_epochs</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">gpus_per_trial</span><span class="o">=</span><span class="mi">2</span><span class="p">):</span>
    <span class="n">data_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s2">&quot;./data&quot;</span><span class="p">)</span>
    <span class="n">load_data</span><span class="p">(</span><span class="n">data_dir</span><span class="p">)</span>
    <span class="n">config</span> <span class="o">=</span> <span class="p">{</span>
        <span class="s2">&quot;l1&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">sample_from</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="mi">2</span> <span class="o">**</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">9</span><span class="p">)),</span>
        <span class="s2">&quot;l2&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">sample_from</span><span class="p">(</span><span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="mi">2</span> <span class="o">**</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">9</span><span class="p">)),</span>
        <span class="s2">&quot;lr&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">loguniform</span><span class="p">(</span><span class="mf">1e-4</span><span class="p">,</span> <span class="mf">1e-1</span><span class="p">),</span>
        <span class="s2">&quot;batch_size&quot;</span><span class="p">:</span> <span class="n">tune</span><span class="o">.</span><span class="n">choice</span><span class="p">([</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">8</span><span class="p">,</span> <span class="mi">16</span><span class="p">])</span>
    <span class="p">}</span>
    <span class="n">scheduler</span> <span class="o">=</span> <span class="n">ASHAScheduler</span><span class="p">(</span>
        <span class="n">metric</span><span class="o">=</span><span class="s2">&quot;loss&quot;</span><span class="p">,</span>
        <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;min&quot;</span><span class="p">,</span>
        <span class="n">max_t</span><span class="o">=</span><span class="n">max_num_epochs</span><span class="p">,</span>
        <span class="n">grace_period</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
        <span class="n">reduction_factor</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
    <span class="n">reporter</span> <span class="o">=</span> <span class="n">CLIReporter</span><span class="p">(</span>
        <span class="c1"># parameter_columns=[&quot;l1&quot;, &quot;l2&quot;, &quot;lr&quot;, &quot;batch_size&quot;],</span>
        <span class="n">metric_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;accuracy&quot;</span><span class="p">,</span> <span class="s2">&quot;training_iteration&quot;</span><span class="p">])</span>
    <span class="n">result</span> <span class="o">=</span> <span class="n">tune</span><span class="o">.</span><span class="n">run</span><span class="p">(</span>
        <span class="n">partial</span><span class="p">(</span><span class="n">train_cifar</span><span class="p">,</span> <span class="n">data_dir</span><span class="o">=</span><span class="n">data_dir</span><span class="p">),</span>
        <span class="n">resources_per_trial</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;cpu&quot;</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span> <span class="s2">&quot;gpu&quot;</span><span class="p">:</span> <span class="n">gpus_per_trial</span><span class="p">},</span>
        <span class="n">config</span><span class="o">=</span><span class="n">config</span><span class="p">,</span>
        <span class="n">num_samples</span><span class="o">=</span><span class="n">num_samples</span><span class="p">,</span>
        <span class="n">scheduler</span><span class="o">=</span><span class="n">scheduler</span><span class="p">,</span>
        <span class="n">progress_reporter</span><span class="o">=</span><span class="n">reporter</span><span class="p">)</span>

    <span class="n">best_trial</span> <span class="o">=</span> <span class="n">result</span><span class="o">.</span><span class="n">get_best_trial</span><span class="p">(</span><span class="s2">&quot;loss&quot;</span><span class="p">,</span> <span class="s2">&quot;min&quot;</span><span class="p">,</span> <span class="s2">&quot;last&quot;</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Best trial config: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">best_trial</span><span class="o">.</span><span class="n">config</span><span class="p">))</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Best trial final validation loss: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
        <span class="n">best_trial</span><span class="o">.</span><span class="n">last_result</span><span class="p">[</span><span class="s2">&quot;loss&quot;</span><span class="p">]))</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Best trial final validation accuracy: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
        <span class="n">best_trial</span><span class="o">.</span><span class="n">last_result</span><span class="p">[</span><span class="s2">&quot;accuracy&quot;</span><span class="p">]))</span>

    <span class="n">best_trained_model</span> <span class="o">=</span> <span class="n">Net</span><span class="p">(</span><span class="n">best_trial</span><span class="o">.</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;l1&quot;</span><span class="p">],</span> <span class="n">best_trial</span><span class="o">.</span><span class="n">config</span><span class="p">[</span><span class="s2">&quot;l2&quot;</span><span class="p">])</span>
    <span class="n">device</span> <span class="o">=</span> <span class="s2">&quot;cpu&quot;</span>
    <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
        <span class="n">device</span> <span class="o">=</span> <span class="s2">&quot;cuda:0&quot;</span>
        <span class="k">if</span> <span class="n">gpus_per_trial</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
            <span class="n">best_trained_model</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">DataParallel</span><span class="p">(</span><span class="n">best_trained_model</span><span class="p">)</span>
    <span class="n">best_trained_model</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>

    <span class="n">best_checkpoint_dir</span> <span class="o">=</span> <span class="n">best_trial</span><span class="o">.</span><span class="n">checkpoint</span><span class="o">.</span><span class="n">value</span>
    <span class="n">model_state</span><span class="p">,</span> <span class="n">optimizer_state</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
        <span class="n">best_checkpoint_dir</span><span class="p">,</span> <span class="s2">&quot;checkpoint&quot;</span><span class="p">))</span>
    <span class="n">best_trained_model</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">model_state</span><span class="p">)</span>

    <span class="n">test_acc</span> <span class="o">=</span> <span class="n">test_accuracy</span><span class="p">(</span><span class="n">best_trained_model</span><span class="p">,</span> <span class="n">device</span><span class="p">)</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Best trial test set accuracy: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">test_acc</span><span class="p">))</span>


<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s2">&quot;__main__&quot;</span><span class="p">:</span>
    <span class="c1"># You can change the number of GPUs per trial here:</span>
    <span class="n">main</span><span class="p">(</span><span class="n">num_samples</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">max_num_epochs</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">gpus_per_trial</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
</pre></div>
</div>
<p class="sphx-glr-script-out">Out:</p>
<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to /workspace/ko-latest/beginner_source/data/cifar-10-python.tar.gz
Extracting /workspace/ko-latest/beginner_source/data/cifar-10-python.tar.gz to /workspace/ko-latest/beginner_source/data
Files already downloaded and verified
== Status ==
Memory usage on this node: 14.3/31.3 GiB
Using AsyncHyperBand: num_stopped=0
Bracket: Iter 8.000: None | Iter 4.000: None | Iter 2.000: None | Iter 1.000: None
Resources requested: 2.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (9 PENDING, 1 RUNNING)
+---------------------+----------+-------+--------------+------+------+-------------+
| Trial name          | status   | loc   |   batch_size |   l1 |   l2 |          lr |
|---------------------+----------+-------+--------------+------+------+-------------|
| DEFAULT_f201f_00000 | RUNNING  |       |           16 |  128 |   16 | 0.0173062   |
| DEFAULT_f201f_00001 | PENDING  |       |            2 |  128 |    8 | 0.000447305 |
| DEFAULT_f201f_00002 | PENDING  |       |            2 |  256 |  256 | 0.0119053   |
| DEFAULT_f201f_00003 | PENDING  |       |            2 |   32 |   16 | 0.00404483  |
| DEFAULT_f201f_00004 | PENDING  |       |            4 |    8 |   16 | 0.00134526  |
| DEFAULT_f201f_00005 | PENDING  |       |            2 |   16 |   16 | 0.00126487  |
| DEFAULT_f201f_00006 | PENDING  |       |            2 |  256 |   16 | 0.0404286   |
| DEFAULT_f201f_00007 | PENDING  |       |            8 |   64 |  256 | 0.000285913 |
| DEFAULT_f201f_00008 | PENDING  |       |            8 |   32 |   16 | 0.00172471  |
| DEFAULT_f201f_00009 | PENDING  |       |            8 |   64 |   16 | 0.0574224   |
+---------------------+----------+-------+--------------+------+------+-------------+


[2m[36m(pid=5085)[0m Files already downloaded and verified
[2m[36m(pid=5088)[0m Files already downloaded and verified
[2m[36m(pid=5090)[0m Files already downloaded and verified
[2m[36m(pid=5086)[0m Files already downloaded and verified
[2m[36m(pid=5085)[0m Files already downloaded and verified
[2m[36m(pid=5088)[0m Files already downloaded and verified
[2m[36m(pid=5090)[0m Files already downloaded and verified
[2m[36m(pid=5086)[0m Files already downloaded and verified
== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=0
Bracket: Iter 8.000: None | Iter 4.000: None | Iter 2.000: None | Iter 1.000: None
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (6 PENDING, 4 RUNNING)
+---------------------+----------+-------+--------------+------+------+-------------+
| Trial name          | status   | loc   |   batch_size |   l1 |   l2 |          lr |
|---------------------+----------+-------+--------------+------+------+-------------|
| DEFAULT_f201f_00000 | RUNNING  |       |           16 |  128 |   16 | 0.0173062   |
| DEFAULT_f201f_00001 | RUNNING  |       |            2 |  128 |    8 | 0.000447305 |
| DEFAULT_f201f_00002 | RUNNING  |       |            2 |  256 |  256 | 0.0119053   |
| DEFAULT_f201f_00003 | RUNNING  |       |            2 |   32 |   16 | 0.00404483  |
| DEFAULT_f201f_00004 | PENDING  |       |            4 |    8 |   16 | 0.00134526  |
| DEFAULT_f201f_00005 | PENDING  |       |            2 |   16 |   16 | 0.00126487  |
| DEFAULT_f201f_00006 | PENDING  |       |            2 |  256 |   16 | 0.0404286   |
| DEFAULT_f201f_00007 | PENDING  |       |            8 |   64 |  256 | 0.000285913 |
| DEFAULT_f201f_00008 | PENDING  |       |            8 |   32 |   16 | 0.00172471  |
| DEFAULT_f201f_00009 | PENDING  |       |            8 |   64 |   16 | 0.0574224   |
+---------------------+----------+-------+--------------+------+------+-------------+


[2m[36m(pid=5086)[0m [1,  2000] loss: 2.144
[2m[36m(pid=5090)[0m [1,  2000] loss: 2.244
[2m[36m(pid=5088)[0m [1,  2000] loss: 2.299
[2m[36m(pid=5085)[0m [1,  2000] loss: 1.794
[2m[36m(pid=5086)[0m [1,  4000] loss: 1.006
[2m[36m(pid=5088)[0m [1,  4000] loss: 1.054
[2m[36m(pid=5090)[0m [1,  4000] loss: 1.110
Result for DEFAULT_f201f_00000:
  accuracy: 0.4215
  date: 2021-07-29_14-12-50
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 1.6012202549934387
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 46.481757164001465
  time_this_iter_s: 46.481757164001465
  time_total_s: 46.481757164001465
  timestamp: 1627567970
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.8/31.3 GiB
Using AsyncHyperBand: num_stopped=0
Bracket: Iter 8.000: None | Iter 4.000: None | Iter 2.000: None | Iter 1.000: -1.6012202549934387
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (6 PENDING, 4 RUNNING)
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status   | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING  | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.60122 |     0.4215 |                    1 |
| DEFAULT_f201f_00001 | RUNNING  |                 |            2 |  128 |    8 | 0.000447305 |         |            |                      |
| DEFAULT_f201f_00002 | RUNNING  |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00003 | RUNNING  |                 |            2 |   32 |   16 | 0.00404483  |         |            |                      |
| DEFAULT_f201f_00004 | PENDING  |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING  |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING  |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING  |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING  |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING  |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5086)[0m [1,  6000] loss: 0.648
[2m[36m(pid=5088)[0m [1,  6000] loss: 0.641
[2m[36m(pid=5086)[0m [1,  8000] loss: 0.499
[2m[36m(pid=5090)[0m [1,  6000] loss: 0.720
[2m[36m(pid=5088)[0m [1,  8000] loss: 0.448
[2m[36m(pid=5085)[0m [2,  2000] loss: 1.567
[2m[36m(pid=5088)[0m [1, 10000] loss: 0.347
[2m[36m(pid=5086)[0m [1, 10000] loss: 0.386
Result for DEFAULT_f201f_00000:
  accuracy: 0.4254
  date: 2021-07-29_14-13-35
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 2
  loss: 1.6653225509643554
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 91.32741332054138
  time_this_iter_s: 44.84565615653992
  time_total_s: 91.32741332054138
  timestamp: 1627568015
  timesteps_since_restore: 0
  training_iteration: 2
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=0
Bracket: Iter 8.000: None | Iter 4.000: None | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6012202549934387
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (6 PENDING, 4 RUNNING)
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status   | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING  | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.66532 |     0.4254 |                    2 |
| DEFAULT_f201f_00001 | RUNNING  |                 |            2 |  128 |    8 | 0.000447305 |         |            |                      |
| DEFAULT_f201f_00002 | RUNNING  |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00003 | RUNNING  |                 |            2 |   32 |   16 | 0.00404483  |         |            |                      |
| DEFAULT_f201f_00004 | PENDING  |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING  |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING  |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING  |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING  |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING  |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5090)[0m [1,  8000] loss: 0.539
[2m[36m(pid=5086)[0m [1, 12000] loss: 0.321
[2m[36m(pid=5088)[0m [1, 12000] loss: 0.279
[2m[36m(pid=5085)[0m [3,  2000] loss: 1.525
[2m[36m(pid=5086)[0m [1, 14000] loss: 0.276
[2m[36m(pid=5090)[0m [1, 10000] loss: 0.427
[2m[36m(pid=5088)[0m [1, 14000] loss: 0.234
Result for DEFAULT_f201f_00000:
  accuracy: 0.4648
  date: 2021-07-29_14-14-16
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 3
  loss: 1.5220761643409728
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 132.20206356048584
  time_this_iter_s: 40.87465023994446
  time_total_s: 132.20206356048584
  timestamp: 1627568056
  timesteps_since_restore: 0
  training_iteration: 3
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=0
Bracket: Iter 8.000: None | Iter 4.000: None | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6012202549934387
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (6 PENDING, 4 RUNNING)
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status   | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING  | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.52208 |     0.4648 |                    3 |
| DEFAULT_f201f_00001 | RUNNING  |                 |            2 |  128 |    8 | 0.000447305 |         |            |                      |
| DEFAULT_f201f_00002 | RUNNING  |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00003 | RUNNING  |                 |            2 |   32 |   16 | 0.00404483  |         |            |                      |
| DEFAULT_f201f_00004 | PENDING  |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING  |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING  |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING  |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING  |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING  |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5086)[0m [1, 16000] loss: 0.244
[2m[36m(pid=5090)[0m [1, 12000] loss: 0.357
[2m[36m(pid=5088)[0m [1, 16000] loss: 0.202
[2m[36m(pid=5086)[0m [1, 18000] loss: 0.211
[2m[36m(pid=5085)[0m [4,  2000] loss: 1.498
[2m[36m(pid=5088)[0m [1, 18000] loss: 0.176
Result for DEFAULT_f201f_00000:
  accuracy: 0.4372
  date: 2021-07-29_14-14-53
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 4
  loss: 1.6066401032447815
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 168.9676070213318
  time_this_iter_s: 36.76554346084595
  time_total_s: 168.9676070213318
  timestamp: 1627568093
  timesteps_since_restore: 0
  training_iteration: 4
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=0
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6012202549934387
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (6 PENDING, 4 RUNNING)
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status   | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING  | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.60664 |     0.4372 |                    4 |
| DEFAULT_f201f_00001 | RUNNING  |                 |            2 |  128 |    8 | 0.000447305 |         |            |                      |
| DEFAULT_f201f_00002 | RUNNING  |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00003 | RUNNING  |                 |            2 |   32 |   16 | 0.00404483  |         |            |                      |
| DEFAULT_f201f_00004 | PENDING  |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING  |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING  |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING  |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING  |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING  |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5090)[0m [1, 14000] loss: 0.314
[2m[36m(pid=5086)[0m [1, 20000] loss: 0.196
[2m[36m(pid=5088)[0m [1, 20000] loss: 0.155
[2m[36m(pid=5090)[0m [1, 16000] loss: 0.279
[2m[36m(pid=5085)[0m [5,  2000] loss: 1.501
Result for DEFAULT_f201f_00000:
  accuracy: 0.4442
  date: 2021-07-29_14-15-34
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 5
  loss: 1.5668423042297364
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 209.80427646636963
  time_this_iter_s: 40.83666944503784
  time_total_s: 209.80427646636963
  timestamp: 1627568134
  timesteps_since_restore: 0
  training_iteration: 5
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=0
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6012202549934387
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (6 PENDING, 4 RUNNING)
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status   | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING  | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.56684 |     0.4442 |                    5 |
| DEFAULT_f201f_00001 | RUNNING  |                 |            2 |  128 |    8 | 0.000447305 |         |            |                      |
| DEFAULT_f201f_00002 | RUNNING  |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00003 | RUNNING  |                 |            2 |   32 |   16 | 0.00404483  |         |            |                      |
| DEFAULT_f201f_00004 | PENDING  |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING  |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING  |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING  |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING  |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING  |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5090)[0m [1, 18000] loss: 0.237
Result for DEFAULT_f201f_00003:
  accuracy: 0.2865
  date: 2021-07-29_14-15-43
  done: true
  experiment_id: 8ff027eec3f14b2b9dc5ce8d11408ff8
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 1.899737369287014
  node_ip: 172.17.0.4
  pid: 5086
  should_checkpoint: true
  time_since_restore: 218.76438760757446
  time_this_iter_s: 218.76438760757446
  time_total_s: 218.76438760757446
  timestamp: 1627568143
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00003

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=1
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.7504788121402264
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (6 PENDING, 4 RUNNING)
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status   | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING  | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.56684 |     0.4442 |                    5 |
| DEFAULT_f201f_00001 | RUNNING  |                 |            2 |  128 |    8 | 0.000447305 |         |            |                      |
| DEFAULT_f201f_00002 | RUNNING  |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00003 | RUNNING  | 172.17.0.4:5086 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | PENDING  |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING  |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING  |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING  |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING  |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING  |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
+---------------------+----------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5083)[0m Files already downloaded and verified
[2m[36m(pid=5083)[0m Files already downloaded and verified
Result for DEFAULT_f201f_00001:
  accuracy: 0.4357
  date: 2021-07-29_14-15-48
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 1.541055219617486
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 224.23737812042236
  time_this_iter_s: 224.23737812042236
  time_total_s: 224.23737812042236
  timestamp: 1627568148
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=1
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6012202549934387
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (5 PENDING, 4 RUNNING, 1 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.56684 |     0.4442 |                    5 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00002 | RUNNING    |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00004 | RUNNING    |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5090)[0m [1, 20000] loss: 0.212
[2m[36m(pid=5085)[0m [6,  2000] loss: 1.502
[2m[36m(pid=5083)[0m [1,  2000] loss: 2.252
[2m[36m(pid=5088)[0m [2,  2000] loss: 1.505
Result for DEFAULT_f201f_00000:
  accuracy: 0.4722
  date: 2021-07-29_14-16-14
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 6
  loss: 1.5356089867591858
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 250.69555974006653
  time_this_iter_s: 40.8912832736969
  time_total_s: 250.69555974006653
  timestamp: 1627568174
  timesteps_since_restore: 0
  training_iteration: 6
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=1
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6012202549934387
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (5 PENDING, 4 RUNNING, 1 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.53561 |     0.4722 |                    6 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00002 | RUNNING    |                 |            2 |  256 |  256 | 0.0119053   |         |            |                      |
| DEFAULT_f201f_00004 | RUNNING    |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5083)[0m [1,  4000] loss: 0.963
[2m[36m(pid=5088)[0m [2,  4000] loss: 0.749
Result for DEFAULT_f201f_00002:
  accuracy: 0.1789
  date: 2021-07-29_14-16-32
  done: true
  experiment_id: 9a45bdb8086e4379905275a77cbba199
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 2.121293088769913
  node_ip: 172.17.0.4
  pid: 5090
  should_checkpoint: true
  time_since_restore: 268.7365050315857
  time_this_iter_s: 268.7365050315857
  time_total_s: 268.7365050315857
  timestamp: 1627568192
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00002

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=2
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.7504788121402264
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (5 PENDING, 4 RUNNING, 1 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.53561 |     0.4722 |                    6 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00002 | RUNNING    | 172.17.0.4:5090 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00004 | RUNNING    |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | PENDING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5087)[0m Files already downloaded and verified
[2m[36m(pid=5087)[0m Files already downloaded and verified
[2m[36m(pid=5083)[0m [1,  6000] loss: 0.580
[2m[36m(pid=5088)[0m [2,  6000] loss: 0.491
[2m[36m(pid=5085)[0m [7,  2000] loss: 1.507
[2m[36m(pid=5083)[0m [1,  8000] loss: 0.414
[2m[36m(pid=5087)[0m [1,  2000] loss: 2.196
Result for DEFAULT_f201f_00000:
  accuracy: 0.4847
  date: 2021-07-29_14-17-00
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 7
  loss: 1.5526806345939637
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 296.0717935562134
  time_this_iter_s: 45.37623381614685
  time_total_s: 296.0717935562134
  timestamp: 1627568220
  timesteps_since_restore: 0
  training_iteration: 7
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=2
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.7504788121402264
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (4 PENDING, 4 RUNNING, 2 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.55268 |     0.4847 |                    7 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00004 | RUNNING    |                 |            4 |    8 |   16 | 0.00134526  |         |            |                      |
| DEFAULT_f201f_00005 | RUNNING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [2,  8000] loss: 0.365
[2m[36m(pid=5083)[0m [1, 10000] loss: 0.318
[2m[36m(pid=5087)[0m [1,  4000] loss: 0.984
[2m[36m(pid=5088)[0m [2, 10000] loss: 0.292
[2m[36m(pid=5085)[0m [8,  2000] loss: 1.514
Result for DEFAULT_f201f_00004:
  accuracy: 0.3837
  date: 2021-07-29_14-17-31
  done: false
  experiment_id: 29b8f76bd84947dda7dcc9751ba1012d
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 1.6751174277424812
  node_ip: 172.17.0.4
  pid: 5083
  should_checkpoint: true
  time_since_restore: 106.32377362251282
  time_this_iter_s: 106.32377362251282
  time_total_s: 106.32377362251282
  timestamp: 1627568251
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00004

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=2
Bracket: Iter 8.000: None | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (4 PENDING, 4 RUNNING, 2 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.55268 |     0.4847 |                    7 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00004 | RUNNING    | 172.17.0.4:5083 |            4 |    8 |   16 | 0.00134526  | 1.67512 |     0.3837 |                    1 |
| DEFAULT_f201f_00005 | RUNNING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5087)[0m [1,  6000] loss: 0.603
[2m[36m(pid=5088)[0m [2, 12000] loss: 0.236
Result for DEFAULT_f201f_00000:
  accuracy: 0.4122
  date: 2021-07-29_14-17-41
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 8
  loss: 1.6352525778770446
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 336.84724020957947
  time_this_iter_s: 40.77544665336609
  time_total_s: 336.84724020957947
  timestamp: 1627568261
  timesteps_since_restore: 0
  training_iteration: 8
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=2
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (4 PENDING, 4 RUNNING, 2 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.63525 |     0.4122 |                    8 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00004 | RUNNING    | 172.17.0.4:5083 |            4 |    8 |   16 | 0.00134526  | 1.67512 |     0.3837 |                    1 |
| DEFAULT_f201f_00005 | RUNNING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5087)[0m [1,  8000] loss: 0.435
[2m[36m(pid=5083)[0m [2,  2000] loss: 1.556
[2m[36m(pid=5088)[0m [2, 14000] loss: 0.202
[2m[36m(pid=5087)[0m [1, 10000] loss: 0.343
[2m[36m(pid=5085)[0m [9,  2000] loss: 1.508
[2m[36m(pid=5088)[0m [2, 16000] loss: 0.173
[2m[36m(pid=5083)[0m [2,  4000] loss: 0.756
Result for DEFAULT_f201f_00000:
  accuracy: 0.4686
  date: 2021-07-29_14-18-23
  done: false
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 9
  loss: 1.5646762167930603
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 378.9824676513672
  time_this_iter_s: 42.13522744178772
  time_total_s: 378.9824676513672
  timestamp: 1627568303
  timesteps_since_restore: 0
  training_iteration: 9
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 16.0/31.3 GiB
Using AsyncHyperBand: num_stopped=2
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (4 PENDING, 4 RUNNING, 2 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.56468 |     0.4686 |                    9 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00004 | RUNNING    | 172.17.0.4:5083 |            4 |    8 |   16 | 0.00134526  | 1.67512 |     0.3837 |                    1 |
| DEFAULT_f201f_00005 | RUNNING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5087)[0m [1, 12000] loss: 0.276
[2m[36m(pid=5083)[0m [2,  6000] loss: 0.508
[2m[36m(pid=5088)[0m [2, 18000] loss: 0.153
[2m[36m(pid=5087)[0m [1, 14000] loss: 0.231
[2m[36m(pid=5083)[0m [2,  8000] loss: 0.368
[2m[36m(pid=5085)[0m [10,  2000] loss: 1.511
[2m[36m(pid=5088)[0m [2, 20000] loss: 0.138
Result for DEFAULT_f201f_00000:
  accuracy: 0.4533
  date: 2021-07-29_14-19-04
  done: true
  experiment_id: 519e73cc629f4a16ad544ba419c613fd
  hostname: 9ec45cf3ba83
  iterations_since_restore: 10
  loss: 1.6734770732879638
  node_ip: 172.17.0.4
  pid: 5085
  should_checkpoint: true
  time_since_restore: 420.25495433807373
  time_this_iter_s: 41.27248668670654
  time_total_s: 420.25495433807373
  timestamp: 1627568344
  timesteps_since_restore: 0
  training_iteration: 10
  trial_id: f201f_00000

== Status ==
Memory usage on this node: 16.0/31.3 GiB
Using AsyncHyperBand: num_stopped=3
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6653225509643554 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (4 PENDING, 4 RUNNING, 2 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | RUNNING    | 172.17.0.4:5085 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.54106 |     0.4357 |                    1 |
| DEFAULT_f201f_00004 | RUNNING    | 172.17.0.4:5083 |            4 |    8 |   16 | 0.00134526  | 1.67512 |     0.3837 |                    1 |
| DEFAULT_f201f_00005 | RUNNING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | PENDING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5087)[0m [1, 16000] loss: 0.201
[2m[36m(pid=5089)[0m Files already downloaded and verified
[2m[36m(pid=5089)[0m Files already downloaded and verified
[2m[36m(pid=5083)[0m [2, 10000] loss: 0.296
[2m[36m(pid=5087)[0m [1, 18000] loss: 0.177
Result for DEFAULT_f201f_00001:
  accuracy: 0.4878
  date: 2021-07-29_14-19-25
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 2
  loss: 1.4001507631493733
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 441.5508849620819
  time_this_iter_s: 217.31350684165955
  time_total_s: 441.5508849620819
  timestamp: 1627568365
  timesteps_since_restore: 0
  training_iteration: 2
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=3
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.5327366570568643 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (3 PENDING, 4 RUNNING, 3 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.40015 |     0.4878 |                    2 |
| DEFAULT_f201f_00004 | RUNNING    | 172.17.0.4:5083 |            4 |    8 |   16 | 0.00134526  | 1.67512 |     0.3837 |                    1 |
| DEFAULT_f201f_00005 | RUNNING    |                 |            2 |   16 |   16 | 0.00126487  |         |            |                      |
| DEFAULT_f201f_00006 | RUNNING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | PENDING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


Result for DEFAULT_f201f_00004:
  accuracy: 0.4063
  date: 2021-07-29_14-19-28
  done: true
  experiment_id: 29b8f76bd84947dda7dcc9751ba1012d
  hostname: 9ec45cf3ba83
  iterations_since_restore: 2
  loss: 1.6531679716050625
  node_ip: 172.17.0.4
  pid: 5083
  should_checkpoint: true
  time_since_restore: 223.1200053691864
  time_this_iter_s: 116.79623174667358
  time_total_s: 223.1200053691864
  timestamp: 1627568368
  timesteps_since_restore: 0
  training_iteration: 2
  trial_id: f201f_00004

[2m[36m(pid=5084)[0m Files already downloaded and verified
[2m[36m(pid=5089)[0m [1,  2000] loss: 2.342
[2m[36m(pid=5084)[0m Files already downloaded and verified
[2m[36m(pid=5087)[0m [1, 20000] loss: 0.157
[2m[36m(pid=5088)[0m [3,  2000] loss: 1.328
[2m[36m(pid=5089)[0m [1,  4000] loss: 1.175
[2m[36m(pid=5084)[0m [1,  2000] loss: 2.300
[2m[36m(pid=5088)[0m [3,  4000] loss: 0.666
[2m[36m(pid=5089)[0m [1,  6000] loss: 0.782
[2m[36m(pid=5084)[0m [1,  4000] loss: 1.112
Result for DEFAULT_f201f_00005:
  accuracy: 0.4437
  date: 2021-07-29_14-20-13
  done: false
  experiment_id: 0aba26e190ca42cba089c32a9006ff74
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 1.5085618595972656
  node_ip: 172.17.0.4
  pid: 5087
  should_checkpoint: true
  time_since_restore: 218.37232851982117
  time_this_iter_s: 218.37232851982117
  time_total_s: 218.37232851982117
  timestamp: 1627568413
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00005

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=4
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6531679716050625 | Iter 1.000: -1.63816884136796
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 PENDING, 4 RUNNING, 4 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.40015 |     0.4878 |                    2 |
| DEFAULT_f201f_00005 | RUNNING    | 172.17.0.4:5087 |            2 |   16 |   16 | 0.00126487  | 1.50856 |     0.4437 |                    1 |
| DEFAULT_f201f_00006 | RUNNING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | RUNNING    |                 |            8 |   64 |  256 | 0.000285913 |         |            |                      |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [3,  6000] loss: 0.432
[2m[36m(pid=5089)[0m [1,  8000] loss: 0.587
[2m[36m(pid=5087)[0m [2,  2000] loss: 1.513
Result for DEFAULT_f201f_00007:
  accuracy: 0.2903
  date: 2021-07-29_14-20-35
  done: true
  experiment_id: b17549cee3d041f19ecfddc572127732
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 1.9859605166435241
  node_ip: 172.17.0.4
  pid: 5084
  should_checkpoint: true
  time_since_restore: 65.69195485115051
  time_this_iter_s: 65.69195485115051
  time_total_s: 65.69195485115051
  timestamp: 1627568435
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00007

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=5
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6531679716050625 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 PENDING, 4 RUNNING, 4 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.40015 |     0.4878 |                    2 |
| DEFAULT_f201f_00005 | RUNNING    | 172.17.0.4:5087 |            2 |   16 |   16 | 0.00126487  | 1.50856 |     0.4437 |                    1 |
| DEFAULT_f201f_00006 | RUNNING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00007 | RUNNING    | 172.17.0.4:5084 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00008 | PENDING    |                 |            8 |   32 |   16 | 0.00172471  |         |            |                      |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [3,  8000] loss: 0.324
[2m[36m(pid=8726)[0m Files already downloaded and verified
[2m[36m(pid=8726)[0m Files already downloaded and verified
[2m[36m(pid=5089)[0m [1, 10000] loss: 0.469
[2m[36m(pid=5087)[0m [2,  4000] loss: 0.768
[2m[36m(pid=5088)[0m [3, 10000] loss: 0.257
[2m[36m(pid=5089)[0m [1, 12000] loss: 0.392
[2m[36m(pid=8726)[0m [1,  2000] loss: 2.275
[2m[36m(pid=5087)[0m [2,  6000] loss: 0.506
[2m[36m(pid=5088)[0m [3, 12000] loss: 0.218
[2m[36m(pid=5087)[0m [2,  8000] loss: 0.377
[2m[36m(pid=8726)[0m [1,  4000] loss: 0.945
[2m[36m(pid=5089)[0m [1, 14000] loss: 0.336
[2m[36m(pid=5088)[0m [3, 14000] loss: 0.182
[2m[36m(pid=5087)[0m [2, 10000] loss: 0.303
[2m[36m(pid=5088)[0m [3, 16000] loss: 0.158
Result for DEFAULT_f201f_00008:
  accuracy: 0.4047
  date: 2021-07-29_14-21-45
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 1.620893050098419
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 66.44279766082764
  time_this_iter_s: 66.44279766082764
  time_total_s: 66.44279766082764
  timestamp: 1627568505
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=5
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.6531679716050625 | Iter 1.000: -1.64800523892045
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 PENDING, 4 RUNNING, 5 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.40015 |     0.4878 |                    2 |
| DEFAULT_f201f_00005 | RUNNING    | 172.17.0.4:5087 |            2 |   16 |   16 | 0.00126487  | 1.50856 |     0.4437 |                    1 |
| DEFAULT_f201f_00006 | RUNNING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.62089 |     0.4047 |                    1 |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5089)[0m [1, 16000] loss: 0.293
[2m[36m(pid=5087)[0m [2, 12000] loss: 0.250
[2m[36m(pid=5088)[0m [3, 18000] loss: 0.142
[2m[36m(pid=5087)[0m [2, 14000] loss: 0.210
[2m[36m(pid=8726)[0m [2,  2000] loss: 1.533
[2m[36m(pid=5089)[0m [1, 18000] loss: 0.261
[2m[36m(pid=5088)[0m [3, 20000] loss: 0.130
[2m[36m(pid=5087)[0m [2, 16000] loss: 0.189
[2m[36m(pid=5089)[0m [1, 20000] loss: 0.235
[2m[36m(pid=8726)[0m [2,  4000] loss: 0.731
[2m[36m(pid=5087)[0m [2, 18000] loss: 0.167
Result for DEFAULT_f201f_00008:
  accuracy: 0.4996
  date: 2021-07-29_14-22-48
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 2
  loss: 1.3640813674926757
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 129.95181798934937
  time_this_iter_s: 63.50902032852173
  time_total_s: 129.95181798934937
  timestamp: 1627568568
  timesteps_since_restore: 0
  training_iteration: 2
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 16.0/31.3 GiB
Using AsyncHyperBand: num_stopped=5
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.526659367377218 | Iter 1.000: -1.64800523892045
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 PENDING, 4 RUNNING, 5 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.40015 |     0.4878 |                    2 |
| DEFAULT_f201f_00005 | RUNNING    | 172.17.0.4:5087 |            2 |   16 |   16 | 0.00126487  | 1.50856 |     0.4437 |                    1 |
| DEFAULT_f201f_00006 | RUNNING    |                 |            2 |  256 |   16 | 0.0404286   |         |            |                      |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.36408 |     0.4996 |                    2 |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5087)[0m [2, 20000] loss: 0.151
Result for DEFAULT_f201f_00001:
  accuracy: 0.5397
  date: 2021-07-29_14-22-53
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 3
  loss: 1.2915692168400623
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 649.3570704460144
  time_this_iter_s: 207.8061854839325
  time_total_s: 649.3570704460144
  timestamp: 1627568573
  timesteps_since_restore: 0
  training_iteration: 3
  trial_id: f201f_00001

Result for DEFAULT_f201f_00006:
  accuracy: 0.0956
  date: 2021-07-29_14-23-00
  done: true
  experiment_id: 7dc069aa9c2144ee9e8e492ed6b3f78a
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 2.3285563125133515
  node_ip: 172.17.0.4
  pid: 5089
  should_checkpoint: true
  time_since_restore: 233.9469187259674
  time_this_iter_s: 233.9469187259674
  time_total_s: 233.9469187259674
  timestamp: 1627568580
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00006

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=6
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.526659367377218 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 PENDING, 4 RUNNING, 5 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.29157 |     0.5397 |                    3 |
| DEFAULT_f201f_00005 | RUNNING    | 172.17.0.4:5087 |            2 |   16 |   16 | 0.00126487  | 1.50856 |     0.4437 |                    1 |
| DEFAULT_f201f_00006 | RUNNING    | 172.17.0.4:5089 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.36408 |     0.4996 |                    2 |
| DEFAULT_f201f_00009 | PENDING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=9546)[0m Files already downloaded and verified
[2m[36m(pid=9546)[0m Files already downloaded and verified
[2m[36m(pid=8726)[0m [3,  2000] loss: 1.361
[2m[36m(pid=5088)[0m [4,  2000] loss: 1.217
Result for DEFAULT_f201f_00005:
  accuracy: 0.4488
  date: 2021-07-29_14-23-27
  done: true
  experiment_id: 0aba26e190ca42cba089c32a9006ff74
  hostname: 9ec45cf3ba83
  iterations_since_restore: 2
  loss: 1.5376347200442106
  node_ip: 172.17.0.4
  pid: 5087
  should_checkpoint: true
  time_since_restore: 411.8736696243286
  time_this_iter_s: 193.50134110450745
  time_total_s: 411.8736696243286
  timestamp: 1627568607
  timesteps_since_restore: 0
  training_iteration: 2
  trial_id: f201f_00005

== Status ==
Memory usage on this node: 15.9/31.3 GiB
Using AsyncHyperBand: num_stopped=7
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.6751174277424812
Resources requested: 8.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (4 RUNNING, 6 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.29157 |     0.5397 |                    3 |
| DEFAULT_f201f_00005 | RUNNING    | 172.17.0.4:5087 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.36408 |     0.4996 |                    2 |
| DEFAULT_f201f_00009 | RUNNING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [4,  4000] loss: 0.596
[2m[36m(pid=9546)[0m [1,  2000] loss: 2.319
[2m[36m(pid=8726)[0m [3,  4000] loss: 0.668
[2m[36m(pid=5088)[0m [4,  6000] loss: 0.405
[2m[36m(pid=9546)[0m [1,  4000] loss: 1.160
Result for DEFAULT_f201f_00008:
  accuracy: 0.529
  date: 2021-07-29_14-23-50
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 3
  loss: 1.320628886294365
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 192.0822982788086
  time_this_iter_s: 62.13048028945923
  time_total_s: 192.0822982788086
  timestamp: 1627568630
  timesteps_since_restore: 0
  training_iteration: 3
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 15.4/31.3 GiB
Using AsyncHyperBand: num_stopped=7
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.6751174277424812
Resources requested: 6.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (3 RUNNING, 7 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.29157 |     0.5397 |                    3 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.32063 |     0.529  |                    3 |
| DEFAULT_f201f_00009 | RUNNING    |                 |            8 |   64 |   16 | 0.0574224   |         |            |                      |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [4,  8000] loss: 0.297
Result for DEFAULT_f201f_00009:
  accuracy: 0.0983
  date: 2021-07-29_14-24-06
  done: true
  experiment_id: e3e77385e2d9496db44502e89f7563e3
  hostname: 9ec45cf3ba83
  iterations_since_restore: 1
  loss: 2.319805237197876
  node_ip: 172.17.0.4
  pid: 9546
  should_checkpoint: true
  time_since_restore: 64.11058259010315
  time_this_iter_s: 64.11058259010315
  time_total_s: 64.11058259010315
  timestamp: 1627568646
  timesteps_since_restore: 0
  training_iteration: 1
  trial_id: f201f_00009

== Status ==
Memory usage on this node: 15.4/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.6066401032447815 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 6.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (3 RUNNING, 7 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.29157 |     0.5397 |                    3 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.32063 |     0.529  |                    3 |
| DEFAULT_f201f_00009 | RUNNING    | 172.17.0.4:9546 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=8726)[0m [4,  2000] loss: 1.258
[2m[36m(pid=5088)[0m [4, 10000] loss: 0.241
[2m[36m(pid=8726)[0m [4,  4000] loss: 0.631
[2m[36m(pid=5088)[0m [4, 12000] loss: 0.197
Result for DEFAULT_f201f_00008:
  accuracy: 0.5519
  date: 2021-07-29_14-24-40
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 4
  loss: 1.273954435968399
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 242.25610089302063
  time_this_iter_s: 50.173802614212036
  time_total_s: 242.25610089302063
  timestamp: 1627568680
  timesteps_since_restore: 0
  training_iteration: 4
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.44029726960659 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.29157 |     0.5397 |                    3 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.27395 |     0.5519 |                    4 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [4, 14000] loss: 0.170
[2m[36m(pid=8726)[0m [5,  2000] loss: 1.193
[2m[36m(pid=5088)[0m [4, 16000] loss: 0.152
[2m[36m(pid=8726)[0m [5,  4000] loss: 0.607
[2m[36m(pid=5088)[0m [4, 18000] loss: 0.134
Result for DEFAULT_f201f_00008:
  accuracy: 0.5585
  date: 2021-07-29_14-25-34
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 5
  loss: 1.2658138296961785
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 296.21075320243835
  time_this_iter_s: 53.954652309417725
  time_total_s: 296.21075320243835
  timestamp: 1627568734
  timesteps_since_restore: 0
  training_iteration: 5
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 14.9/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.44029726960659 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.29157 |     0.5397 |                    3 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.26581 |     0.5585 |                    5 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [4, 20000] loss: 0.117
[2m[36m(pid=8726)[0m [6,  2000] loss: 1.159
[2m[36m(pid=8726)[0m [6,  4000] loss: 0.583
Result for DEFAULT_f201f_00001:
  accuracy: 0.5831
  date: 2021-07-29_14-26-17
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 4
  loss: 1.1855550648670643
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 853.2846131324768
  time_this_iter_s: 203.9275426864624
  time_total_s: 853.2846131324768
  timestamp: 1627568777
  timesteps_since_restore: 0
  training_iteration: 4
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 14.9/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.18556 |     0.5831 |                    4 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.26581 |     0.5585 |                    5 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


Result for DEFAULT_f201f_00008:
  accuracy: 0.5757
  date: 2021-07-29_14-26-31
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 6
  loss: 1.213883140027523
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 352.9563558101654
  time_this_iter_s: 56.74560260772705
  time_total_s: 352.9563558101654
  timestamp: 1627568791
  timesteps_since_restore: 0
  training_iteration: 6
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 14.9/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.18556 |     0.5831 |                    4 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.21388 |     0.5757 |                    6 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [5,  2000] loss: 1.107
[2m[36m(pid=5088)[0m [5,  4000] loss: 0.575
[2m[36m(pid=8726)[0m [7,  2000] loss: 1.115
[2m[36m(pid=5088)[0m [5,  6000] loss: 0.372
[2m[36m(pid=8726)[0m [7,  4000] loss: 0.568
[2m[36m(pid=5088)[0m [5,  8000] loss: 0.280
Result for DEFAULT_f201f_00008:
  accuracy: 0.5837
  date: 2021-07-29_14-27-20
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 7
  loss: 1.1828787779331207
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 401.7406098842621
  time_this_iter_s: 48.78425407409668
  time_total_s: 401.7406098842621
  timestamp: 1627568840
  timesteps_since_restore: 0
  training_iteration: 7
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 14.9/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.6352525778770446 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.18556 |     0.5831 |                    4 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.18288 |     0.5837 |                    7 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [5, 10000] loss: 0.230
[2m[36m(pid=8726)[0m [8,  2000] loss: 1.091
[2m[36m(pid=5088)[0m [5, 12000] loss: 0.187
[2m[36m(pid=5088)[0m [5, 14000] loss: 0.159
[2m[36m(pid=8726)[0m [8,  4000] loss: 0.553
[2m[36m(pid=5088)[0m [5, 16000] loss: 0.141
Result for DEFAULT_f201f_00008:
  accuracy: 0.5696
  date: 2021-07-29_14-28-16
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 8
  loss: 1.2416244268774985
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 458.10355401039124
  time_this_iter_s: 56.36294412612915
  time_total_s: 458.10355401039124
  timestamp: 1627568896
  timesteps_since_restore: 0
  training_iteration: 8
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.4384385023772714 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.18556 |     0.5831 |                    4 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.24162 |     0.5696 |                    8 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [5, 18000] loss: 0.127
[2m[36m(pid=8726)[0m [9,  2000] loss: 1.074
[2m[36m(pid=5088)[0m [5, 20000] loss: 0.113
[2m[36m(pid=8726)[0m [9,  4000] loss: 0.545
Result for DEFAULT_f201f_00008:
  accuracy: 0.5852
  date: 2021-07-29_14-29-06
  done: false
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 9
  loss: 1.1980089530587197
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 508.1109640598297
  time_this_iter_s: 50.00741004943848
  time_total_s: 508.1109640598297
  timestamp: 1627568946
  timesteps_since_restore: 0
  training_iteration: 9
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.4384385023772714 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.18556 |     0.5831 |                    4 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.19801 |     0.5852 |                    9 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


Result for DEFAULT_f201f_00001:
  accuracy: 0.5986
  date: 2021-07-29_14-29-13
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 5
  loss: 1.15476273704404
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 1029.5660145282745
  time_this_iter_s: 176.28140139579773
  time_total_s: 1029.5660145282745
  timestamp: 1627568953
  timesteps_since_restore: 0
  training_iteration: 5
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=8
Bracket: Iter 8.000: -1.4384385023772714 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.15476 |     0.5986 |                    5 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.19801 |     0.5852 |                    9 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=8726)[0m [10,  2000] loss: 1.034
[2m[36m(pid=5088)[0m [6,  2000] loss: 1.071
[2m[36m(pid=8726)[0m [10,  4000] loss: 0.538
[2m[36m(pid=5088)[0m [6,  4000] loss: 0.515
Result for DEFAULT_f201f_00008:
  accuracy: 0.5787
  date: 2021-07-29_14-29-53
  done: true
  experiment_id: 828c343465b746a6833f7a13f52e8c21
  hostname: 9ec45cf3ba83
  iterations_since_restore: 10
  loss: 1.2242828347325325
  node_ip: 172.17.0.4
  pid: 8726
  should_checkpoint: true
  time_since_restore: 554.6373965740204
  time_this_iter_s: 46.526432514190674
  time_total_s: 554.6373965740204
  timestamp: 1627568993
  timesteps_since_restore: 0
  training_iteration: 10
  trial_id: f201f_00008

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=9
Bracket: Iter 8.000: -1.4384385023772714 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 4.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (2 RUNNING, 8 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.15476 |     0.5986 |                    5 |
| DEFAULT_f201f_00008 | RUNNING    | 172.17.0.4:8726 |            8 |   32 |   16 | 0.00172471  | 1.22428 |     0.5787 |                   10 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [6,  6000] loss: 0.354
[2m[36m(pid=5088)[0m [6,  8000] loss: 0.259
[2m[36m(pid=5088)[0m [6, 10000] loss: 0.218
[2m[36m(pid=5088)[0m [6, 12000] loss: 0.179
[2m[36m(pid=5088)[0m [6, 14000] loss: 0.156
[2m[36m(pid=5088)[0m [6, 16000] loss: 0.136
[2m[36m(pid=5088)[0m [6, 18000] loss: 0.116
[2m[36m(pid=5088)[0m [6, 20000] loss: 0.108
Result for DEFAULT_f201f_00001:
  accuracy: 0.5862
  date: 2021-07-29_14-31-07
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 6
  loss: 1.1807675102452049
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 1143.6345767974854
  time_this_iter_s: 114.06856226921082
  time_total_s: 1143.6345767974854
  timestamp: 1627569067
  timesteps_since_restore: 0
  training_iteration: 6
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 14.9/31.3 GiB
Using AsyncHyperBand: num_stopped=9
Bracket: Iter 8.000: -1.4384385023772714 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 2.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 RUNNING, 9 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.18077 |     0.5862 |                    6 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00008 | TERMINATED |                 |            8 |   32 |   16 | 0.00172471  | 1.22428 |     0.5787 |                   10 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [7,  2000] loss: 0.969
[2m[36m(pid=5088)[0m [7,  4000] loss: 0.489
[2m[36m(pid=5088)[0m [7,  6000] loss: 0.332
[2m[36m(pid=5088)[0m [7,  8000] loss: 0.258
[2m[36m(pid=5088)[0m [7, 10000] loss: 0.207
[2m[36m(pid=5088)[0m [7, 12000] loss: 0.170
[2m[36m(pid=5088)[0m [7, 14000] loss: 0.144
[2m[36m(pid=5088)[0m [7, 16000] loss: 0.130
[2m[36m(pid=5088)[0m [7, 18000] loss: 0.114
[2m[36m(pid=5088)[0m [7, 20000] loss: 0.104
Result for DEFAULT_f201f_00001:
  accuracy: 0.5818
  date: 2021-07-29_14-32-42
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 7
  loss: 1.220972312376625
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 1238.2733232975006
  time_this_iter_s: 94.63874650001526
  time_total_s: 1238.2733232975006
  timestamp: 1627569162
  timesteps_since_restore: 0
  training_iteration: 7
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=9
Bracket: Iter 8.000: -1.4384385023772714 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 2.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 RUNNING, 9 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.22097 |     0.5818 |                    7 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00008 | TERMINATED |                 |            8 |   32 |   16 | 0.00172471  | 1.22428 |     0.5787 |                   10 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [8,  2000] loss: 0.954
[2m[36m(pid=5088)[0m [8,  4000] loss: 0.465
[2m[36m(pid=5088)[0m [8,  6000] loss: 0.314
[2m[36m(pid=5088)[0m [8,  8000] loss: 0.246
[2m[36m(pid=5088)[0m [8, 10000] loss: 0.196
[2m[36m(pid=5088)[0m [8, 12000] loss: 0.161
[2m[36m(pid=5088)[0m [8, 14000] loss: 0.142
[2m[36m(pid=5088)[0m [8, 16000] loss: 0.124
[2m[36m(pid=5088)[0m [8, 18000] loss: 0.108
[2m[36m(pid=5088)[0m [8, 20000] loss: 0.098
Result for DEFAULT_f201f_00001:
  accuracy: 0.5836
  date: 2021-07-29_14-34-13
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 8
  loss: 1.246836426538159
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 1329.7241804599762
  time_this_iter_s: 91.45085716247559
  time_total_s: 1329.7241804599762
  timestamp: 1627569253
  timesteps_since_restore: 0
  training_iteration: 8
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=9
Bracket: Iter 8.000: -1.246836426538159 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 2.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 RUNNING, 9 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.24684 |     0.5836 |                    8 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00008 | TERMINATED |                 |            8 |   32 |   16 | 0.00172471  | 1.22428 |     0.5787 |                   10 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [9,  2000] loss: 0.924
[2m[36m(pid=5088)[0m [9,  4000] loss: 0.440
[2m[36m(pid=5088)[0m [9,  6000] loss: 0.298
[2m[36m(pid=5088)[0m [9,  8000] loss: 0.228
[2m[36m(pid=5088)[0m [9, 10000] loss: 0.183
[2m[36m(pid=5088)[0m [9, 12000] loss: 0.158
[2m[36m(pid=5088)[0m [9, 14000] loss: 0.138
[2m[36m(pid=5088)[0m [9, 16000] loss: 0.117
[2m[36m(pid=5088)[0m [9, 18000] loss: 0.110
[2m[36m(pid=5088)[0m [9, 20000] loss: 0.096
Result for DEFAULT_f201f_00001:
  accuracy: 0.5968
  date: 2021-07-29_14-35-49
  done: false
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 9
  loss: 1.2085561733988057
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 1425.0174763202667
  time_this_iter_s: 95.29329586029053
  time_total_s: 1425.0174763202667
  timestamp: 1627569349
  timesteps_since_restore: 0
  training_iteration: 9
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 15.0/31.3 GiB
Using AsyncHyperBand: num_stopped=9
Bracket: Iter 8.000: -1.246836426538159 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 2.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 RUNNING, 9 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.20856 |     0.5968 |                    9 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00008 | TERMINATED |                 |            8 |   32 |   16 | 0.00172471  | 1.22428 |     0.5787 |                   10 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


[2m[36m(pid=5088)[0m [10,  2000] loss: 0.828
[2m[36m(pid=5088)[0m [10,  4000] loss: 0.427
[2m[36m(pid=5088)[0m [10,  6000] loss: 0.290
[2m[36m(pid=5088)[0m [10,  8000] loss: 0.225
[2m[36m(pid=5088)[0m [10, 10000] loss: 0.176
[2m[36m(pid=5088)[0m [10, 12000] loss: 0.152
[2m[36m(pid=5088)[0m [10, 14000] loss: 0.131
[2m[36m(pid=5088)[0m [10, 16000] loss: 0.115
[2m[36m(pid=5088)[0m [10, 18000] loss: 0.102
[2m[36m(pid=5088)[0m [10, 20000] loss: 0.091
Result for DEFAULT_f201f_00001:
  accuracy: 0.5926
  date: 2021-07-29_14-37-26
  done: true
  experiment_id: 3f191cc7a1ef496990f3537841cc639f
  hostname: 9ec45cf3ba83
  iterations_since_restore: 10
  loss: 1.23822381478647
  node_ip: 172.17.0.4
  pid: 5088
  should_checkpoint: true
  time_since_restore: 1522.388427734375
  time_this_iter_s: 97.37095141410828
  time_total_s: 1522.388427734375
  timestamp: 1627569446
  timesteps_since_restore: 0
  training_iteration: 10
  trial_id: f201f_00001

== Status ==
Memory usage on this node: 15.1/31.3 GiB
Using AsyncHyperBand: num_stopped=10
Bracket: Iter 8.000: -1.246836426538159 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 2.0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (1 RUNNING, 9 TERMINATED)
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc             |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00001 | RUNNING    | 172.17.0.4:5088 |            2 |  128 |    8 | 0.000447305 | 1.23822 |     0.5926 |                   10 |
| DEFAULT_f201f_00000 | TERMINATED |                 |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |                 |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |                 |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |                 |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |                 |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |                 |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |                 |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00008 | TERMINATED |                 |            8 |   32 |   16 | 0.00172471  | 1.22428 |     0.5787 |                   10 |
| DEFAULT_f201f_00009 | TERMINATED |                 |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-----------------+--------------+------+------+-------------+---------+------------+----------------------+


== Status ==
Memory usage on this node: 14.9/31.3 GiB
Using AsyncHyperBand: num_stopped=10
Bracket: Iter 8.000: -1.246836426538159 | Iter 4.000: -1.273954435968399 | Iter 2.000: -1.5376347200442106 | Iter 1.000: -1.7874273985147475
Resources requested: 0/8 CPUs, 0/2 GPUs, 0.0/11.74 GiB heap, 0.0/5.87 GiB objects (0.0/1.0 accelerator_type:GTX)
Result logdir: /root/ray_results/DEFAULT_2021-07-29_14-12-02
Number of trials: 10/10 (10 TERMINATED)
+---------------------+------------+-------+--------------+------+------+-------------+---------+------------+----------------------+
| Trial name          | status     | loc   |   batch_size |   l1 |   l2 |          lr |    loss |   accuracy |   training_iteration |
|---------------------+------------+-------+--------------+------+------+-------------+---------+------------+----------------------|
| DEFAULT_f201f_00000 | TERMINATED |       |           16 |  128 |   16 | 0.0173062   | 1.67348 |     0.4533 |                   10 |
| DEFAULT_f201f_00001 | TERMINATED |       |            2 |  128 |    8 | 0.000447305 | 1.23822 |     0.5926 |                   10 |
| DEFAULT_f201f_00002 | TERMINATED |       |            2 |  256 |  256 | 0.0119053   | 2.12129 |     0.1789 |                    1 |
| DEFAULT_f201f_00003 | TERMINATED |       |            2 |   32 |   16 | 0.00404483  | 1.89974 |     0.2865 |                    1 |
| DEFAULT_f201f_00004 | TERMINATED |       |            4 |    8 |   16 | 0.00134526  | 1.65317 |     0.4063 |                    2 |
| DEFAULT_f201f_00005 | TERMINATED |       |            2 |   16 |   16 | 0.00126487  | 1.53763 |     0.4488 |                    2 |
| DEFAULT_f201f_00006 | TERMINATED |       |            2 |  256 |   16 | 0.0404286   | 2.32856 |     0.0956 |                    1 |
| DEFAULT_f201f_00007 | TERMINATED |       |            8 |   64 |  256 | 0.000285913 | 1.98596 |     0.2903 |                    1 |
| DEFAULT_f201f_00008 | TERMINATED |       |            8 |   32 |   16 | 0.00172471  | 1.22428 |     0.5787 |                   10 |
| DEFAULT_f201f_00009 | TERMINATED |       |            8 |   64 |   16 | 0.0574224   | 2.31981 |     0.0983 |                    1 |
+---------------------+------------+-------+--------------+------+------+-------------+---------+------------+----------------------+


Best trial config: {&#39;l1&#39;: 32, &#39;l2&#39;: 16, &#39;lr&#39;: 0.001724709016268422, &#39;batch_size&#39;: 8}
Best trial final validation loss: 1.2242828347325325
Best trial final validation accuracy: 0.5787
Files already downloaded and verified
Files already downloaded and verified
Best trial test set accuracy: 0.5797
</pre></div>
</div>
<p>If you run the code, an example output could look like this:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Number</span> <span class="n">of</span> <span class="n">trials</span><span class="p">:</span> <span class="mi">10</span> <span class="p">(</span><span class="mi">10</span> <span class="n">TERMINATED</span><span class="p">)</span>
<span class="o">+-----+------+------+-------------+--------------+---------+------------+--------------------+</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>   <span class="n">l1</span> <span class="o">|</span>   <span class="n">l2</span> <span class="o">|</span>          <span class="n">lr</span> <span class="o">|</span>   <span class="n">batch_size</span> <span class="o">|</span>    <span class="n">loss</span> <span class="o">|</span>   <span class="n">accuracy</span> <span class="o">|</span> <span class="n">training_iteration</span> <span class="o">|</span>
<span class="o">|-----+------+------+-------------+--------------+---------+------------+--------------------|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>   <span class="mi">64</span> <span class="o">|</span>    <span class="mi">4</span> <span class="o">|</span> <span class="mf">0.00011629</span>  <span class="o">|</span>            <span class="mi">2</span> <span class="o">|</span> <span class="mf">1.87273</span> <span class="o">|</span>     <span class="mf">0.244</span>  <span class="o">|</span>                  <span class="mi">2</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>   <span class="mi">32</span> <span class="o">|</span>   <span class="mi">64</span> <span class="o">|</span> <span class="mf">0.000339763</span> <span class="o">|</span>            <span class="mi">8</span> <span class="o">|</span> <span class="mf">1.23603</span> <span class="o">|</span>     <span class="mf">0.567</span>  <span class="o">|</span>                  <span class="mi">8</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>    <span class="mi">8</span> <span class="o">|</span>   <span class="mi">16</span> <span class="o">|</span> <span class="mf">0.00276249</span>  <span class="o">|</span>           <span class="mi">16</span> <span class="o">|</span> <span class="mf">1.1815</span>  <span class="o">|</span>     <span class="mf">0.5836</span> <span class="o">|</span>                 <span class="mi">10</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>    <span class="mi">4</span> <span class="o">|</span>   <span class="mi">64</span> <span class="o">|</span> <span class="mf">0.000648721</span> <span class="o">|</span>            <span class="mi">4</span> <span class="o">|</span> <span class="mf">1.31131</span> <span class="o">|</span>     <span class="mf">0.5224</span> <span class="o">|</span>                  <span class="mi">8</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>   <span class="mi">32</span> <span class="o">|</span>   <span class="mi">16</span> <span class="o">|</span> <span class="mf">0.000340753</span> <span class="o">|</span>            <span class="mi">8</span> <span class="o">|</span> <span class="mf">1.26454</span> <span class="o">|</span>     <span class="mf">0.5444</span> <span class="o">|</span>                  <span class="mi">8</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>    <span class="mi">8</span> <span class="o">|</span>    <span class="mi">4</span> <span class="o">|</span> <span class="mf">0.000699775</span> <span class="o">|</span>            <span class="mi">8</span> <span class="o">|</span> <span class="mf">1.99594</span> <span class="o">|</span>     <span class="mf">0.1983</span> <span class="o">|</span>                  <span class="mi">2</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>  <span class="mi">256</span> <span class="o">|</span>    <span class="mi">8</span> <span class="o">|</span> <span class="mf">0.0839654</span>   <span class="o">|</span>           <span class="mi">16</span> <span class="o">|</span> <span class="mf">2.3119</span>  <span class="o">|</span>     <span class="mf">0.0993</span> <span class="o">|</span>                  <span class="mi">1</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>   <span class="mi">16</span> <span class="o">|</span>  <span class="mi">128</span> <span class="o">|</span> <span class="mf">0.0758154</span>   <span class="o">|</span>           <span class="mi">16</span> <span class="o">|</span> <span class="mf">2.33575</span> <span class="o">|</span>     <span class="mf">0.1327</span> <span class="o">|</span>                  <span class="mi">1</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>   <span class="mi">16</span> <span class="o">|</span>    <span class="mi">8</span> <span class="o">|</span> <span class="mf">0.0763312</span>   <span class="o">|</span>           <span class="mi">16</span> <span class="o">|</span> <span class="mf">2.31129</span> <span class="o">|</span>     <span class="mf">0.1042</span> <span class="o">|</span>                  <span class="mi">4</span> <span class="o">|</span>
<span class="o">|</span> <span class="o">...</span> <span class="o">|</span>  <span class="mi">128</span> <span class="o">|</span>   <span class="mi">16</span> <span class="o">|</span> <span class="mf">0.000124903</span> <span class="o">|</span>            <span class="mi">4</span> <span class="o">|</span> <span class="mf">2.26917</span> <span class="o">|</span>     <span class="mf">0.1945</span> <span class="o">|</span>                  <span class="mi">1</span> <span class="o">|</span>
<span class="o">+-----+------+------+-------------+--------------+---------+------------+--------------------+</span>


<span class="n">Best</span> <span class="n">trial</span> <span class="n">config</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;l1&#39;</span><span class="p">:</span> <span class="mi">8</span><span class="p">,</span> <span class="s1">&#39;l2&#39;</span><span class="p">:</span> <span class="mi">16</span><span class="p">,</span> <span class="s1">&#39;lr&#39;</span><span class="p">:</span> <span class="mf">0.00276249</span><span class="p">,</span> <span class="s1">&#39;batch_size&#39;</span><span class="p">:</span> <span class="mi">16</span><span class="p">,</span> <span class="s1">&#39;data_dir&#39;</span><span class="p">:</span> <span class="s1">&#39;...&#39;</span><span class="p">}</span>
<span class="n">Best</span> <span class="n">trial</span> <span class="n">final</span> <span class="n">validation</span> <span class="n">loss</span><span class="p">:</span> <span class="mf">1.181501</span>
<span class="n">Best</span> <span class="n">trial</span> <span class="n">final</span> <span class="n">validation</span> <span class="n">accuracy</span><span class="p">:</span> <span class="mf">0.5836</span>
<span class="n">Best</span> <span class="n">trial</span> <span class="n">test</span> <span class="nb">set</span> <span class="n">accuracy</span><span class="p">:</span> <span class="mf">0.5806</span>
</pre></div>
</div>
<p>Most trials have been stopped early in order to avoid wasting resources.
The best performing trial achieved a validation accuracy of about 58%, which could
be confirmed on the test set.</p>
<p>So that’s it! You can now tune the parameters of your PyTorch models.</p>
<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 26 minutes  23.448 seconds)</p>
<div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-beginner-hyperparameter-tuning-tutorial-py">
<div class="sphx-glr-download docutils container">
<a class="reference download internal" download="" href="../_downloads/681767ad1f2e0356d3ce30263c8e07ff/hyperparameter_tuning_tutorial.py"><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Python</span> <span class="pre">source</span> <span class="pre">code:</span> <span class="pre">hyperparameter_tuning_tutorial.py</span></code></a></div>
<div class="sphx-glr-download docutils container">
<a class="reference download internal" download="" href="../_downloads/5beb23d962f74f0b24a548f2e82e06ce/hyperparameter_tuning_tutorial.ipynb"><code class="xref download docutils literal notranslate"><span class="pre">Download</span> <span class="pre">Jupyter</span> <span class="pre">notebook:</span> <span class="pre">hyperparameter_tuning_tutorial.ipynb</span></code></a></div>
</div>
<p class="sphx-glr-signature"><a class="reference external" href="https://sphinx-gallery.readthedocs.io">Gallery generated by Sphinx-Gallery</a></p>
</div>
</div>


             </article>
             
            </div>
            <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="../intermediate/parametrizations.html" class="btn btn-neutral float-right" title="Parametrizations Tutorial" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
      
      
        <a href="../intermediate/tensorboard_profiler_tutorial.html" class="btn btn-neutral" title="PyTorch Profiler With TensorBoard" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
      
    </div>
  

    <hr class="rating-hr hr-top">
      <div class="rating-container">
        <div class="rating-prompt">Rate this Tutorial</div>
        <div class="stars-outer">
          <i class="far fa-star" title="1 Star" data-behavior="tutorial-rating" data-count="1"></i>
          <i class="far fa-star" title="2 Stars" data-behavior="tutorial-rating" data-count="2"></i>
          <i class="far fa-star" title="3 Stars" data-behavior="tutorial-rating" data-count="3"></i>
          <i class="far fa-star" title="4 Stars" data-behavior="tutorial-rating" data-count="4"></i>
          <i class="far fa-star" title="5 Stars" data-behavior="tutorial-rating" data-count="5"></i>
        </div>
      </div>
    <hr class="rating-hr hr-bottom"/>

  
  <div role="contentinfo">
    <p>
        &copy; Copyright 2021, PyTorch &amp; PyTorch Korea Community.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Hyperparameter tuning with Ray Tune</a><ul>
<li><a class="reference internal" href="#setup-imports">Setup / Imports</a></li>
<li><a class="reference internal" href="#data-loaders">Data loaders</a></li>
<li><a class="reference internal" href="#configurable-neural-network">Configurable neural network</a></li>
<li><a class="reference internal" href="#the-train-function">The train function</a><ul>
<li><a class="reference internal" href="#adding-multi-gpu-support-with-dataparallel">Adding (multi) GPU support with DataParallel</a></li>
<li><a class="reference internal" href="#communicating-with-ray-tune">Communicating with Ray Tune</a></li>
<li><a class="reference internal" href="#full-training-function">Full training function</a></li>
</ul>
</li>
<li><a class="reference internal" href="#test-set-accuracy">Test set accuracy</a></li>
<li><a class="reference internal" href="#configuring-the-search-space">Configuring the search space</a></li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  
       <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
         <script type="text/javascript" src="../_static/jquery.js"></script>
         <script type="text/javascript" src="../_static/underscore.js"></script>
         <script type="text/javascript" src="../_static/doctools.js"></script>
         <script type="text/javascript" src="../_static/clipboard.min.js"></script>
         <script type="text/javascript" src="../_static/copybutton.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/contrib/auto-render.min.js"></script>
         <script type="text/javascript" src="../_static/katex_autorenderer.js"></script>
     

  <script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 

<script>
//add microsoft link

if(window.location.href.indexOf("/beginner/basics/")!= -1)
{
  var url="https://docs.microsoft.com/learn/paths/pytorch-fundamentals/?wt.mc_id=aiml-7486-cxa";
  switch(window.location.pathname.split("/").pop().replace('.html',''))
  {
    case"quickstart_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/9-quickstart?WT.mc_id=aiml-7486-cxa";
      break;
    case"tensorqs_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/2-tensors?WT.mc_id=aiml-7486-cxa";
      break;
    case"data_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/3-data?WT.mc_id=aiml-7486-cxa";
      break;
    case"transforms_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/4-transforms?WT.mc_id=aiml-7486-cxa";
      break;
    case"buildmodel_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/5-model?WT.mc_id=aiml-7486-cxa";
      break;
    case"autogradqs_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/6-autograd?WT.mc_id=aiml-7486-cxa";
      break;
    case"optimization_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/7-optimization?WT.mc_id=aiml-7486-cxa";
      break;
    case"saveloadrun_tutorial":
      url="https://docs.microsoft.com/learn/modules/intro-machine-learning-pytorch/8-inference?WT.mc_id=aiml-7486-cxa";
    }

    $(".pytorch-call-to-action-links").children().first().before("<a href="+url+' data-behavior="call-to-action-event" data-response="Run in Microsoft Learn" target="_blank"><div id="microsoft-learn-link" style="padding-bottom: 0.625rem;border-bottom: 1px solid #f3f4f7;padding-right: 2.5rem;display: -webkit-box;  display: -ms-flexbox; isplay: flex; -webkit-box-align: center;-ms-flex-align: center;align-items: center;"><img class="call-to-action-img" src="../../_static/images/microsoft-logo.svg"/><div class="call-to-action-desktop-view">Run in Microsoft Learn</div><div class="call-to-action-mobile-view">Learn</div></div></a>')
  }
</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-71919972-3"></script>
<script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());
  gtag('config', 'UA-71919972-3');
</script>


<script>
  $("[data-behavior='call-to-action-event']").on('click', function(){
    ga('send', {
      hitType: 'event',
      eventCategory: $(this).attr("data-response"),
      eventAction: 'click',
      eventLabel: window.location.href
    });

    gtag('event', 'click', {
      'event_category': $(this).attr("data-response"),
      'event_label': $("h1").first().text(),
      'tutorial_link': window.location.href
    });
   });

   $("[data-behavior='tutorial-rating']").on('click', function(){
    gtag('event', 'click', {
      'event_category': 'Tutorial Rating',
      'event_label': $("h1").first().text(),
      'value': $(this).attr("data-count")
    });
   });

   if (location.pathname == "/") {
     $(".rating-container").hide();
     $(".hr-bottom").hide();
   }
</script>

<script type="text/javascript">
  var collapsedSections = ['파이토치(PyTorch) 레시피', '파이토치(PyTorch) 배우기', '이미지/비디오', '오디오', '텍스트', '강화학습', 'PyTorch 모델을 프로덕션 환경에 배포하기', 'Code Transforms with FX', '프론트엔드 API', 'PyTorch 확장하기', '모델 최적화', '병렬 및 분산 학습', 'Mobile'];
</script>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>공식 문서 (영어)</h2>
          <p>PyTorch 공식 문서입니다.</p>
          <a id="orgTutorialLink" class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html" target="_blank">공식 문서로 이동</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>한국어 튜토리얼</h2>
          <p>한국어로 번역 중인 PyTorch 튜토리얼입니다.</p>
          <a class="with-right-arrow" href="https://tutorials.pytorch.kr">튜토리얼로 이동</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>커뮤니티</h2>
          <p>다른 사용자들과 의견을 나눠보세요!</p>
          <a class="with-right-arrow" href="https://discuss.pytorch.kr">커뮤니티로 이동</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.kr/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org">PyTorch 홈페이지 (공식)</a></li>
            <li><a href="https://pytorch.org" target="_blank">공식 홈페이지</a></li>
            <li><a href="https://pytorch.org/tutorials" target="_blank">공식 튜토리얼</a></li>
            <li><a href="https://pytorch.org/docs" target="_blank">공식 문서</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="">한국 사용자 모임</a></li>
            <li><a href="https://pytorch.kr/about">사이트 소개</a></li>
            <li><a href="https://tutorials.pytorch.kr/">한국어 튜토리얼</a></li>
            <li><a href="https://github.com/9bow/PyTorch-tutorials-kr" target="_blank">한국어 튜토리얼 저장소</a></li>
          </ul>
        </div>

      </div>
      <div class="trademark-disclaimer">
        <ul>
          <li>이 사이트는 PyTorch 한국 사용자 커뮤니티로 Facebook, Inc에서 운영하는 사이트가 아닙니다. PyTorch, PyTorch 로고 및 모든 관련 표기는 Facebook, Inc의 상표입니다.</li>
          <li>This site is a user community and is not operated by Facebook, Inc. PyTorch, the PyTorch logo and any related marks are trademarks of Facebook, Inc.</li>
        </ul>
      </div>
    </div>
  </footer>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.kr/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.kr/get-started">시작하기</a>
          </li>

          <li class="active">
            <a href="https://tutorials.pytorch.kr">튜토리얼</a>
          </li>

          <li>
            <a href="https://pytorch.kr/hub">허브</a>
          </li>

          <li>
            <a href="https://discuss.pytorch.kr">커뮤니티</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>