Merge pull request #18 from eltociear/patch-1

Update initialize.py

Author: HoratioJSY

megatron_mini/initialize.py

@@ -119,7 +119,7 @@ def _initialize_distributed():
 
 
 def _set_random_seed(seed_, data_parallel_random_init=False):
-    """Set random seed for reproducability."""
+    """Set random seed for reproducibility."""
     if seed_ is not None and seed_ > 0:
         # Ensure that different pipeline MP stages get different seeds.
         seed = seed_ + (100 * mpu.get_pipeline_model_parallel_rank())
@@ -135,7 +135,7 @@ def _set_random_seed(seed_, data_parallel_random_init=False):
         raise ValueError('Seed ({}) should be a positive integer.'.format(seed_))
 
 def _set_random_seed(seed_, data_parallel_random_init=False):
-    """Set random seed for reproducability."""
+    """Set random seed for reproducibility."""
     if seed_ is not None and seed_ > 0:
         # Ensure that different pipeline MP stages get different seeds.
         seed = seed_ + (100 * mpu.get_pipeline_model_parallel_rank())
@@ -148,4 +148,4 @@ def _set_random_seed(seed_, data_parallel_random_init=False):
         if torch.cuda.device_count() > 0:
             tensor_parallel.model_parallel_cuda_manual_seed(seed)
     else:
-        raise ValueError('Seed ({}) should be a positive integer.'.format(seed_))
+        raise ValueError('Seed ({}) should be a positive integer.'.format(seed_))