detach arg option for run scripts

scripts/performance/argument_parser.py

@@ -422,5 +422,12 @@ def list_of_strings(arg):
         required=False,
         default=False,
     )
+    parser.add_argument(
+        "--detach",
+        help="Detach from experiment. Default is True. Unset to keep the process running.",
+        type=bool_arg,
+        required=False,
+        default=True,
+    )
 
     return parser

scripts/performance/diffusion/pretrain_flux_12b.py

@@ -127,6 +127,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/finetune_deepseek_v3.py

@@ -188,6 +188,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/finetune_llama31_405b.py

@@ -222,6 +222,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/finetune_llama3_70b.py

@@ -230,6 +230,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/finetune_llama3_8b.py

@@ -167,6 +167,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/finetune_llama4_e128.py

@@ -210,6 +210,6 @@ def override_recipe_configs(
             plugins=plugins,
         )
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/mlperf_lora_llama2_70b.py

@@ -368,6 +368,6 @@ def mlperf_lora_llama2_70b_recipe(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_automodel_llama3_8b.py

@@ -121,6 +121,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_deepseek_v3.py

@@ -254,7 +254,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_gpt3_175b.py

@@ -203,7 +203,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_llama31_405b.py

@@ -138,7 +138,7 @@ def override_recipe_configs(
     recipe.trainer.callbacks[comm_overlap_callback_idx].tp_comm_overlap_cfg = tp_comm_overlap_cfg
 
     if use_mcore_fsdp and gpu_type == 'gb200':
-        recipe.trainer.strategy.num_distributed_optimizer_instances = (num_nodes * 4) / 64
+        recipe.trainer.strategy.num_distributed_optimizer_instances = (num_nodes * 4) // 64
 
     return recipe
 
@@ -223,7 +223,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_llama3_70b.py

@@ -214,7 +214,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_llama3_8b.py

@@ -181,7 +181,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_llama4_e128.py

@@ -149,7 +149,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_llama4_e16.py

@@ -154,7 +154,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_mixtral_8x22b.py

@@ -174,6 +174,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_mixtral_8x7b.py

@@ -142,6 +142,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_nemotron3_22b.py

@@ -165,6 +165,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_nemotron3_8b.py

@@ -134,6 +134,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_nemotron4_15b.py

@@ -149,6 +149,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_nemotron4_340b.py

@@ -190,6 +190,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/llm/pretrain_nemotronh_47b.py

@@ -163,7 +163,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_nemotronh_56b.py

@@ -164,7 +164,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_nemotronh_8b.py

@@ -163,7 +163,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_qwen3_235b_a22b.py

@@ -154,7 +154,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/llm/pretrain_qwen3_30b_a3b.py

@@ -153,7 +153,7 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()
 

scripts/performance/vlm/finetune_neva_8b.py

@@ -132,6 +132,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/vlm/finetune_qwen25vl_32b.py

@@ -135,6 +135,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/vlm/finetune_qwen25vl_7b.py

@@ -135,6 +135,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/vlm/pretrain_vlm_llama4_e128.py

@@ -160,6 +160,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()

scripts/performance/vlm/pretrain_vlm_llama4_e16.py

@@ -160,6 +160,6 @@ def override_recipe_configs(
         )
 
         if not args.dryrun:
-            exp.run(sequential=True, detach=True)
+            exp.run(sequential=True, detach=args.detach)
         else:
             exp.dryrun()