mirror of
https://github.com/deepspeedai/DeepSpeed.git
synced 2025-10-20 15:33:51 +08:00
18 lines
484 B
JSON
18 lines
484 B
JSON
{
|
|
"zero_optimization": {
|
|
"stage": 3,
|
|
"allgather_partitions": true,
|
|
"allgather_bucket_size": 5e8,
|
|
"overlap_comm": false,
|
|
"reduce_scatter": true,
|
|
"reduce_bucket_size": 5e8,
|
|
"contiguous_gradients": false,
|
|
"stage3_max_live_parameters": 1e9,
|
|
"stage3_max_reuse_distance": 1e9,
|
|
"stage3_prefetch_bucket_size": 5e8,
|
|
"stage3_param_persistence_threshold": 1e6,
|
|
"stage3_gather_fp16_weights_on_model_save": false,
|
|
"sub_group_size": 1e12
|
|
}
|
|
}
|