add llama3.1 config

Author: zhiqiu

tests/test_tipc/static/auto_parallel/llama2/pretrain_config_llama3.1_8b/pretrain_llama3.1_8b.json

@@ -0,0 +1,57 @@
+{
+  "model_name_or_path": "meta-llama/Meta-Llama-3.1-8B",
+  "tokenizer_name_or_path": "meta-llama/Meta-Llama-3.1-8B",
+  "input_dir": "./data",
+  "output_dir": "./checkpoints/llama3.1_pretrain_ckpts",
+  "per_device_train_batch_size": 1,
+  "gradient_accumulation_steps": 1,
+  "per_device_eval_batch_size": 2,
+  "tensor_parallel_degree": 1,
+  "pipeline_parallel_degree": 1,
+  "sharding": "stage1",
+  "data_parallel_config": "enable_allreduce_avg_in_gradinent_scale gradient_sync_after_accumulate",
+  "sharding_parallel_config": "enable_stage1_overlap",
+  "tensor_parallel_config": "enable_mp_async_allreduce",
+  "pipeline_parallel_config": "",
+  "virtual_pp_degree": 1,
+  "sequence_parallel": 0,   
+  "use_flash_attention": true,
+  "use_fused_rms_norm": true,
+  "fuse_attention_ffn": true,
+  "fuse_attention_qkv": true,
+  "use_fused_rope": true,
+  "fused_linear_param_grad_add": true,
+  "max_seq_length": 4096,
+  "learning_rate": 3e-05,
+  "min_learning_rate": 3e-06,
+  "warmup_steps": 30,
+  "logging_steps": 10,
+  "max_steps": 500,
+  "save_steps": 5000,
+  "eval_steps": 1000,
+  "weight_decay": 0.01,
+  "bf16": true,
+  "fp16_opt_level": "O2",
+  "amp_master_grad": true,
+  "amp_custom_black_list": ["reduce_sum", "c_softmax_with_cross_entropy"],
+  "amp_custom_white_list": ["lookup_table", "lookup_table_v2"],
+  "warmup_ratio": 0.01,
+  "max_grad_norm": 1.0,
+  "dataloader_num_workers": 1,
+  "continue_training": 0,
+  "do_train": true,
+  "do_eval": false,
+  "do_predict": false,
+  "disable_tqdm": true,
+  "skip_profile_timer": true,
+  "recompute": false,
+  "recompute_use_reentrant": true,
+  "distributed_dataloader": 0,
+  "recompute_granularity": "full",
+  "save_total_limit": 2,
+  "device": "gpu",
+  "to_static": true,
+  "enable_auto_parallel": true
+}
+
+