Dsv3 dev

.pre-commit-config.yaml

@@ -62,3 +62,12 @@ repos:
         language: python
         files: \.(md|markdown|rst)$
         pass_filenames: true
+
+-   repo: local
+    hooks:
+    -   id: clang-format
+        name: clang-format
+        description: Format files with ClangFormat.
+        entry: bash ./tools/codestyle/clang_format.sh -i
+        language: system
+        files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$

llm/config/deepseek-v3/pretrain_argument.json

@@ -1,41 +1,47 @@
 {
-  "model_name_or_path": "deepseek-ai/DeepSeek-V3",
+  "model_name_or_path": "./model_config/DeepSeek-V3-test",
   "tokenizer_name_or_path": "deepseek-ai/DeepSeek-V3",
   "input_dir": "./data",
   "output_dir": "./checkpoints/pretrain_ckpts",
   "per_device_train_batch_size": 1,
-  "gradient_accumulation_steps": 4,
+  "gradient_accumulation_steps": 120,
   "per_device_eval_batch_size": 1,
   "tensor_parallel_degree": 1,
   "pipeline_parallel_degree": 1,
-  "sharding_parallel_degree": 8,
-  "expert_parallel_degree": 4,
+  "sharding_parallel_degree": 64,
+  "sharding_parallel_config": "split_param enable_fuse_optimizer_states",
+  "sharding_comm_buffer_size_MB": 2048,
+  "expert_parallel_degree": 64,
   "sharding": "stage1",
   "virtual_pp_degree": 1,
   "sequence_parallel": 0,
   "use_flash_attention": true,
-  "max_seq_length": 4096,
+  "max_seq_length": 4097,
   "learning_rate": 3e-05,
   "min_learning_rate": 3e-06,
   "warmup_steps": 30,
   "logging_steps": 1,
-  "max_steps": 10000,
+  "max_steps": 200,
   "save_steps": 5000,
   "eval_steps": 1000,
   "weight_decay": 0.01,
   "bf16": true,
   "fp16_opt_level": "O2",
   "warmup_ratio": 0.01,
   "max_grad_norm": 1.0,
-  "dataloader_num_workers": 1,
+  "amp_master_grad": 1,
+  "dataloader_num_workers": 8,
   "continue_training": 0,
   "do_train": true,
   "do_eval": true,
   "do_predict": false,
   "disable_tqdm": true,
-  "recompute": true,
+  "recompute": false,
   "distributed_dataloader": 1,
-  "recompute_granularity": "full",
   "unified_checkpoint": true,
-  "save_total_limit": 2
-}
+  "save_total_limit": 2,
+  "skip_profile_timer": false,
+  "use_fused_rms_norm": true,
+  "fuse_attention_ffn": true,
+  "use_fused_rope": true
+}

llm/model_config/DeepSeek-V3-test/config.json

@@ -0,0 +1,67 @@
+{
+    "architectures": [
+      "DeepseekV3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_deepseek.DeepseekV3Config",
+      "AutoModel": "modeling_deepseek.DeepseekV3Model",
+      "AutoModelForCausalLM": "modeling_deepseek.DeepseekV3ForCausalLM"
+    },
+    "aux_loss_alpha": 0.001,
+    "bos_token_id": 0,
+    "eos_token_id": 1,
+    "ep_size": 1,
+    "first_k_dense_replace": 0,
+    "hidden_act": "silu",
+    "hidden_size": 7168,
+    "initializer_range": 0.02,
+    "intermediate_size": 18432,
+    "kv_lora_rank": 512,
+    "max_position_embeddings": 163840,
+    "model_type": "deepseek_v3",
+    "moe_intermediate_size": 2048,
+    "moe_layer_freq": 1,
+    "n_group": 8,
+    "n_routed_experts": 256,
+    "n_shared_experts": 1,
+    "norm_topk_prob": true,
+    "num_attention_heads": 128,
+    "num_experts_per_tok": 8,
+    "num_hidden_layers": 2,
+    "num_key_value_heads": 128,
+    "num_nextn_predict_layers": 1,
+    "pretraining_tp": 1,
+    "q_lora_rank": 1536,
+    "qk_nope_head_dim": 128,
+    "qk_rope_head_dim": 64,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "beta_fast": 32,
+      "beta_slow": 1,
+      "factor": 40,
+      "mscale": 1.0,
+      "mscale_all_dim": 1.0,
+      "original_max_position_embeddings": 4096,
+      "type": "yarn"
+    },
+    "rope_theta": 10000,
+    "routed_scaling_factor": 2.5,
+    "scoring_func": "sigmoid",
+    "seq_aux": true,
+    "tie_word_embeddings": false,
+    "topk_group": 4,
+    "topk_method": "noaux_tc",
+    "dtype": "bfloat16",
+    "transformers_version": "4.33.1",
+    "use_cache": true,
+    "v_head_dim": 128,
+    "vocab_size": 129280,
+    "using_flex_token": true,
+    "using_fake_gate": true,
+    "use_fused_rms_norm": true,
+    "fuse_attention_ffn": true,
+    "use_fused_rope": true,
+    "token_drop_steps": true
+  }