[LLM]Support Gemma model

llm/data.py

@@ -44,11 +44,11 @@ def get_convert_example(model):
 
     if base_model_prefix == "chatglm":
         return convert_example_chatglm
-    elif base_model_prefix in ["chatglm_v2", "llama", "bloom", "opt", "qwen", "mixtral"]:
+    elif base_model_prefix in ["chatglm_v2", "llama", "bloom", "opt", "qwen", "mixtral", "gemma"]:
         return convert_example_common
     else:
         raise ValueError(
-            f"Unknown base_model_prefix: {model.base_model_prefix}. Supported base_model_prefix list: chatglm, bloom, llama, qwen, mixtral"
+            f"Unknown base_model_prefix: {model.base_model_prefix}. Supported base_model_prefix list: chatglm, bloom, llama, qwen, mixtral, gemma"
         )
 
 

llm/gemma/README.md

@@ -0,0 +1,18 @@
+# Gemma
+
+## 1.模型介绍
+
+[Gemma](https://blog.google/technology/developers/gemma-open-models/)  由谷歌DeepMind和谷歌其他团队开发，是一个轻量级、最先进的开放式模型家族，采用与Gemini模型相同的研究和技术构建。
+
+**支持模型权重:**
+
+| Model              |
+| ------------------ |
+| google/gemma-7b    |
+| google/gemma-7b-it |
+| google/gemma-2b    |
+| google/gemma-2b-it |
+
+## 2. 模型精调
+
+请参考[LLM全流程工具介绍](../README.md)

llm/gemma/sft_argument.json

@@ -0,0 +1,30 @@
+{
+    "model_name_or_path": "google/gemma-2b/",
+    "dataset_name_or_path": "./data",
+    "output_dir": "./checkpoints/gemma_sft_ckpts",
+    "per_device_train_batch_size": 2,
+    "gradient_accumulation_steps": 1,
+    "per_device_eval_batch_size": 8,
+    "eval_accumulation_steps":16,
+    "num_train_epochs": 3,
+    "learning_rate": 3e-05,
+    "warmup_steps": 30,
+    "logging_steps": 1,
+    "evaluation_strategy": "epoch",
+    "save_strategy": "epoch",
+    "src_length": 512,
+    "max_length": 1024,
+    "fp16": true,
+    "fp16_opt_level": "O2",
+    "do_train": true,
+    "do_eval": true,
+    "disable_tqdm": true,
+    "load_best_model_at_end": true,
+    "eval_with_do_generation": false,
+    "metric_for_best_model": "accuracy",
+    "recompute": true,
+    "save_total_limit": 1,
+    "tensor_parallel_degree": 2,
+    "zero_padding": false,
+    "use_flash_attention": false
+  }

llm/gemma/sft_argument_7b.json

@@ -0,0 +1,32 @@
+{
+  "model_name_or_path": "google/gemma-7b",
+  "dataset_name_or_path": "./data",
+  "output_dir": "./checkpoints/gemma_sft_ckpts",
+  "per_device_train_batch_size": 8,
+  "gradient_accumulation_steps": 1,
+  "per_device_eval_batch_size": 8,
+  "eval_accumulation_steps":1,
+  "num_train_epochs": 3,
+  "learning_rate": 3e-06,
+  "warmup_steps": 30,
+  "logging_steps": 1,
+  "evaluation_strategy": "epoch",
+  "save_strategy": "epoch",
+  "src_length": 512,
+  "max_length": 1024,
+  "bf16": true,
+  "fp16_opt_level": "O2",
+  "do_train": true,
+  "do_eval": true,
+  "do_predict": true,
+  "disable_tqdm": true,
+  "load_best_model_at_end": true,
+  "eval_with_do_generation": false,
+  "metric_for_best_model": "accuracy",
+  "recompute": true,
+  "save_total_limit": 1,
+  "tensor_parallel_degree": 8,
+  "pipeline_parallel_degree": 1,
+  "zero_padding": false,
+  "use_flash_attention": false
+}

llm/gemma/sft_argument_7b_sharding.json

@@ -0,0 +1,33 @@
+{
+  "model_name_or_path": "google/gemma-7b",
+  "dataset_name_or_path": "./data",
+  "output_dir": "./checkpoints/llama_sft_ckpts",
+  "per_device_train_batch_size": 1,
+  "gradient_accumulation_steps": 1,
+  "per_device_eval_batch_size": 8,
+  "eval_accumulation_steps":1,
+  "num_train_epochs": 3,
+  "learning_rate": 3e-06,
+  "warmup_steps": 30,
+  "logging_steps": 1,
+  "evaluation_strategy": "epoch",
+  "save_strategy": "epoch",
+  "src_length": 1024,
+  "max_length": 2048,
+  "fp16": true,
+  "fp16_opt_level": "O2",
+  "do_train": true,
+  "do_eval": true,
+  "do_predict": true,
+  "disable_tqdm": true,
+  "load_best_model_at_end": true,
+  "eval_with_do_generation": false,
+  "metric_for_best_model": "accuracy",
+  "recompute": true,
+  "save_total_limit": 1,
+  "sharding_parallel_degree": 8,
+  "sharding": "stage3",
+  "pipeline_parallel_degree": 1,
+  "zero_padding": false,
+  "use_flash_attention": false
+}

llm/gemma/sft_argument_sharding.json

@@ -0,0 +1,31 @@
+{
+    "model_name_or_path": "google/gemma-2b/",
+    "dataset_name_or_path": "./data",
+    "output_dir": "./checkpoints/chatglm2_sft_ckpts",
+    "per_device_train_batch_size": 1,
+    "gradient_accumulation_steps": 1,
+    "per_device_eval_batch_size": 1,
+    "eval_accumulation_steps":1,
+    "num_train_epochs": 3,
+    "learning_rate": 3e-05,
+    "warmup_steps": 30,
+    "logging_steps": 1,
+    "evaluation_strategy": "epoch",
+    "save_strategy": "epoch",
+    "src_length": 512,
+    "max_length": 1024,
+    "fp16": true,
+    "fp16_opt_level": "O2",
+    "do_train": true,
+    "do_eval": true,
+    "disable_tqdm": true,
+    "load_best_model_at_end": true,
+    "eval_with_do_generation": false,
+    "metric_for_best_model": "accuracy",
+    "recompute": true,
+    "save_total_limit": 1,
+    "sharding_parallel_degree": 2,
+    "sharding": "stage3",
+    "zero_padding": false,
+    "use_flash_attention": false
+  }

paddlenlp/transformers/__init__.py

@@ -200,6 +200,7 @@
 from .gau_alpha.modeling import *
 from .gau_alpha.tokenizer import *
 from .gau_alpha.configuration import *
+from .gemma import *
 from .roformerv2.modeling import *
 from .roformerv2.tokenizer import *
 from .roformerv2.configuration import *

paddlenlp/transformers/auto/modeling.py

@@ -118,6 +118,7 @@
         ("Bloom", "bloom"),
         ("QWen", "qwen"),
         ("Mixtral", "mixtral"),
+        ("Gemma", "gemma"),
     ]
 )
 

paddlenlp/transformers/auto/tokenizer.py

@@ -97,6 +97,7 @@
         ("BloomTokenizer", "bloom"),
         ("SpeechT5Tokenizer", "speecht5"),
         ("QWenTokenizer", "qwen"),
+        ("GemmaTokenizer", "gemma"),
     ]
 )
 

paddlenlp/transformers/gemma/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .configuration import *
+from .modeling import *
+from .modeling_pp import *
+from .tokenizer import *