From 37134d561fc9f3d4839069a479f58eae059ebae2 Mon Sep 17 00:00:00 2001 From: lugimzzz Date: Thu, 11 Apr 2024 15:52:56 +0800 Subject: [PATCH 1/4] add scaling --- paddlenlp/peft/lora/lora_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paddlenlp/peft/lora/lora_config.py b/paddlenlp/peft/lora/lora_config.py index 3a0897bc4892..29d4cdca928d 100644 --- a/paddlenlp/peft/lora/lora_config.py +++ b/paddlenlp/peft/lora/lora_config.py @@ -93,6 +93,7 @@ def __post_init__(self): "We will automatically set `use_quick_lora` to `False` to avoid potential inconsistencies." ) self.use_quick_lora = False + self.lora_scaling = self.lora_alpha / self.r @property def __dict__(self): From 5b00bad8d7adbc0d7a59df814e332cd47b67fd64 Mon Sep 17 00:00:00 2001 From: lugimzzz Date: Thu, 11 Apr 2024 16:45:53 +0800 Subject: [PATCH 2/4] add scaling --- paddlenlp/peft/lora/lora_config.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/paddlenlp/peft/lora/lora_config.py b/paddlenlp/peft/lora/lora_config.py index 29d4cdca928d..6083507a420b 100644 --- a/paddlenlp/peft/lora/lora_config.py +++ b/paddlenlp/peft/lora/lora_config.py @@ -14,6 +14,7 @@ import json import os +import math from dataclasses import asdict, dataclass, field from typing import List, Optional, Union @@ -93,7 +94,15 @@ def __post_init__(self): "We will automatically set `use_quick_lora` to `False` to avoid potential inconsistencies." ) self.use_quick_lora = False - self.lora_scaling = self.lora_alpha / self.r + + @property + def scaling(self): + if not self.rslora and not self.pissa: + return self.lora_alpha / self.r + elif self.pissa: + return 1.0 + else: + return self.lora_alpha / math.sqrt(self.r) @property def __dict__(self): @@ -115,6 +124,7 @@ def save_pretrained(self, save_directory): os.makedirs(save_directory, exist_ok=True) output_dict = self.__dict__ + output_dict["scaling"] = self.scaling output_path = os.path.join(save_directory, LORA_CONFIG_NAME) # save it From 0a201c6b5c1d4609eb26de3546afaefccb3eb510 Mon Sep 17 00:00:00 2001 From: lugimzzz Date: Thu, 11 Apr 2024 17:10:59 +0800 Subject: [PATCH 3/4] add scaling --- paddlenlp/peft/lora/lora_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/paddlenlp/peft/lora/lora_config.py b/paddlenlp/peft/lora/lora_config.py index 6083507a420b..dadcacf07c54 100644 --- a/paddlenlp/peft/lora/lora_config.py +++ b/paddlenlp/peft/lora/lora_config.py @@ -147,6 +147,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): raise ValueError(f"Can't find lora_config.json at '{pretrained_model_name_or_path}'") loaded_attributes = cls.from_json_file(config_file) + loaded_attributes.pop("scaling", None) config = cls(**kwargs) From 2549551da19939d2a0c9e776f28f9f5e6385e808 Mon Sep 17 00:00:00 2001 From: lugimzzz Date: Thu, 11 Apr 2024 19:44:16 +0800 Subject: [PATCH 4/4] format --- paddlenlp/peft/lora/lora_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddlenlp/peft/lora/lora_config.py b/paddlenlp/peft/lora/lora_config.py index dadcacf07c54..8d58321e988b 100644 --- a/paddlenlp/peft/lora/lora_config.py +++ b/paddlenlp/peft/lora/lora_config.py @@ -13,8 +13,8 @@ # limitations under the License. import json -import os import math +import os from dataclasses import asdict, dataclass, field from typing import List, Optional, Union