Skip to content

[AutoNLP]add visualdl #4990

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 27, 2023
Merged

[AutoNLP]add visualdl #4990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0213665
add visualdl
lugimzzz Feb 24, 2023
8eb0fb3
add
lugimzzz Feb 24, 2023
315a58b
update
lugimzzz Feb 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion paddlenlp/experimental/autonlp/auto_trainer_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ class AutoTrainerBase(metaclass=ABCMeta):
export_path = "exported_model" # filepath for the exported static model
results_filename = "experiment_results.csv" # filepath for storing experiment results
experiment_path = None # filepath for the experiment results
visualdl_path = "visualdl" # filepath for the visualdl

def __init__(
self,
Expand Down Expand Up @@ -98,6 +99,14 @@ def _default_training_argument(self) -> TrainingArguments:
"""
Default TrainingArguments for the Trainer
"""
return TrainingArguments(
output_dir=self.training_path,
disable_tqdm=True,
load_best_model_at_end=True,
save_total_limit=1,
report_to=["visualdl", "autonlp"],
logging_dir=self.visualdl_path, # if logging_dir is redefined, the function visualdl() should be redefined as well.
)

@property
@abstractmethod
Expand Down Expand Up @@ -189,7 +198,10 @@ def _override_hp(self, config: Dict[str, Any], default_hp: Any) -> Any:
new_hp = copy.deepcopy(default_hp)
for key, value in config.items():
if key in new_hp.to_dict():
setattr(new_hp, key, value)
if key in ["output_dir", "logging_dir"]:
logger.warning(f"{key} cannot be overridden")
else:
setattr(new_hp, key, value)
return new_hp

def _filter_model_candidates(
Expand Down Expand Up @@ -324,3 +336,10 @@ def train(
)

return self.training_results

def visualdl(self, trial_id: Optional[str] = None):
"""
Return visualdl path to represent the results of the taskflow training.
"""
model_result = self._get_model_result(trial_id=trial_id)
return os.path.join(model_result.log_dir, self.visualdl_path)
5 changes: 5 additions & 0 deletions paddlenlp/experimental/autonlp/text_classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ def supported_languages(self) -> List[str]:

@property
def _default_training_argument(self) -> TrainingArguments:
"""
Default TrainingArguments for the Trainer
"""
return TrainingArguments(
output_dir=self.training_path,
disable_tqdm=True,
Expand All @@ -115,6 +118,7 @@ def _default_training_argument(self) -> TrainingArguments:
save_strategy="epoch",
save_total_limit=1,
report_to=["visualdl", "autonlp"],
logging_dir=self.visualdl_path,
)

@property
Expand All @@ -129,6 +133,7 @@ def _default_prompt_tuning_arguments(self) -> PromptTuningArguments:
save_strategy="epoch",
save_total_limit=1,
report_to=["visualdl", "autonlp"],
logging_dir=self.visualdl_path,
)

@property
Expand Down
2 changes: 1 addition & 1 deletion paddlenlp/trainer/integrations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def on_log(self, args, state, control, logs=None, **kwargs):
return

if self.vdl_writer is None:
self._init_summary_writer(args)
return
Copy link
Contributor Author

@lugimzzz lugimzzz Feb 27, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

trainer里evaluate调用log函数的时候会新增一个visualdl的log文件。

这块修改需要 @ZHUI 确认是否没问题

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

huggingface/transformers#13636


if self.vdl_writer is not None:
logs = rewrite_logs(logs)
Expand Down
9 changes: 9 additions & 0 deletions tests/experimental/autonlp/test_text_classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,9 @@ def test_multiclass(self, custom_model_candidate, hp_overrides):
self.assertTrue(os.path.exists(os.path.join(save_path, "template_config.json")))
self.assertTrue(os.path.exists(os.path.join(save_path, "verbalizer_config.json")))

# test visualdl
self.assertTrue(os.path.isdir(auto_trainer.visualdl()))

# test evaluate
copy_dev_ds = copy.deepcopy(self.multi_class_dev_ds)
eval_metrics1 = auto_trainer.evaluate()
Expand Down Expand Up @@ -259,6 +262,9 @@ def test_multilabel(self, custom_model_candidate, hp_overrides):
self.assertTrue(os.path.exists(os.path.join(save_path, "template_config.json")))
self.assertTrue(os.path.exists(os.path.join(save_path, "verbalizer_config.json")))

# test visualdl
self.assertTrue(os.path.isdir(auto_trainer.visualdl()))

# test evaluate
copy_dev_ds = copy.deepcopy(self.multi_label_dev_ds)
eval_metrics1 = auto_trainer.evaluate()
Expand Down Expand Up @@ -367,6 +373,9 @@ def test_default_model_candidate(self, language, hp_overrides):
self.assertTrue(os.path.exists(os.path.join(save_path, "template_config.json")))
self.assertTrue(os.path.exists(os.path.join(save_path, "verbalizer_config.json")))

# test visualdl
self.assertTrue(os.path.isdir(auto_trainer.visualdl()))

# test evaluate
copy_dev_ds = copy.deepcopy(self.multi_class_dev_ds)
eval_metrics1 = auto_trainer.evaluate()
Expand Down