shen/ms-swift
1
0
Fork 0
mirror of https://github.com/modelscope/ms-swift.git synced 2024-11-25 18:32:44 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix eval_dataset no (#2497)

Browse Source
This commit is contained in:
Jintao 2024-11-25 13:47:31 +08:00 committed by GitHub
parent 279b524314
commit 5431a57928
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
swift/llm/sft.py
View File

@ -355,11 +355,6 @@ def prepare_dataset(args, template: Template, msg: Optional[Dict[str, Any]] = No
if use_torchacc():
training_args.train_dataset_sample = train_dataset.shape[0] if train_dataset is not None else 0
if val_dataset is None:
training_args.evaluation_strategy = IntervalStrategy.NO
training_args.eval_strategy = IntervalStrategy.NO
training_args.do_eval = False
tokenizer = template.tokenizer
dataset_info = {}
if args.packing:
@ -415,6 +410,11 @@ def prepare_dataset(args, template: Template, msg: Optional[Dict[str, Any]] = No
train_dataset = LazyLLMDataset(train_dataset, template.encode)
if val_dataset is not None:
val_dataset = LazyLLMDataset(val_dataset, template.encode)
if val_dataset is None:
training_args.evaluation_strategy = IntervalStrategy.NO
training_args.eval_strategy = IntervalStrategy.NO
training_args.do_eval = False
if isinstance(msg, dict):
msg['dataset_info'] = dataset_info
return train_dataset, val_dataset