CU-8696kd70p improve the eval mode and add plot on metrics

Author: baixiac

app/api/routers/supervised_training.py

@@ -29,7 +29,7 @@ async def train_supervised(request: Request,
                            trainer_export: Annotated[List[UploadFile], File(description="One or more trainer export files to be uploaded")],
                            epochs: Annotated[int, Query(description="The number of training epochs", ge=0)] = 1,
                            lr_override: Annotated[Union[float, None], Query(description="The override of the initial learning rate", gt=0.0)] = None,
-                           test_size: Annotated[Union[float, None], Query(description="The override of the test size in percentage", ge=0.0)] = 0.2,
+                           test_size: Annotated[Union[float, None], Query(description="The override of the test size in percentage. (For a 'huggingface-ner' model, a negative value can be used to apply the train-validation-test split if implicitly defined in trainer export: 'projects[0]' is used for training, 'projects[1]' for validation, and 'projects[2]' for testing)")] = 0.2,
                            log_frequency: Annotated[int, Query(description="The number of processed documents after which training metrics will be logged", ge=1)] = 1,
                            description: Annotated[Union[str, None], Form(description="The description of the training or change logs")] = None,
                            model_service: AbstractModelService = Depends(cms_globals.model_service_dep)) -> JSONResponse:

app/management/tracker_client.py

@@ -72,7 +72,7 @@ def send_model_stats(stats: Dict, step: int) -> None:
         mlflow.log_metrics(metrics, step)
 
     @staticmethod
-    def send_hf_training_logs(logs: Dict, step: int) -> None:
+    def send_hf_metrics_logs(logs: Dict, step: int) -> None:
         mlflow.log_metrics(logs, step)
 
     @staticmethod

app/model_services/huggingface_ner_model.py

@@ -79,8 +79,8 @@ def from_model(cls, model: PreTrainedModel, tokenizer: PreTrainedTokenizerBase)
         model_service.tokenizer = tokenizer
         _pipeline = partial(pipeline,
                             task="ner",
-                            model=model,
-                            tokenizer=tokenizer,
+                            model=model_service.model,
+                            tokenizer=model_service.tokenizer,
                             stride=10,
                             aggregation_strategy=get_settings().HF_PIPELINE_AGGREGATION_STRATEGY)
         if non_default_device_is_available(get_settings().DEVICE):
@@ -139,7 +139,7 @@ def annotate(self, text: str) -> Dict:
             df = pd.DataFrame(columns=["label_name", "label_id", "start", "end", "accuracy"])
         else:
             for idx, row in df.iterrows():
-                df.loc[idx, "label_id"] = str(self._model.config.label2id[row["entity_group"]])
+                df.loc[idx, "label_id"] = row["entity_group"]
             df.rename(columns={"entity_group": "label_name", "score": "accuracy"}, inplace=True)
         records = df.to_dict("records")
         return records

app/trainers/base.py

@@ -73,7 +73,10 @@ def start_training(self,
                     self._tracker_client.save_processed_artifact(data_file.name, self._model_name)
 
                     dataset = None
-                    if training_type == TrainingType.UNSUPERVISED.value and isinstance(data_file, TextIO):
+                    if training_type == TrainingType.UNSUPERVISED.value and isinstance(data_file, tempfile.TemporaryDirectory):
+                        dataset = datasets.load_from_disk(data_file.name)
+                        self._tracker_client.save_train_dataset(dataset)
+                    elif training_type == TrainingType.UNSUPERVISED.value:
                         try:
                             dataset = datasets.load_dataset(doc_dataset.__file__,
                                                             data_files={"documents": data_file.name},
@@ -84,7 +87,7 @@ def start_training(self,
                         finally:
                             if dataset is not None:
                                 dataset.cleanup_cache_files()
-                    elif training_type == TrainingType.SUPERVISED.value and isinstance(data_file, TextIO):
+                    elif training_type == TrainingType.SUPERVISED.value:
                         try:
                             dataset = datasets.load_dataset(anno_dataset.__file__,
                                                             data_files={"annotations": data_file.name},
@@ -95,9 +98,6 @@ def start_training(self,
                         finally:
                             if dataset is not None:
                                 dataset.cleanup_cache_files()
-                    elif training_type == TrainingType.UNSUPERVISED.value and isinstance(data_file, tempfile.TemporaryDirectory):
-                        dataset = datasets.load_from_disk(data_file.name)
-                        self._tracker_client.save_train_dataset(dataset)
                     else:
                         raise ValueError(f"Unknown training type: {training_type}")