Merge pull request #151 from AutomatedProcessImprovement/146-generate-schema-for-yaml-configuration

146 generate schema for yaml configuration

Author: iharsuvorau

poetry.lock

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "simod"
-version = "3.6.10"
+version = "3.6.11"
 authors = [
     "Ihar Suvorau <ihar.suvorau@gmail.com>",
     "David Chapela <david.chapela@ut.ee>",
@@ -18,14 +18,13 @@ packages = [{ include = "simod", from = "src" }]
 python = "^3.9,<3.12"
 click = "^8.1.3"
 hyperopt = "^0.2.7"
-jellyfish = "^0.11"
 lxml = "^4.9.1"
 matplotlib = "^3.6.0"
 networkx = "^3.1"
 numpy = "^1.23.4"
-pandas = "^2"
+pandas = "^2.1.0"
 pendulum = "^2.1.2"
-pydantic = "^1.10.3"
+pydantic = "^2.3.0"
 python-dotenv = "^1.0.0"
 python-multipart = "^0.0.6"
 pytz = "^2023.3"

pyproject.toml

@@ -14,7 +14,7 @@ common:
     start_time: "start_time"
     end_time: "end_time"
   # Use this process model and skip its discovery
-  model_path: ../models/LoanApp_simplified.bpmn
+  process_model_path: ../models/LoanApp_simplified.bpmn
   # Event log to evaluate the discovered BPS model with
   test_log_path: ../event_logs/LoanApp_simplified_test.csv.gz
   # Flag to perform evaluation (if 'test_log_path' not provided) with a test partition of the input log

resources/config/complete_configuration.yml

@@ -13,7 +13,7 @@ common:
     start_time: "start_time"
     end_time: "end_time"
   # Use this process model and skip its discovery
-  model_path: ../models/LoanApp_simplified.bpmn
+  process_model_path: ../models/LoanApp_simplified.bpmn
   # Whether to discover case attributes or not
   discover_case_attributes: false
 #################

resources/config/configuration_example_with_provided_process_model.yml

@@ -2,7 +2,7 @@ version: 4
 common:
   train_log_path: ../event_logs/LoanApp_simplified_train.csv.gz
   test_log_path: ../event_logs/LoanApp_simplified_test.csv.gz
-  # model_path: resources/models/LoanApp_simplified.bpmn  # Uncomment to use this BPMN model as process model
+  # process_model_path: resources/models/LoanApp_simplified.bpmn  # Uncomment to use this BPMN model as process model
   num_final_evaluations: 10  # Number of evaluations of the discovered BPS model.
   evaluation_metrics: # Metrics to evaluate the discovered BPS model with.
     - 3_gram_distance

resources/config/sample.yml

@@ -1,7 +1,9 @@
+import json
 from pathlib import Path
 from typing import Optional
 
 import click
+import yaml
 from pix_framework.filesystem.file_manager import get_random_folder_id
 
 from simod.event_log.event_log import EventLog
@@ -47,7 +49,35 @@
     help="Path to the event log file when using the --one-shot flag. "
     "Columns must be named 'case_id', 'activity', 'start_time', 'end_time', 'resource'.",
 )
-def main(configuration: Optional[Path], output: Optional[Path], one_shot: bool, event_log: Optional[Path]) -> Path:
+@click.option(
+    "--schema-yaml",
+    required=False,
+    is_flag=True,
+    help="Print the configuration YAML schema and exit.",
+)
+@click.option(
+    "--schema-json",
+    required=False,
+    is_flag=True,
+    help="Print the configuration JSON schema and exit.",
+)
+@click.version_option()
+def main(
+    configuration: Optional[Path],
+    output: Optional[Path],
+    one_shot: bool,
+    event_log: Optional[Path],
+    schema_yaml: bool,
+    schema_json: bool,
+) -> None:
+    if schema_yaml:
+        print(yaml.dump(SimodSettings().model_json_schema()))
+        return
+
+    if schema_json:
+        print(json.dumps(SimodSettings().model_json_schema()))
+        return
+
     if one_shot:
         settings = SimodSettings.one_shot()
         settings.common.train_log_path = event_log
@@ -70,8 +100,6 @@ def main(configuration: Optional[Path], output: Optional[Path], one_shot: bool,
     simod = Simod(settings, event_log=event_log, output_dir=output)
     simod.run()
 
-    return output
-
 
 if __name__ == "__main__":
     main()

src/simod/cli.py

@@ -160,7 +160,9 @@ def run(self) -> HyperoptIterationParams:
         # Process best results
         results = pd.DataFrame(self._bayes_trials.results).sort_values("loss")
         best_result = results[results.status == STATUS_OK].iloc[0]
-        assert best_result["model_path"].exists(), f"Best model path {best_result['model_path']} does not exist"
+        assert best_result[
+            "process_model_path"
+        ].exists(), f"Best model path {best_result['process_model_path']} does not exist"
 
         # Re-build parameters of the best hyperopt iteration
         best_hyperopt_parameters = HyperoptIterationParams.from_hyperopt_dict(
@@ -177,7 +179,7 @@ def run(self) -> HyperoptIterationParams:
         # Update best process model (save it in base directory)
         self.best_bps_model.process_model = get_process_model_path(self.base_directory, self.event_log.process_name)
         best_model_path = (
-            best_result["model_path"] if self._need_to_discover_model else self.initial_bps_model.process_model
+            best_result["process_model_path"] if self._need_to_discover_model else self.initial_bps_model.process_model
         )
         shutil.copyfile(best_model_path, self.best_bps_model.process_model)
         # Update simulation parameters (save them in base directory)
@@ -245,7 +247,7 @@ def cleanup(self):
 
     @staticmethod
     def _define_response(
-        status: str, evaluation_measurements: list, output_dir: Path, model_path: Path
+        status: str, evaluation_measurements: list, output_dir: Path, process_model_path: Path
     ) -> Tuple[str, dict]:
         # Compute mean distance if status is OK
         if status is STATUS_OK:
@@ -260,7 +262,7 @@ def _define_response(
             "loss": distance,  # Loss value for the fmin function
             "status": status,  # Status of the optimization iteration
             "output_dir": output_dir,
-            "model_path": model_path,
+            "process_model_path": process_model_path,
         }
         # Return updated status and processed response
         return status, response
@@ -309,7 +311,7 @@ def _simulate_bps_model(self, bps_model: BPSModel, output_dir: Path) -> List[dic
         json_parameters_path = bps_model.to_json(output_dir, self.event_log.process_name)
 
         evaluation_measures = simulate_and_evaluate(
-            model_path=bps_model.process_model,
+            process_model_path=bps_model.process_model,
             parameters_path=json_parameters_path,
             output_dir=output_dir,
             simulation_cases=self.event_log.validation_partition[self.event_log.log_ids.case].nunique(),

src/simod/control_flow/optimizer.py

@@ -15,6 +15,8 @@
 from pix_framework.discovery.resource_profiles import discover_pool_resource_profiles
 from pix_framework.filesystem.file_manager import create_folder, get_random_folder_id, remove_asset
 
+from .repair import repair_with_missing_activities
+from .settings import HyperoptIterationParams
 from ..batching.discovery import discover_batching_rules
 from ..cli_formatter import print_message, print_step, print_subsection
 from ..event_log.event_log import EventLog
@@ -23,8 +25,6 @@
 from ..simulation.parameters.BPS_model import BPSModel
 from ..simulation.prosimos import simulate_and_evaluate
 from ..utilities import get_process_model_path, get_simulation_parameters_path, hyperopt_step
-from .repair import repair_with_missing_activities
-from .settings import HyperoptIterationParams
 
 
 class ResourceModelOptimizer:
@@ -118,7 +118,7 @@ def _hyperopt_iteration(self, hyperopt_iteration_dict: dict):
             optimization_metric=self.settings.optimization_metric,
             discovery_type=self.settings.discovery_type,
             output_dir=output_dir,
-            model_path=current_bps_model.process_model,
+            process_model_path=current_bps_model.process_model,
             project_name=self.event_log.process_name,
         )
         print_message(f"Parameters: {hyperopt_iteration_params}")
@@ -194,14 +194,14 @@ def run(self) -> HyperoptIterationParams:
             discovery_type=self.settings.discovery_type,
             output_dir=best_result["output_dir"],
             project_name=self.event_log.process_name,
-            model_path=self.initial_bps_model.process_model,
+            process_model_path=self.initial_bps_model.process_model,
         )
 
         # Instantiate best BPS model
         self.best_bps_model = self.initial_bps_model.deep_copy()
         # Update best process model (save it in base directory)
         self.best_bps_model.process_model = get_process_model_path(self.base_directory, self.event_log.process_name)
-        shutil.copyfile(best_result["model_path"], self.best_bps_model.process_model)
+        shutil.copyfile(best_result["process_model_path"], self.best_bps_model.process_model)
         # Update simulation parameters (save them in base directory)
         best_parameters_path = get_simulation_parameters_path(self.base_directory, self.event_log.process_name)
         shutil.copyfile(
@@ -314,7 +314,7 @@ def _process_measurements(self, params: HyperoptIterationParams, status: str, ev
 
     @staticmethod
     def _define_response(
-        status: str, evaluation_measurements: list, output_dir: Path, model_path: Path
+        status: str, evaluation_measurements: list, output_dir: Path, process_model_path: Path
     ) -> Tuple[str, dict]:
         # Compute mean distance if status is OK
         if status is STATUS_OK:
@@ -329,7 +329,7 @@ def _define_response(
             "loss": distance,  # Loss value for the fmin function
             "status": status,  # Status of the optimization iteration
             "output_dir": output_dir,
-            "model_path": model_path,
+            "process_model_path": process_model_path,
         }
         # Return updated status and processed response
         return status, response
@@ -340,7 +340,7 @@ def _simulate_bps_model(self, bps_model: BPSModel, output_dir: Path, granularity
         json_parameters_path = bps_model.to_json(output_dir, self.event_log.process_name, granule_size=granularity)
 
         evaluation_measures = simulate_and_evaluate(
-            model_path=bps_model.process_model,
+            process_model_path=bps_model.process_model,
             parameters_path=json_parameters_path,
             output_dir=output_dir,
             simulation_cases=self.event_log.validation_partition[self.event_log.log_ids.case].nunique(),

src/simod/resource_model/optimizer.py

@@ -16,7 +16,7 @@ class HyperoptIterationParams:
 
     # General settings
     output_dir: Path  # Directory where to output all the files of the current iteration
-    model_path: Path  # Path to BPMN model
+    process_model_path: Path  # Path to BPMN model
     project_name: str  # Name of the project for file naming
 
     optimization_metric: Metric  # Metric to evaluate the candidate of this iteration
@@ -29,7 +29,7 @@ def to_dict(self) -> dict:
         # Save common params
         optimization_parameters = {
             "output_dir": str(self.output_dir),
-            "model_path": str(self.model_path),
+            "process_model_path": str(self.process_model_path),
             "project_name": str(self.project_name),
             "optimization_metric": str(self.optimization_metric),
             "discover_prioritization_rules": str(self.discover_prioritization_rules),
@@ -44,7 +44,7 @@ def from_hyperopt_dict(
         optimization_metric: Metric,
         discovery_type: CalendarType,
         output_dir: Path,
-        model_path: Path,
+        process_model_path: Path,
         project_name: str,
     ) -> "HyperoptIterationParams":
         """Create the params for this run from the hyperopt dictionary returned by the fmin function."""
@@ -78,7 +78,7 @@ def safe_granularity(granularity: int) -> int:
 
         return HyperoptIterationParams(
             output_dir=output_dir,
-            model_path=model_path,
+            process_model_path=process_model_path,
             project_name=project_name,
             optimization_metric=optimization_metric,
             calendar_discovery_params=CalendarDiscoveryParameters(

src/simod/resource_model/settings.py

@@ -1,9 +1,10 @@
-from dataclasses import dataclass, field
+from dataclasses import field
 from enum import Enum
 from pathlib import Path
 from typing import Union, List, Optional
 
 from pix_framework.io.event_log import EventLogIDs, PROSIMOS_LOG_IDS
+from pydantic import BaseModel
 
 from ..utilities import get_project_dir
 
@@ -76,13 +77,12 @@ def __str__(self):
         return f"Unknown Metric {str(self)}"
 
 
-@dataclass
-class CommonSettings:
+class CommonSettings(BaseModel):
     # Log & Model parameters
     train_log_path: Path = Path("default_path.csv")
     log_ids: EventLogIDs = PROSIMOS_LOG_IDS
     test_log_path: Optional[Path] = None
-    model_path: Optional[Path] = None
+    process_model_path: Optional[Path] = None
     # Final evaluation parameters
     perform_final_evaluation: bool = False
     num_final_evaluations: int = 10
@@ -116,12 +116,12 @@ def from_dict(config: dict, config_dir: Optional[Path] = None) -> "CommonSetting
             test_log_path = None
 
         # Process model path
-        if "model_path" in config:
-            model_path = Path(config["model_path"])
-            if not model_path.is_absolute():
-                model_path = base_files_dir / model_path
+        if "process_model_path" in config:
+            process_model_path = Path(config["process_model_path"])
+            if not process_model_path.is_absolute():
+                process_model_path = base_files_dir / process_model_path
         else:
-            model_path = None
+            process_model_path = None
 
         # Flag to perform final evaluation (set to true if there is a test log)
         if test_log_path is not None:
@@ -165,7 +165,7 @@ def from_dict(config: dict, config_dir: Optional[Path] = None) -> "CommonSetting
             train_log_path=train_log_path,
             log_ids=log_ids,
             test_log_path=test_log_path,
-            model_path=model_path,
+            process_model_path=process_model_path,
             perform_final_evaluation=perform_final_evaluation,
             num_final_evaluations=num_final_evaluations,
             evaluation_metrics=metrics,
@@ -179,7 +179,7 @@ def to_dict(self) -> dict:
             "train_log_path": str(self.train_log_path),
             "test_log_path": str(self.test_log_path) if self.test_log_path is not None else None,
             "log_ids": self.log_ids.to_dict(),
-            "model_path": str(self.model_path) if self.model_path is not None else None,
+            "process_model_path": str(self.process_model_path) if self.process_model_path is not None else None,
             "num_final_evaluations": self.num_final_evaluations,
             "evaluation_metrics": [str(metric) for metric in self.evaluation_metrics],
             "use_observed_arrival_distribution": self.use_observed_arrival_distribution,