stitchlab-optimization 0.0.4__tar.gz → 0.1.0__tar.gz

{stitchlab_optimization-0.0.4/src/stitchlab_optimization.egg-info → stitchlab_optimization-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stitchlab-optimization
-Version: 0.0.4
+Version: 0.1.0
 Summary: StitchLab Optimization application
 Author-email: StitchLab Team <evazuliya97@gmail.com>
 License: MIT

{stitchlab_optimization-0.0.4 → stitchlab_optimization-0.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "stitchlab-optimization"
-version = "0.0.4"
+version = "0.1.0"
 description = "StitchLab Optimization application"
 readme = "README.md"
 license = {text = "MIT"}

{stitchlab_optimization-0.0.4 → stitchlab_optimization-0.1.0}/src/stitchlab_optimization/__init__.py

@@ -1,6 +1,6 @@
 """StitchLab Agent Core - A powerful agent core application for AI development."""
 
-__version__ = "0.0.4"
+__version__ = "0.1.0"
 __author__ = "StitchLab Team"
 __license__ = "MIT"
 

{stitchlab_optimization-0.0.4 → stitchlab_optimization-0.1.0}/src/stitchlab_optimization/builder/model.py

@@ -11,8 +11,8 @@ from typing import Any, Dict, Generic, Type, Optional, TypeVar, final
 
 from ..solver.engine import SolverEngine
 from ..solver.status import SolverStatus
-from ..logger.manager import ModelLog, LogManager
-from ..solver.config import SolverConfig
+from ..solver.params import SolverParams
+from ..logger.manager import ModelLog, LogManager, ResourceMonitor, NullResourceMonitor
 
 
 ParamsBaseModel = TypeVar("ParamsBaseModel", bound="ModelParams")
@@ -51,17 +51,19 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
     solution: Optional[SolutionBaseModel] = None
     solver_engine: SolverEngine
     solver_status: SolverStatus
+    solver_params: SolverParams
     model: Any = None
     model_output: Any = None
-    model_vars: Optional[Dict[str, Any]] = None  # Mandatory for OR-Tools CPSAT
+    model_vars: Optional[Dict[str, Any]] = None
     runtime_message: str = ""
     runtime_seconds: float = 0
     
     @final
-    def __init__(self, params: ParamsBaseModel, solver_engine: SolverEngine):
+    def __init__(self, params: ParamsBaseModel, solver_engine: SolverEngine, solver_params: SolverParams):
         self.params = params
         self.solver_engine = solver_engine
         self.solver_status = SolverStatus.UNSOLVED
+        self.solver_params = solver_params
 
     @final
     def _set_model(self, model: Any):
@@ -78,13 +80,15 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
         if self.model is None:
             raise ValueError("Model must be built before execution.")
         
-        if self.solver_engine == SolverEngine.ORTOOLS_CPSAT and self.model_vars is None:    
-            raise ValueError("Model variables (model_vars) must be set in the builder when using OR-Tools CPSAT.")
+        if self.model_vars is None:    
+            raise ValueError("Model variables (model_vars) must be set in the builder before execution.")
 
         self.solve()
-        self.solution = self.construct_solution()
+        
+        if SolverStatus.is_solution_found(self.solver_status):
+            return self.construct_solution()
 
-        return self.solution
+        return None
 
     @abstractmethod
     def build(self):
@@ -112,42 +116,38 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
 
             raise ValueError(f"ERROR while Solving Model : Vars is not saved while building model using solver engine {self.solver_engine}")
 
-        if self.solver_engine == SolverEngine.PYSCIPOPT:
-            self.solve_pyscipopt()
-
-        elif self.solver_engine == SolverEngine.GUROBI:
-            self.solve_gurobi()
-
-        elif self.solver_engine == SolverEngine.ORTOOLS_SCIP:
-            self.solve_ortools_scip()
+        SOLVER = {
+            SolverEngine.PYSCIPOPT: self.solve_pyscipopt,
+            SolverEngine.GUROBI: self.solve_gurobi,
+            SolverEngine.ORTOOLS_SCIP: self.solve_ortools_scip,
+            SolverEngine.ORTOOLS_ROUTING: self.solve_ortools_routing,
+            SolverEngine.ORTOOLS_CPSAT: self.solve_ortools_cpsat,
+        }
 
-        elif self.solver_engine == SolverEngine.ORTOOLS_ROUTING:
-            self.solve_ortools_routing()
-        
-        elif self.solver_engine == SolverEngine.ORTOOLS_CPSAT:
-            self.solve_ortools_cpsat()
-
-        else:
+        # with ResourceMonitor() as monitor:
+        try:
+            SOLVER[self.solver_engine]()
+        except KeyError:
             raise ValueError(f"Solver engine {self.solver_engine} not supported")
     
     def solve_pyscipopt(self):
-        SOLVER_CONFIG = SolverConfig()
+        PARAMS = self.solver_params
         start_sol = None
         
-        self.model.setParam("display/verblevel", SOLVER_CONFIG.MODEL_SOLVER_VERBOSE)
+        self.model.setParam("display/verblevel", PARAMS.MODEL_SOLVER_VERBOSE)
 
-        self.model.setIntParam("parallel/maxnthreads", SOLVER_CONFIG.LIMIT_MULTI_THREAD)
-        self.model.setIntParam("parallel/minnthreads", SOLVER_CONFIG.LIMIT_MULTI_THREAD)
+        self.model.setIntParam("parallel/maxnthreads", PARAMS.LIMIT_MULTI_THREAD)
+        self.model.setIntParam("parallel/minnthreads", PARAMS.LIMIT_MULTI_THREAD)
 
-        if SOLVER_CONFIG.APPLY_HEURISTICS:
+        if PARAMS.APPLY_HEURISTICS:
             # Phase 1: Heuristics only
             self.model.setHeuristics(SCIP_PARAMSETTING.AGGRESSIVE)
 
-            self.model.setParam("limits/time", SOLVER_CONFIG.LIMIT_TIME_MINUTES_HEURISTICS*60)
-            self.model.setParam("limits/gap", SOLVER_CONFIG.LIMIT_OPTIMALITY_GAP_HEURISTICS)
+            self.model.setParam("limits/time", PARAMS.LIMIT_TIME_MINUTES_HEURISTICS*60)
+            self.model.setParam("limits/gap", PARAMS.LIMIT_OPTIMALITY_GAP_HEURISTICS)
             self.model.setParam("limits/nodes", 500)   # limit nodes so B&B doesn't go far
             self.model.setParam("presolving/maxrounds", 0)  # skip heavy presolve if desired
-            self.model.setParam("limits/memory", SOLVER_CONFIG.LIMIT_MEMORY_MB)
+            self.model.setParam("limits/memory", PARAMS.LIMIT_MEMORY_MB)
 
             self.model.optimize()
 
@@ -162,9 +162,9 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
         self.model.resetParams()
         self.model.setHeuristics(SCIP_PARAMSETTING.DEFAULT)
 
-        self.model.setParam("limits/time", SOLVER_CONFIG.LIMIT_TIME_MINUTES_DETERMINISTIC*60)
-        self.model.setParam("limits/gap", SOLVER_CONFIG.LIMIT_OPTIMALITY_GAP_DETERMINISTIC)
-        self.model.setParam("limits/memory", SOLVER_CONFIG.LIMIT_MEMORY_MB)
+        self.model.setParam("limits/time", PARAMS.LIMIT_TIME_MINUTES_DETERMINISTIC*60)
+        self.model.setParam("limits/gap", PARAMS.LIMIT_OPTIMALITY_GAP_DETERMINISTIC)
+        self.model.setParam("limits/memory", PARAMS.LIMIT_MEMORY_MB)
 
         try:
             if start_sol is not None:
@@ -182,23 +182,20 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
     
         self.solver_status = SolverStatus.from_pyscipopt_status(self.model.getStatus())
         print("STATUS", self.model.getStatus(), self.solver_status, "\n\n")
-
-        if SolverStatus.is_solution_found(self.solver_status):
-            self.construct_solution()
-
+            
     def solve_gurobi(self):
-        SOLVER_CONFIG = SolverConfig()
+        PARAMS = self.solver_params
 
-        self.model.setParam('OutputFlag', SOLVER_CONFIG.MODEL_SOLVER_VERBOSE)
+        self.model.setParam('OutputFlag', PARAMS.MODEL_SOLVER_VERBOSE)
 
         start_sol = None
-        if SOLVER_CONFIG.APPLY_HEURISTICS:
+        if PARAMS.APPLY_HEURISTICS:
             # Phase 1: Heuristics only
-            self.model.setParam('TimeLimit', SOLVER_CONFIG.LIMIT_TIME_MINUTES_HEURISTICS * 60)
-            self.model.setParam('MIPGap', SOLVER_CONFIG.LIMIT_OPTIMALITY_GAP_HEURISTICS)
+            self.model.setParam('TimeLimit', PARAMS.LIMIT_TIME_MINUTES_HEURISTICS * 60)
+            self.model.setParam('MIPGap', PARAMS.LIMIT_OPTIMALITY_GAP_HEURISTICS)
             self.model.setParam('NodeLimit', 500)  # limit nodes so B&B doesn't go far
             self.model.setParam('Presolve', 0)  # skip heavy presolve if desired
-            self.model.setParam('Threads', SOLVER_CONFIG.LIMIT_MULTI_THREAD)
+            self.model.setParam('Threads', PARAMS.LIMIT_MULTI_THREAD)
 
             # Set heuristic focus
             self.model.setParam('Heuristics', 0.8)  # Aggressive heuristics
@@ -216,12 +213,12 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
 
         # Phase 2: Exact MILP solving
         # Reset parameters for exact solving
-        self.model.setParam('TimeLimit', SOLVER_CONFIG.LIMIT_TIME_MINUTES_DETERMINISTIC * 60)
-        self.model.setParam('MIPGap', SOLVER_CONFIG.LIMIT_OPTIMALITY_GAP_DETERMINISTIC)
+        self.model.setParam('TimeLimit', PARAMS.LIMIT_TIME_MINUTES_DETERMINISTIC * 60)
+        self.model.setParam('MIPGap', PARAMS.LIMIT_OPTIMALITY_GAP_DETERMINISTIC)
         self.model.setParam('NodeLimit', 1000000)
         self.model.setParam('Presolve', -1)  # Default presolve
         self.model.setParam('Heuristics', 0.05)  # Default heuristics
-        self.model.setParam('Threads', SOLVER_CONFIG.LIMIT_MULTI_THREAD)
+        self.model.setParam('Threads', PARAMS.LIMIT_MULTI_THREAD)
 
         try:
             if start_sol is not None:
@@ -237,11 +234,8 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
         self.solver_status = SolverStatus.from_gurobi_status(self.model.status)
         print("STATUS", self.model.status, self.solver_status, "\n\n")
 
-        if SolverStatus.is_solution_found(self.solver_status):
-            self.construct_solution()
-
     def solve_ortools_routing(self):
-        SOLVER_CONFIG = SolverConfig()
+        PARAMS = self.solver_params
 
         search_parameters = pywrapcp.DefaultRoutingSearchParameters()
         search_parameters.first_solution_strategy = (
@@ -253,24 +247,21 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
         )
 
         search_parameters.solution_limit = 100
-        search_parameters.time_limit.seconds = int(SOLVER_CONFIG.LIMIT_TIME_MINUTES_DETERMINISTIC * 60)
+        search_parameters.time_limit.seconds = int(PARAMS.LIMIT_TIME_MINUTES_DETERMINISTIC * 60)
         
         self.solution = self.model.SolveWithParameters(search_parameters)
     
         self.solver_status = SolverStatus.from_ortools_routing_status(self.model.status())
         print("STATUS", self.solver_status)
 
-        if SolverStatus.is_solution_found(self.solver_status):
-            self.construct_solution()
-
     def solve_ortools_cpsat(self):
-        SOLVER_CONFIG = SolverConfig()
+        PARAMS = self.solver_params
 
         solver = cp_model.CpSolver()
-        solver.parameters.max_time_in_seconds = SOLVER_CONFIG.LIMIT_TIME_MINUTES_DETERMINISTIC * 60
-        solver.parameters.num_search_workers = SOLVER_CONFIG.LIMIT_MULTI_THREAD
+        solver.parameters.max_time_in_seconds = PARAMS.LIMIT_TIME_MINUTES_DETERMINISTIC * 60
+        solver.parameters.num_search_workers = PARAMS.LIMIT_MULTI_THREAD
 
-        solver.parameters.log_search_progress = SOLVER_CONFIG.MODEL_SOLVER_VERBOSE
+        solver.parameters.log_search_progress = PARAMS.MODEL_SOLVER_VERBOSE
 
         result_status = solver.Solve(self.model)
         self.model_output = solver
@@ -278,20 +269,17 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
         self.solver_status = SolverStatus.from_ortools_cpsat_status(result_status)
         print("STATUS", result_status, self.solver_status)
 
-        if SolverStatus.is_solution_found(self.solver_status):
-            self.construct_solution()
-
     def solve_ortools_scip(self):
-        SOLVER_CONFIG = SolverConfig()
+        PARAMS = self.solver_params
 
-        self.model.SetTimeLimit(int(SOLVER_CONFIG.LIMIT_TIME_MINUTES_DETERMINISTIC * 60 * 1000))
-        self.model.SetNumThreads(int(SOLVER_CONFIG.LIMIT_MULTI_THREAD))
+        self.model.SetTimeLimit(int(PARAMS.LIMIT_TIME_MINUTES_DETERMINISTIC * 60 * 1000))
+        self.model.SetNumThreads(int(PARAMS.LIMIT_MULTI_THREAD))
 
         params_str = (
-            f"limits/gap={SOLVER_CONFIG.LIMIT_OPTIMALITY_GAP_DETERMINISTIC}\n"
-            f"limits/memory={SOLVER_CONFIG.LIMIT_MEMORY_MB}\n"
-            f"parallel/maxnthreads={int(SOLVER_CONFIG.LIMIT_MULTI_THREAD)}\n"
-            f"lp/threads={int(SOLVER_CONFIG.LIMIT_MULTI_THREAD)}\n"
+            f"limits/gap={PARAMS.LIMIT_OPTIMALITY_GAP_DETERMINISTIC}\n"
+            f"limits/memory={PARAMS.LIMIT_MEMORY_MB}\n"
+            f"parallel/maxnthreads={int(PARAMS.LIMIT_MULTI_THREAD)}\n"
+            f"lp/threads={int(PARAMS.LIMIT_MULTI_THREAD)}\n"
             f"display/verblevel=5\n"
         )
 
@@ -301,9 +289,6 @@ class ModelBuilder(Generic[ParamsBaseModel, SolutionBaseModel], ABC):
         self.solver_status = SolverStatus.from_ortools_scip_status(status)
         print("STATUS", status, self.solver_status, "\n\n")
 
-        if SolverStatus.is_solution_found(self.solver_status):
-            self.construct_solution()
-
 
 class OptimizationModel(Generic[ParamsBaseModel, SolutionBaseModel], ABC, metaclass=ModelMeta):
     id: str
@@ -311,15 +296,19 @@ class OptimizationModel(Generic[ParamsBaseModel, SolutionBaseModel], ABC, metacl
     builders_registry: Dict[SolverEngine, Type[ModelBuilder[ParamsBaseModel, SolutionBaseModel]]]
     builder: ModelBuilder[ParamsBaseModel, SolutionBaseModel]
 
-    def __init__(self, params: ParamsBaseModel, solver_engine: Optional[SolverEngine] = None):
+    def __init__(self, params: ParamsBaseModel, solver_engine: Optional[SolverEngine] = None, solver_params: Optional[SolverParams] = None):
         self.id = str(uuid.uuid4())
 
         if solver_engine is None or solver_engine not in self.builders_registry.keys():
             solver_engine = next(iter(self.builders_registry.keys()))
 
+        if solver_params is None:
+            solver_params = SolverParams()
+
         self.builder = self.builders_registry[solver_engine](
             params=params,
-            solver_engine=solver_engine
+            solver_engine=solver_engine,
+            solver_params=solver_params
         )
 
     @final
@@ -327,9 +316,18 @@ class OptimizationModel(Generic[ParamsBaseModel, SolutionBaseModel], ABC, metacl
         start_time = time.time()
         solution = None
 
+        monitor = (
+            ResourceMonitor(
+                interval_seconds=logger.monitor_resource_interval_seconds
+            )
+            if logger and logger.enable_monitor_resource
+            else NullResourceMonitor()
+        )
+
         try :
-            solution = self.builder.execute()
-            self.builder.runtime_message = "success"
+            with monitor:
+                solution = self.builder.execute()
+                self.builder.runtime_message = "success"
 
         except Exception as e:
             print(f"\033[91m\n>>> ERROR while Solving Model : {e}\n\033[0m")
@@ -339,8 +337,11 @@ class OptimizationModel(Generic[ParamsBaseModel, SolutionBaseModel], ABC, metacl
             end_time = time.time()
             self.builder.runtime_seconds = end_time - start_time
 
-            if logger is not None and logger.is_monitor_optimality:
-                logger.put_model_log(model_log=self._model_log)
+            if logger is not None and logger.enable_monitor_optimality:
+                log = self._model_log
+                log.resource_stats = monitor.stats
+
+                logger.put_model_log(model_log=log)
         
         return solution
 
@@ -354,6 +355,7 @@ class OptimizationModel(Generic[ParamsBaseModel, SolutionBaseModel], ABC, metacl
     def _model_log(self) -> ModelLog:
         builder = self.builder
         solver_engine = self.builder.solver_engine
+        solver_params = self.builder.solver_params
 
         if solver_engine == SolverEngine.GUROBI:
             # Gurobi Python API
@@ -427,6 +429,7 @@ class OptimizationModel(Generic[ParamsBaseModel, SolutionBaseModel], ABC, metacl
 
         return ModelLog(
             solver_engine=solver_engine,
+            solver_params=solver_params,
             model_id=self.id,
             model_name=self.name,
             status=builder.solver_status,

{stitchlab_optimization-0.0.4 → stitchlab_optimization-0.1.0}/src/stitchlab_optimization/builder/workflow.py

@@ -10,7 +10,7 @@ import traceback
 from ..logger.manager import LogManager, WorkflowLog
 from ..builder.model import OptimizationModel
 from ..solver.engine import SolverEngine
-from ..solver.config import SolverConfig
+from ..solver.params import SolverParams
 
 
 InputBaseModel = TypeVar("InputBaseModel", bound=BaseModel)
@@ -105,7 +105,7 @@ class OptimizationWorkflow(Generic[InputBaseModel, OutputBaseModel], ABC, metacl
             self.runtime_seconds = time.time() - start_time
             self.end_timestamp = datetime.now(timezone.utc).isoformat()
 
-            if self._logger is not None and self._logger.is_monitor_runtime:
+            if self._logger is not None and self._logger.enable_monitor_runtime:
                 self._logger.put_workflow_log(workflow_log=self._workflow_log)
 
         if self.runtime_message == "success" and result is not None:
@@ -118,14 +118,15 @@ class OptimizationWorkflow(Generic[InputBaseModel, OutputBaseModel], ABC, metacl
         pass
 
     @final
-    def execute_model(self, model_name: str, params: Any, solver_engine: Optional[SolverEngine] = None) -> Any:
+    def execute_model(self, model_name: str, params: Any, solver_engine: Optional[SolverEngine] = None, solver_params: Optional[SolverParams] = None) -> Any:
         if model_name not in self.models_registry:
             raise ValueError(f"Model '{model_name}' not found in models_registry.")
         
         model_cls = self.models_registry[model_name]
         model_instance = model_cls(
             params=params,
-            solver_engine=solver_engine
+            solver_engine=solver_engine,
+            solver_params=solver_params
         )
 
         output = model_instance.execute(logger=self._logger)
@@ -140,7 +141,6 @@ class OptimizationWorkflow(Generic[InputBaseModel, OutputBaseModel], ABC, metacl
             workflow_name=self.name,
             model_ids_execution=self.model_ids_execution,
             payload=self.payload.model_dump(),
-            solver_parameter=SolverConfig().SOLVER_PARAMETER,
             message=self.runtime_message,
             start_timestamp=self.start_timestamp,
             end_timestamp=self.end_timestamp,

stitchlab_optimization-0.1.0/src/stitchlab_optimization/logger/json_logger.py

@@ -0,0 +1,46 @@
+from .manager import LogManager, ModelLog, WorkflowLog
+from pathlib import Path
+
+
+class JSONLogManager(LogManager):
+    directory_path: str
+
+    def __init__(
+            self, directory_path: str,
+            enable_monitor_optimality: bool = True,
+            enable_monitor_runtime: bool = True,
+            enable_monitor_resource: bool = False
+        ):
+        self.directory_path = directory_path
+        self.enable_monitor_optimality = enable_monitor_optimality
+        self.enable_monitor_resource = enable_monitor_resource
+        self.enable_monitor_runtime = enable_monitor_runtime
+    
+        self.init_directory()
+
+    def init_directory(self):
+        if self.enable_monitor_optimality:
+            Path(
+                f"{self.directory_path}/{self._dir_model_execution_log}"
+            ).mkdir(parents=True, exist_ok=True)
+
+        if self.enable_monitor_runtime:
+            Path(
+                f"{self.directory_path}/{self._dir_workflow_execution_log}"
+            ).mkdir(parents=True, exist_ok=True)
+
+    def put_model_log(self, model_log: ModelLog):
+        file_path = Path(f"{self.directory_path}/{self._dir_model_execution_log}/{model_log.model_id}.json")
+        
+        file_path.write_text(
+            model_log.model_dump_json(indent=2),
+            encoding="utf-8"
+        )
+
+    def put_workflow_log(self, workflow_log: WorkflowLog):
+        file_path = Path(f"{self.directory_path}/{self._dir_workflow_execution_log}/{workflow_log.workflow_id}.json")
+        
+        file_path.write_text(
+            workflow_log.model_dump_json(indent=2),
+            encoding="utf-8"
+        )

stitchlab_optimization-0.1.0/src/stitchlab_optimization/logger/manager.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+from abc import abstractmethod, ABC
+from pydantic import BaseModel
+from typing import Optional
+import json
+import os
+import time
+import psutil
+import threading
+from statistics import mean
+
+from ..solver.engine import SolverEngine
+from ..solver.status import SolverStatus
+from ..solver.params import SolverParams
+
+
+class ResourceStats(BaseModel):
+    peak_cpu_percent: float
+    avg_cpu_percent: float
+    peak_memory_mb: float
+    avg_memory_mb: float
+    max_num_threads: int
+
+
+class ResourceSnapshot(BaseModel):
+    timestamp: float
+    cpu_percent: float
+    memory_mb: float
+    num_threads: int
+
+
+class ModelLog(BaseModel):
+    solver_engine: SolverEngine
+    solver_params: SolverParams
+    model_id: str
+    model_name: str
+    status: SolverStatus
+    problem_size_vars: Optional[int]
+    problem_size_cons: Optional[int]
+    optimality_gap: Optional[float]
+    objective_value: Optional[float]
+    message: Optional[str]
+    runtime_sec: float
+    resource_stats: Optional[ResourceStats] = None
+    created_timestamp: str
+
+    def to_sql_log(self) -> dict:
+        return {
+            "id": None,
+            "solver_engine": self.solver_engine.value,
+            "solver_params": self.solver_params.model_dump_json(),
+            "model_id": self.model_id,
+            "model_name": self.model_name,
+            "problem_size_vars": self.problem_size_vars,
+            "problem_size_cons": self.problem_size_cons,
+            "optimality_gap": self.optimality_gap,
+            "objective_value": self.objective_value,
+            "status": self.status.value,
+            "message": self.message,
+            "runtime_sec": self.runtime_sec,
+            "resource_stats": self.resource_stats.model_dump_json(),
+            "created_timestamp": self.created_timestamp
+        }
+
+
+class WorkflowLog(BaseModel):
+    workflow_id: str
+    workflow_name: str
+    model_ids_execution: dict
+    payload: dict
+    message: Optional[str]
+    start_timestamp: str
+    end_timestamp: str
+    runtime_sec: float
+    created_timestamp: str
+
+    def to_sql_log(self) -> dict:
+        model_ids = json.dumps(self.model_ids_execution)
+        payload = json.dumps(self.payload)
+
+        return {
+            "id": None,
+            "workflow_id": self.workflow_id,
+            "workflow_name": self.workflow_name,
+            "model_ids": model_ids,
+            "payload": payload,
+            "message": self.message,
+            "start_timestamp": self.start_timestamp,
+            "end_timestamp": self.end_timestamp,
+            "runtime_sec": self.runtime_sec,
+            "created_timestamp": self.created_timestamp
+        }
+
+
+class LogManager(ABC):
+    enable_monitor_optimality: bool = True
+    enable_monitor_runtime: bool = True
+    enable_monitor_resource: bool = False
+    monitor_resource_interval_seconds: int = 5
+
+    _dir_model_execution_log: str = "log_execution_model"
+    _dir_workflow_execution_log: str = "log_execution_workflow"
+
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def put_model_log(self, model_log: ModelLog):
+        pass
+
+    @abstractmethod
+    def put_workflow_log(self, workflow_log: WorkflowLog):
+        pass
+
+
+class BaseResourceMonitor(ABC):
+    stats: Optional[ResourceStats] = None
+
+    @abstractmethod
+    def __enter__(self):
+        pass
+
+    @abstractmethod
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+class NullResourceMonitor(BaseResourceMonitor):
+    stats = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+class ResourceMonitor(BaseResourceMonitor):
+
+    def __init__(self, interval_seconds: int = 1):
+        self._running = False
+        self.interval_seconds = interval_seconds
+        self.stats: Optional[ResourceStats] = None
+        self._thread: Optional[threading.Thread] = None
+        self._snapshots: list[ResourceSnapshot] = []
+        self._stop_event = threading.Event()
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop()
+
+    def start(self):
+        if self._running:
+            return
+
+        self._running = True
+        self._stop_event.clear()
+
+        proc = psutil.Process(os.getpid())
+        proc.cpu_percent(interval=None)
+        self._take_sample(proc)
+
+        self._thread = threading.Thread(
+            target=self._sampling_loop,
+            daemon=True
+        )
+
+        self._thread.start()
+
+    def stop(self):
+        if not self._running:
+            return
+
+        self._running = False
+        self._stop_event.set()
+
+        if self._thread is not None:
+            self._thread.join()
+
+        self.stats = self._aggregate()
+
+    def _take_sample(self, proc: psutil.Process):
+        try:
+            self._snapshots.append(
+                ResourceSnapshot(
+                    timestamp=time.time(),
+                    cpu_percent=proc.cpu_percent(interval=None),
+                    memory_mb=proc.memory_info().rss / 1024 / 1024,
+                    num_threads=proc.num_threads()
+                )
+            )
+        except (
+            psutil.NoSuchProcess,
+            psutil.AccessDenied,
+        ):
+            pass
+
+    def _sampling_loop(self):
+        proc = psutil.Process(os.getpid())
+
+        while not self._stop_event.wait(self.interval_seconds):
+            self._take_sample(proc)
+
+    def _aggregate(self) -> Optional[ResourceStats]:
+        if not self._snapshots:
+            return None
+
+        cpu_values = [x.cpu_percent for x in self._snapshots]
+        mem_values = [x.memory_mb for x in self._snapshots]
+        thread_values = [x.num_threads for x in self._snapshots]
+
+        return ResourceStats(
+            peak_cpu_percent=max(cpu_values),
+            avg_cpu_percent=mean(cpu_values),
+            peak_memory_mb=max(mem_values),
+            avg_memory_mb=mean(mem_values),
+            max_num_threads=max(thread_values)
+        )

{stitchlab_optimization-0.0.4 → stitchlab_optimization-0.1.0}/src/stitchlab_optimization/logger/sqlite_logger.py

@@ -6,21 +6,27 @@ import pandas as pd
 class SQLiteLogManager(LogManager):
     db_path: str
 
-    def __init__(self, db_path: str, is_monitor_optimality: bool = True, is_monitor_runtime: bool = True, is_monitor_resource: bool = False):
+    def __init__(
+            self, db_path: str,
+            enable_monitor_optimality: bool = True,
+            enable_monitor_runtime: bool = True,
+            enable_monitor_resource: bool = False
+        ):
         self.db_path = db_path
-        self.is_monitor_optimality = is_monitor_optimality
-        self.is_monitor_resource = is_monitor_resource
-        self.is_monitor_runtime = is_monitor_runtime
+        self.enable_monitor_optimality = enable_monitor_optimality
+        self.enable_monitor_resource = enable_monitor_resource
+        self.enable_monitor_runtime = enable_monitor_runtime
     
         self.init_db()
 
     def init_db(self):
         with sqlite3.connect(self.db_path) as conn:
-            if self.is_monitor_optimality:
+            if self.enable_monitor_optimality:
                 conn.execute(f"""
                     CREATE TABLE IF NOT EXISTS {self._dir_model_execution_log} (
                         id                 INTEGER PRIMARY KEY AUTOINCREMENT,
                         solver_engine      TEXT,            -- gurobi / ortools / routing
+                        solver_params      TEXT,            -- store JSON solver parameter as string
                         model_id           TEXT,
                         model_name         TEXT,
                         problem_size_vars  INTEGER,         -- number of decision variables
@@ -30,11 +36,12 @@ class SQLiteLogManager(LogManager):
                         status             TEXT,            -- START / DONE / ERROR
                         message            TEXT,            -- optional note
                         runtime_sec        REAL,            -- optional
+                        resource_stats     TEXT,            -- store JSON resource stats as string
                         created_timestamp  TEXT
                     );
                 """)
 
-            if self.is_monitor_runtime:
+            if self.enable_monitor_runtime:
                 conn.execute(f"""
                     CREATE TABLE IF NOT EXISTS {self._dir_workflow_execution_log} (
                         id                 INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -42,7 +49,6 @@ class SQLiteLogManager(LogManager):
                         workflow_name      TEXT,
                         model_ids          TEXT,      -- store list of str as JSON string
                         payload            TEXT,      -- store JSON payload as string
-                        solver_parameter   TEXT,      -- store JSON solver parameter as string
                         message            TEXT,      -- store error message
                         start_timestamp    TEXT,      -- store start timestamp
                         end_timestamp      TEXT,      -- store end timestamp
@@ -51,21 +57,6 @@ class SQLiteLogManager(LogManager):
                     );
                 """)
 
-            if self.is_monitor_resource:
-                conn.execute(f"""
-                    CREATE TABLE IF NOT EXISTS {self._dir_resource_occupation_log} (
-                        id                 INTEGER PRIMARY KEY AUTOINCREMENT,
-                        machine_id         TEXT,
-                        model_id           TEXT,
-                        cpu_percent        REAL,
-                        num_cores          INTEGER,
-                        limit_thread       INTEGER,
-                        memory_mb          REAL,
-                        limit_memory_mb    REAL,
-                        timestamp          TEXT
-                    );
-                """)
-
             conn.commit()
 
     def put_model_log(self, model_log: ModelLog):
@@ -76,7 +67,4 @@ class SQLiteLogManager(LogManager):
     def put_workflow_log(self, workflow_log: WorkflowLog):
         with sqlite3.connect(self.db_path) as conn:
             data = pd.DataFrame([workflow_log.to_sql_log()])
-            data.to_sql(self._dir_workflow_execution_log, conn, if_exists="append", index=False)
-
-    def put_resource_log(self, resource_log):
-        return super().put_resource_log(resource_log)
+            data.to_sql(self._dir_workflow_execution_log, conn, if_exists="append", index=False)

stitchlab_optimization-0.1.0/src/stitchlab_optimization/solver/params.py

@@ -0,0 +1,15 @@
+import os
+from pydantic import BaseModel
+
+
+class SolverParams(BaseModel):
+    TRUE_THRESHOLD: float = 0.5
+    APPLY_HEURISTICS: bool = True
+    MODEL_SOLVER_VERBOSE: bool = False
+
+    LIMIT_TIME_MINUTES_HEURISTICS: float = 3
+    LIMIT_TIME_MINUTES_DETERMINISTIC: float = 3
+    LIMIT_OPTIMALITY_GAP_DETERMINISTIC: float = 0.35
+    LIMIT_OPTIMALITY_GAP_HEURISTICS: float = 0.75
+    LIMIT_MEMORY_MB: int = 1024*8
+    LIMIT_MULTI_THREAD: int = 6

{stitchlab_optimization-0.0.4 → stitchlab_optimization-0.1.0/src/stitchlab_optimization.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stitchlab-optimization
-Version: 0.0.4
+Version: 0.1.0
 Summary: StitchLab Optimization application
 Author-email: StitchLab Team <evazuliya97@gmail.com>
 License: MIT

{stitchlab_optimization-0.0.4 → stitchlab_optimization-0.1.0}/src/stitchlab_optimization.egg-info/SOURCES.txt

@@ -11,12 +11,12 @@ src/stitchlab_optimization/builder/__init__.py
 src/stitchlab_optimization/builder/model.py
 src/stitchlab_optimization/builder/workflow.py
 src/stitchlab_optimization/logger/__init__.py
-src/stitchlab_optimization/logger/file_logger.py
+src/stitchlab_optimization/logger/json_logger.py
 src/stitchlab_optimization/logger/manager.py
 src/stitchlab_optimization/logger/sqlite_logger.py
 src/stitchlab_optimization/solver/__init__.py
-src/stitchlab_optimization/solver/config.py
 src/stitchlab_optimization/solver/engine.py
+src/stitchlab_optimization/solver/params.py
 src/stitchlab_optimization/solver/status.py
 src/stitchlab_optimization/tools/utils.py
 src/stitchlab_optimization/tools/query/snowflake.py

stitchlab_optimization-0.0.4/src/stitchlab_optimization/logger/file_logger.py

@@ -1,120 +0,0 @@
-class ModelLog(BaseModel):
-    solver_engine: SolverEngine
-    model_id: str
-    exec_id: str
-    model_name: str
-    status: SolverStatus
-    problem_size_vars: Optional[int]
-    problem_size_cons: Optional[int]
-    optimality_gap: Optional[float]
-    objective_value: Optional[float]
-    message: Optional[str]
-    runtime_sec: float
-    created_at: str
-
-    @classmethod
-    def from_model(cls, model: OptimizationModel, solver_engine: SolverEngine):
-
-        builder = model.builders[solver_engine]
-
-        if solver_engine == SolverEngine.GUROBI:
-            # Gurobi Python API
-            problem_size_vars = builder.model.NumVars
-            problem_size_cons = builder.model.NumConstrs
-
-            if builder.model.NumObj <= 1:
-                optimality_gap = builder.model.MIPGap
-                objective_value = builder.model.ObjVal
-
-            else:
-                objective_value = builder.model.getAttr("ObjNVal")
-                optimality_gap   = builder.model.getAttr("ObjNRelTol")
-
-        elif solver_engine == SolverEngine.ORTOOLS_SCIP:
-            # OR-Tools CP-SAT solver (pywraplp.Solver)
-            problem_size_vars = builder.model.NumVariables()
-            problem_size_cons = builder.model.NumConstraints()
-
-            try:
-                optimality_gap = builder.model.MipGap()
-            except AttributeError:
-                optimality_gap = None
-                
-            objective_value = builder.model.Objective().Value()
-            
-        elif solver_engine == SolverEngine.ORTOOLS_ROUTING:
-            # OR-Tools RoutingModel
-            count_nodes = builder.model.Size()
-            count_vehicles = builder.model.vehicles()
-            problem_size_vars = count_nodes * count_nodes * count_vehicles
-
-            # RoutingModel does not expose number of constraints directly
-            problem_size_cons = None
-            objective_value = builder.solution.ObjectiveValue()
-            optimality_gap = None
-        
-        elif solver_engine == SolverEngine.ORTOOLS_CPSAT:
-            problem_size_vars = len(builder.model.Proto().variables)
-            problem_size_cons = len(builder.model.Proto().constraints)
-
-            # Objective value (only available if model solved)
-            try:
-                objective_value = builder.solution.ObjectiveValue()
-            except Exception:
-                objective_value = None
-
-            optimality_gap = None
-
-        elif solver_engine == SolverEngine.PYSCIPOPT:
-            problem_size_vars = builder.model.getNVars()
-            problem_size_cons = builder.model.getNConss()
-            optimality_gap = builder.model.getGap()
-
-            try:
-                objective_value = builder.model.getObjVal()
-            except Exception:
-                objective_value = None
-
-        elif solver_engine == SolverEngine.SKLEARN:
-            # scikit-learn is not an optimization solver, so these are not applicable
-            problem_size_vars = None
-            problem_size_cons = None
-            optimality_gap = None
-            objective_value = None
-
-        else:
-            raise ValueError(f"Solver engine {solver_engine} not supported")
-
-        return ModelLog(
-            solver_engine=solver_engine,
-            model_id=model.id,
-            exec_id=builder.id,
-            model_name=model.name,
-            status=builder.solver_status,
-            problem_size_vars=problem_size_vars,
-            problem_size_cons=problem_size_cons,
-            optimality_gap=optimality_gap,
-            objective_value=objective_value,
-            message=builder.runtime_message,
-            runtime_sec=builder.runtime_seconds,
-            created_at=datetime.now(timezone.utc).isoformat()
-        )
-
-    def write_to_db(self):
-        log = {
-            "id": None,
-            "solver_engine": self.solver_engine.value,
-            "model_id": self.model_id,
-            "exec_id": self.exec_id,
-            "model_name": self.model_name,
-            "problem_size_vars": self.problem_size_vars,
-            "problem_size_cons": self.problem_size_cons,
-            "optimality_gap": self.optimality_gap,
-            "objective_value": self.objective_value,
-            "status": self.status.value,
-            "message": self.message,
-            "runtime_sec": self.runtime_sec,
-            "created_at": self.created_at
-        }
-
-        insert_to_sqlite(table_name="execution_log", data=log)

stitchlab_optimization-0.0.4/src/stitchlab_optimization/logger/manager.py

@@ -1,94 +0,0 @@
-from abc import abstractmethod, ABC
-from pydantic import BaseModel
-from typing import Optional
-import json
-
-from ..solver.engine import SolverEngine
-from ..solver.status import SolverStatus
-
-
-class ModelLog(BaseModel):
-    solver_engine: SolverEngine
-    model_id: str
-    model_name: str
-    status: SolverStatus
-    problem_size_vars: Optional[int]
-    problem_size_cons: Optional[int]
-    optimality_gap: Optional[float]
-    objective_value: Optional[float]
-    message: Optional[str]
-    runtime_sec: float
-    created_timestamp: str
-
-    def to_sql_log(self) -> dict:
-        return {
-            "id": None,
-            "solver_engine": self.solver_engine.value,
-            "model_id": self.model_id,
-            "model_name": self.model_name,
-            "problem_size_vars": self.problem_size_vars,
-            "problem_size_cons": self.problem_size_cons,
-            "optimality_gap": self.optimality_gap,
-            "objective_value": self.objective_value,
-            "status": self.status.value,
-            "message": self.message,
-            "runtime_sec": self.runtime_sec,
-            "created_timestamp": self.created_timestamp
-        }
-
-
-class WorkflowLog(BaseModel):
-    workflow_id: str
-    workflow_name: str
-    model_ids_execution: dict
-    payload: dict
-    solver_parameter: dict
-    message: Optional[str]
-    start_timestamp: str
-    end_timestamp: str
-    runtime_sec: float
-    created_timestamp: str
-
-    def to_sql_log(self) -> dict:
-        model_ids = json.dumps(self.model_ids_execution)
-        payload = json.dumps(self.payload)
-        solver_parameter = json.dumps(self.solver_parameter)
-
-        return {
-            "id": None,
-            "workflow_id": self.workflow_id,
-            "workflow_name": self.workflow_name,
-            "model_ids": model_ids,
-            "payload": payload,
-            "solver_parameter": solver_parameter,
-            "message": self.message,
-            "start_timestamp": self.start_timestamp,
-            "end_timestamp": self.end_timestamp,
-            "runtime_sec": self.runtime_sec,
-            "created_timestamp": self.created_timestamp
-        }
-
-
-class LogManager(ABC):
-    is_monitor_optimality: bool = True
-    is_monitor_runtime: bool = True
-    is_monitor_resource: bool = False
-
-    _dir_model_execution_log: str = "log_execution_model"
-    _dir_workflow_execution_log: str = "log_execution_workflow"
-    _dir_resource_occupation_log: str = "log_resource_occupation"
-
-    def __init__(self):
-        pass
-
-    @abstractmethod
-    def put_model_log(self, model_log: ModelLog):
-        pass
-
-    @abstractmethod
-    def put_workflow_log(self, workflow_log: WorkflowLog):
-        pass
-
-    @abstractmethod
-    def put_resource_log(self, resource_log: dict):
-        pass

stitchlab_optimization-0.0.4/src/stitchlab_optimization/solver/config.py

@@ -1,63 +0,0 @@
-import os
-from dotenv import load_dotenv
-
-
-class SolverConfig:
-    _instance = None
-
-    def __new__(cls):
-        if cls._instance is None:
-            if cls._instance is None:
-                load_dotenv()  # load .env
-                cls._instance = super().__new__(cls)
-                cls._instance._load_env()
-
-        return cls._instance
-
-    def _load_env(self):
-        self.TRUE_THRESHOLD = float(os.getenv("TRUE_THRESHOLD", 0.5))
-        self.APPLY_HEURISTICS = os.getenv("APPLY_HEURISTICS", "True").lower() in ("true", "1", "yes")
-        self.MODEL_SOLVER_VERBOSE = os.getenv("MODEL_SOLVER_VERBOSE", "False").lower() in ("true", "1", "yes")
-
-        self.LIMIT_TIME_MINUTES_HEURISTICS = float(os.getenv("LIMIT_TIME_MINUTES_HEURISTICS", 3))
-        self.LIMIT_TIME_MINUTES_DETERMINISTIC = float(os.getenv("LIMIT_TIME_MINUTES_DETERMINISTIC", 3))
-        self.LIMIT_OPTIMALITY_GAP_DETERMINISTIC = float(os.getenv("LIMIT_OPTIMALITY_GAP_DETERMINISTIC", 0.35))
-        self.LIMIT_OPTIMALITY_GAP_HEURISTICS = float(os.getenv("LIMIT_OPTIMALITY_GAP_HEURISTICS", 0.75))
-        self.LIMIT_MEMORY_MB = int(os.getenv("LIMIT_MEMORY_MB", 1024*8))
-        self.LIMIT_MULTI_THREAD = int(os.getenv("LIMIT_MULTI_THREAD", 6))
-
-        self.SOLVER_PARAMETER = {
-            "true_threshold": self.TRUE_THRESHOLD,
-            "limit_minutes_heuristics": self.LIMIT_TIME_MINUTES_HEURISTICS,
-            "limit_minutes_deterministic": self.LIMIT_TIME_MINUTES_DETERMINISTIC,
-            "limit_optimality_gap_deterministic": self.LIMIT_OPTIMALITY_GAP_DETERMINISTIC,
-            "limit_optimality_gap_heuristics": self.LIMIT_OPTIMALITY_GAP_HEURISTICS,
-            "limit_memory_mb": self.LIMIT_MEMORY_MB,
-            "limit_multi_thread": self.LIMIT_MULTI_THREAD,
-            "apply_heuristics": self.APPLY_HEURISTICS,
-        }
-
-        self.MONITOR_EXECUTION = os.getenv("MONITOR_EXECUTION", "False").lower() in ("true", "1", "yes")
-        self.MONITOR_RESOURCE = os.getenv("MONITOR_RESOURCE", "False").lower() in ("true", "1", "yes")
-        self.MONITOR_RESOURCE_INTERVAL_SECONDS = int(os.getenv("MONITOR_RESOURCE_INTERVAL_SECONDS", 60))
-
-        self.SQLITE_DB_PATH = os.getenv("SQLITE_DB_PATH", "log.sqlite")
-        self.EXPORT_DIRECTORY = os.getenv("EXPORT_DIRECTORY", "export")
-
-        self._load_gurobi_credentials()
-
-    
-    def _load_gurobi_credentials(self):
-        self.GUROBI_ENV = None
-
-        GUROBI_KEYS = {
-            "WLSAccessID": os.getenv("WLSACCESSID", None),
-            "WLSSecret": os.getenv("WLSSECRET", None),
-            "LicenseID": int(os.getenv("LICENSEID", 0)),
-        }
-
-        if GUROBI_KEYS["WLSAccessID"] is not None and GUROBI_KEYS["WLSSecret"] is not None:
-            import gurobipy as gp
-
-            self.GUROBI_ENV = gp.Env(params=GUROBI_KEYS)
-