cornflow-client 1.3.4rc2__tar.gz → 1.3.5__tar.gz

{cornflow_client-1.3.4rc2/cornflow_client.egg-info → cornflow_client-1.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cornflow-client
-Version: 1.3.4rc2
+Version: 1.3.5
 Summary: Client to connect to a cornflow server
 Home-page: https://github.com/baobabsoluciones/cornflow
 Author: baobab soluciones
@@ -16,6 +16,7 @@ Requires-Dist: requests==2.32.4
 Requires-Dist: genson==1.2.2
 Requires-Dist: jsonschema==4.21.1
 Requires-Dist: marshmallow==3.26.2
+Requires-Dist: openpyxl==3.1.5
 Requires-Dist: pytups==0.86.2
 Requires-Dist: ortools==9.8.3296
 Requires-Dist: PuLP==2.9.0

{cornflow_client-1.3.4rc2 → cornflow_client-1.3.5}/cornflow_client/airflow/dag_utilities.py

@@ -8,6 +8,10 @@ from datetime import datetime, timedelta
 from urllib.parse import urlparse, urljoin
 
 from cornflow_client import CornFlow, CornFlowApiError
+from cornflow_client.constants import (
+    EXECUTION_FILES_STATUS_ERROR,
+    EXECUTION_FILES_STATUS_NOT_GENERATED,
+)
 
 # TODO: convert everything to an object that encapsulates everything
 #  to make it clear and avoid all the arguments.
@@ -147,6 +151,22 @@ def try_to_write_solution(client, exec_id, payload):
             raise AirflowDagException("The writing of the instance checks failed")
 
 
+def try_to_write_file(client, exec_id, payload):
+    """
+    Tries to write the generated execution files into cornflow.
+    If it fails, tries to write again that it failed.
+    If it fails at least once: raises an exception.
+    """
+    try:
+        client.write_execution_files(execution_id=exec_id, **payload)
+    except CornFlowApiError:
+        client.write_execution_files(
+            execution_id=exec_id, execution_files_status=EXECUTION_FILES_STATUS_ERROR
+        )
+        # attempt to update the execution with a failed status.
+        raise AirflowDagException("The writing of the files failed")
+
+
 def get_schema(dag_name):
     # TODO: check if in use
     _file = os.path.join(os.path.dirname(__file__), f"{dag_name}_output.json")
@@ -185,9 +205,16 @@ def cf_solve(fun, dag_name, secrets, **kwargs):
         config = execution_data["config"]
         inst_id = execution_data["id"]
 
-        solution, sol_checks, inst_checks, kpis, log, log_json = fun(
-            data, config, solution_data
-        )
+        (
+            solution,
+            sol_checks,
+            inst_checks,
+            kpis,
+            execution_zip_file,
+            zip_file_status,
+            log,
+            log_json,
+        ) = fun(data, config, solution_data)
 
         # We connect again to cornflow in case that more than 24 hours
         # have passed from the first time we connect
@@ -201,11 +228,20 @@ def cf_solve(fun, dag_name, secrets, **kwargs):
             inst_checks=inst_checks,
             inst_id=inst_id,
         )
+
         if not solution:
             # No solution found: we just send everything to cornflow.
             if config.get("msg", True):
                 print("No solution found: we save what we have.")
             try_to_write_solution(client, exec_id, payload)
+
+            if zip_file_status != EXECUTION_FILES_STATUS_NOT_GENERATED:
+                file_payload = {
+                    "execution_files_status": zip_file_status,
+                    "execution_file": execution_zip_file,
+                }
+
+                try_to_write_file(client, exec_id, file_payload)
             return "Solution was not saved"
         # There is a solution:
         # we first need to validate the schema.
@@ -223,6 +259,14 @@ def cf_solve(fun, dag_name, secrets, **kwargs):
 
         try_to_write_solution(client, exec_id, payload)
 
+        if zip_file_status != EXECUTION_FILES_STATUS_NOT_GENERATED:
+            file_payload = {
+                "execution_files_status": zip_file_status,
+                "execution_file": execution_zip_file,
+            }
+
+            try_to_write_file(client, exec_id, file_payload)
+
         # The validation went correctly: can save the solution without problem
         return "Solution saved"
 
@@ -268,8 +312,8 @@ def cf_check_generate_kpis(fun_check_generate_kpis, dag_name, secrets, **kwargs)
         inst_id = execution_data["id"]
         solution_data = execution_data["solution_data"]
 
-        inst_checks, sol_checks, kpis, log_json = fun_check_generate_kpis(
-            instance_data, solution_data
+        inst_checks, sol_checks, kpis, execution_zip_file, zip_file_status, log_json = (
+            fun_check_generate_kpis(instance_data, solution_data)
         )
 
         if config.get("checks_and_kpis_only"):
@@ -299,6 +343,12 @@ def cf_check_generate_kpis(fun_check_generate_kpis, dag_name, secrets, **kwargs)
             payload["kpis"] = kpis
 
         try_to_write_solution(client, exec_id, payload)
+        if zip_file_status != EXECUTION_FILES_STATUS_NOT_GENERATED:
+            file_payload = {
+                "execution_files_status": zip_file_status,
+                "execution_file": execution_zip_file,
+            }
+            try_to_write_file(client, exec_id, file_payload)
 
         case_id = kwargs["dag_run"].conf.get("case_id")
         if case_id is not None:

{cornflow_client-1.3.4rc2 → cornflow_client-1.3.5}/cornflow_client/constants.py

@@ -27,6 +27,7 @@ SOLUTION_SCHEMA = "solution"
 CONFIG_SCHEMA = "config"
 INSTANCE_CHECKS_SCHEMA = "instance_checks"
 SOLUTION_CHECKS_SCHEMA = "solution_checks"
+KPIS_SCHEMA = "kpis"
 
 # why it stopped
 STATUS_NOT_SOLVED = 0
@@ -60,6 +61,13 @@ STATUS_CONV = {
 SOLUTION_STATUS_INFEASIBLE = 0
 SOLUTION_STATUS_FEASIBLE = 2
 
+# are there execution files?
+EXECUTION_FILES_STATUS_NOT_GENERATED = 0
+EXECUTION_FILES_STATUS_ERROR = -1
+EXECUTION_FILES_STATUS_DELETED = -2
+EXECUTION_FILES_STATUS_NOT_UP_TO_DATE = -3
+EXECUTION_FILES_STATUS_OK = 1
+
 PYOMO_STOP_MAPPING = {
     "unbounded": STATUS_UNBOUNDED,
     "infeasible": STATUS_INFEASIBLE,

{cornflow_client-1.3.4rc2 → cornflow_client-1.3.5}/cornflow_client/core/application.py

@@ -21,6 +21,7 @@ from cornflow_client.constants import (
     BadConfiguration,
     BadSolution,
     BadInstance,
+    EXECUTION_FILES_STATUS_NOT_GENERATED,
 )
 from .experiment import ExperimentCore
 
@@ -186,7 +187,8 @@ class ApplicationCore(ABC):
                 )
         return inst, sol
 
-    def _check_instance_errors(self, inst):
+    @staticmethod
+    def _check_instance_errors(inst):
         """Performs instance data checks and identifies critical errors."""
         instance_checks = SuperDict(inst.data_checks())
         warnings_tables = (
@@ -209,6 +211,31 @@ class ApplicationCore(ABC):
 
         return instance_checks, has_errors
 
+    @staticmethod
+    def _check_solution_errors(exp):
+        """Performs solution checks and identifies critical errors."""
+        solution_checks = SuperDict(exp.data_checks())
+        warnings_tables = (
+            SuperDict.from_dict(exp.schema_checks)
+            .get("properties", SuperDict())
+            .vfilter(lambda v: v.get("is_warning", False))
+            .keys()
+        )
+        solution_errors = solution_checks.kfilter(lambda k: k not in warnings_tables)
+
+        has_errors = False
+        for error_table in solution_errors.values():
+            # Check if the error table/value indicates an error
+            if isinstance(error_table, (list, dict)) and len(error_table) > 0:
+                has_errors = True
+                break
+            # Check for non-empty scalar values considered errors
+            elif error_table and not isinstance(error_table, (list, dict)):
+                has_errors = True
+                break
+
+        return solution_checks, has_errors
+
     def _execute_solver(self, inst, sol, config):
         """Instantiates and runs the solver, returning output and timing."""
         solver_name = config.get("solver", self.get_default_solver_name())
@@ -285,16 +312,16 @@ class ApplicationCore(ABC):
         # and the solver implements data_checks
         # Checks for non-None and non-empty dict
         if final_sol_dict:
-            solution_checks = SuperDict(algo.data_checks()).vfilter(lambda v: len(v))
+            solution_checks, solution_has_errors = self._check_solution_errors(algo)
 
             # Generate KPIs only if there are no solution checks
             #   (i.e., no issues found in the solution)
-            if not solution_checks:
+            if not solution_has_errors:
                 kpis = algo.get_kpis()
                 kpis_checks = algo.kpis_checks()
                 solution_checks.update(kpis_checks)
 
-        return final_sol_dict, solution_checks, kpis
+        return final_sol_dict, solution_checks, solution_has_errors, kpis
 
     def _format_log_and_solution(self, algo, output, solver_name, elapsed_time):
         """Formats logs, validates and checks the solution, generates the kpis"""
@@ -313,14 +340,22 @@ class ApplicationCore(ABC):
         # 4. Validate and check solution if feasible/optimal
         final_sol_dict = None
         solution_checks = None
+        solution_has_errors = None
         kpis = None
         if final_sol_code > 0:
             # Raises BadSolution if schema validation fails
-            final_sol_dict, solution_checks, kpis = self._validate_and_check_solution(
-                algo
+            final_sol_dict, solution_checks, solution_has_errors, kpis = (
+                self._validate_and_check_solution(algo)
             )
 
-        return final_sol_dict, solution_checks, kpis, log_txt, log_json
+        return (
+            final_sol_dict,
+            solution_checks,
+            solution_has_errors,
+            kpis,
+            log_txt,
+            log_json,
+        )
 
     def solve(
         self, data: dict, config: dict, solution_data: dict = None
@@ -345,9 +380,9 @@ class ApplicationCore(ABC):
         inst, sol_initial = self._prepare_instance_and_solution(data, solution_data)
 
         # 3. Check instance for critical errors
-        instance_checks, has_errors = self._check_instance_errors(inst)
+        instance_checks, instance_has_errors = self._check_instance_errors(inst)
 
-        if has_errors:
+        if instance_has_errors:
             # Return early if instance has critical errors
             solver_name = config.get("solver", self.get_default_solver_name())
             log_json = dict(
@@ -358,7 +393,16 @@ class ApplicationCore(ABC):
                 sol_code=SOLUTION_STATUS_INFEASIBLE,
             )
             # Ensure solution dict is empty, not None, consistent with successful returns
-            return dict(), None, instance_checks, None, "", log_json
+            return (
+                dict(),
+                None,
+                instance_checks,
+                None,
+                None,
+                EXECUTION_FILES_STATUS_NOT_GENERATED,
+                "",
+                log_json,
+            )
 
         # 4. Execute solver
         algo, output, solver_name, elapsed_time = self._execute_solver(
@@ -366,15 +410,28 @@ class ApplicationCore(ABC):
         )
 
         # 5. Format log and process solution
-        sol_final, solution_checks, kpis, log_txt, log_json = (
+        sol_final, solution_checks, solution_has_errors, kpis, log_txt, log_json = (
             self._format_log_and_solution(algo, output, solver_name, elapsed_time)
         )
 
+        execution_zip_file, zip_file_status = algo._get_zip_file(
+            instance_checks, instance_has_errors, solution_checks, solution_has_errors
+        )
+
         # Ensure solution is an empty dict if None for type consistency
         if sol_final is None:
             sol_final = dict()
 
-        return sol_final, solution_checks, instance_checks, kpis, log_txt, log_json
+        return (
+            sol_final,
+            solution_checks,
+            instance_checks,
+            kpis,
+            execution_zip_file,
+            zip_file_status,
+            log_txt,
+            log_json,
+        )
 
     def check_generate_kpis(
         self, instance_data: dict, solution_data: dict = None
@@ -391,27 +448,33 @@ class ApplicationCore(ABC):
             raise NoSolverException("No solver is available")
         inst = self.instance.from_dict(instance_data)
 
-        instance_checks = inst.data_checks()
+        instance_checks, instance_has_errors = self._check_instance_errors(inst)
 
+        solution_has_errors = None
         if solution_data is not None:
             sol = self.solution.from_dict(solution_data)
             algo = solver_class(inst, sol)
             start = timer()
-            solution_checks = SuperDict(algo.data_checks()).vfilter(lambda v: len(v))
+            solution_checks, solution_has_errors = self._check_solution_errors(algo)
             kpis = None
 
             # Generate KPIs only if there are no solution checks
             #   (i.e., no issues found in the solution)
             # Then, run the checks on the KPIs.
-            if not solution_checks:
+            if not instance_has_errors and not solution_has_errors:
                 kpis = algo.get_kpis()
                 kpis_checks = algo.kpis_checks()
                 solution_checks.update(kpis_checks)
         else:
+            algo = solver_class(inst, None)
             start = timer()
             solution_checks = None
             kpis = None
 
+        execution_zip_file, zip_file_status = algo._get_zip_file(
+            instance_checks, instance_has_errors, solution_checks, solution_has_errors
+        )
+
         log = dict(
             time=timer() - start,
             solver=solver,
@@ -420,7 +483,14 @@ class ApplicationCore(ABC):
             sol_code=SOLUTION_STATUS_FEASIBLE,
         )
 
-        return instance_checks, solution_checks, kpis, log
+        return (
+            instance_checks,
+            solution_checks,
+            kpis,
+            execution_zip_file,
+            zip_file_status,
+            log,
+        )
 
     def get_solver(self, name: str = "default") -> Union[Type[ExperimentCore], None]:
         """
@@ -450,4 +520,5 @@ class ApplicationCore(ABC):
             config=self.schema,
             instance_checks=self.instance.schema_checks,
             solution_checks=list(self.solvers.values())[0].schema_checks,
+            kpis=list(self.solvers.values())[0].schema_kpis,
         )

{cornflow_client-1.3.4rc2 → cornflow_client-1.3.5}/cornflow_client/core/experiment.py

@@ -3,20 +3,28 @@ Base code for the experiment template.
 """
 
 from abc import ABC, abstractmethod
-from typing import List, Dict, Union, Tuple
-import logging as log
-
 from jsonschema import Draft7Validator
+from typing import List, Dict, Union, Tuple, Optional
+import io
+import logging as log
+import os
+import shutil
+import traceback
+import zipfile
 
 from cornflow_client.constants import (
     PARAMETER_SOLVER_TRANSLATING_MAPPING,
     SOLVER_CONVERTER,
     BadSolutionChecks,
     BadKPIs,
+    EXECUTION_FILES_STATUS_NOT_GENERATED,
+    EXECUTION_FILES_STATUS_ERROR,
+    EXECUTION_FILES_STATUS_OK,
 )
 from .instance import InstanceCore
 from .instance_solution import CheckCore
 from .solution import SolutionCore
+from .tools import to_excel_memory_file
 
 
 class ExperimentCore(CheckCore, ABC):
@@ -24,6 +32,8 @@ class ExperimentCore(CheckCore, ABC):
     The solver template.
     """
 
+    schema_kpis = {}
+
     def __init__(
         self,
         instance: InstanceCore,
@@ -191,14 +201,6 @@ class ExperimentCore(CheckCore, ABC):
         kpis = {k: v for k, v in kpis.items() if v is not None and len(v)}
         return kpis
 
-    @property
-    def schema_kpis(self) -> dict:
-        """
-        A dictionary representation of the json-schema for the dictionary returned by
-            the method ExperimentCore.generate_kpis()
-        """
-        return {}
-
     def check_kpis(self) -> dict:
         """
         Method that checks the solution with the KPIs. By default, it does not perform any check.
@@ -218,6 +220,148 @@ class ExperimentCore(CheckCore, ABC):
         """
         return self.kpis
 
+    def generate_output_files(
+        self, instance_checks, instance_has_errors, solution_checks, solution_has_errors
+    ) -> Optional[Dict[str, Union[str, io.BytesIO]]]:
+        """
+        Method that generates the output files.
+        This method can be overriden by the user.
+        :return: a dictionary, where:
+        - the keys are the names of the files or directories
+        - the values are either:
+            * an io.BytesIO instance
+            * an io.StringIO instance
+            * a string representing an absolute path to a file or directory.
+        If the values are paths, the files or directories will be deleted after generating the zip file.
+        """
+        return None
+
+    @staticmethod
+    def _clean_output_files(output_files):
+        """
+        Removes output files after generating the zip file.
+        """
+        for _, item in output_files.items():
+            try:
+                if not isinstance(item, str):
+                    # Item is a bytesIO or stringIO object
+                    continue
+                if not os.path.exists(item):
+                    # Path was not found
+                    continue
+                if os.path.isdir(item):
+                    # Path is a directory
+                    shutil.rmtree(item)
+                else:
+                    # Path is a file
+                    os.remove(item)
+            except:
+                print(f"File/Directory {item} could not be deleted")
+                continue
+
+    def _get_default_output_files(self, instance_checks, solution_checks):
+        """
+        Method that generates the default output files:
+            - Instance data Excel
+            - Solution data Excel
+            - Checks Excel
+            - Solution checks Excel
+            - KPIs Excel
+        """
+        default_files = {}
+        for excel_name, data in [
+            ("instance", self.instance.to_dict()),
+            ("solution", self.solution.to_dict()),
+            ("checks", instance_checks),
+            ("solution_checks", solution_checks),
+            ("kpis", self.kpis),
+        ]:
+            if data is None:
+                continue
+            data = {k: v for k, v in data.items() if len(v)}
+            if not len(data):
+                continue
+            default_files[f"{excel_name}.xlsx"] = to_excel_memory_file(data)
+        return default_files
+
+    @classmethod
+    def _add_folder_to_zip(cls, zip_file, dir_path_in_zip, dir_path):
+        """
+        Recursively add the folder contents to the zip file.
+        """
+        for item in os.listdir(dir_path):
+            full_path = os.path.join(dir_path, item)
+            if os.path.isfile(full_path):
+                # File
+                file_path_in_zip = os.path.join(dir_path_in_zip, item)
+                zip_file.write(full_path, file_path_in_zip)
+            else:
+                # Subdirectory
+                subdirectory_path = os.path.join(dir_path, item)
+                subdir_path_in_zip = os.path.join(dir_path_in_zip, item)
+                cls._add_folder_to_zip(zip_file, subdir_path_in_zip, subdirectory_path)
+
+    @classmethod
+    def _generate_zip_file(cls, output_items):
+        """
+        Method that generates a zip file for the execution files.
+        :return: a zip file as a BytesIO object.
+        """
+        memory_file = io.BytesIO()
+
+        with zipfile.ZipFile(memory_file, "w", zipfile.ZIP_DEFLATED) as zf:
+            for item_path_in_zip, item in output_items.items():
+                if item_path_in_zip.startswith("/") or item_path_in_zip.startswith(
+                    "\\"
+                ):
+                    item_path_in_zip = item_path_in_zip[1:]
+                if isinstance(item, str) and os.path.isdir(item):
+                    # Directory path
+                    cls._add_folder_to_zip(zf, item_path_in_zip, item)
+                elif isinstance(item, str):
+                    # File path
+                    zf.write(item, item_path_in_zip)
+                elif isinstance(item, io.BytesIO) or isinstance(item, io.StringIO):
+                    # File stream
+                    zf.writestr(item_path_in_zip, item.getvalue())
+                else:
+                    print(item_path_in_zip, item)
+                    raise Exception(
+                        "Only paths, BytesIO and StringIO objects are accepted for output files"
+                    )
+
+        memory_file.seek(0)
+        return memory_file
+
+    def _get_zip_file(
+        self, instance_checks, instance_has_errors, solution_checks, solution_has_errors
+    ):
+        """
+        Method that generates a zip file for the execution files.
+        :return: a zip file, and a status indicating whether the generation went correctly.
+        """
+        try:
+            output_files = self.generate_output_files(
+                instance_checks,
+                instance_has_errors,
+                solution_checks,
+                solution_has_errors,
+            )
+            if output_files is None:
+                return None, EXECUTION_FILES_STATUS_NOT_GENERATED
+            output_files = {
+                **output_files,
+                **self._get_default_output_files(instance_checks, solution_checks),
+            }
+            zip_file = self._generate_zip_file(output_files)
+            self._clean_output_files(output_files)
+            zip_file_status = EXECUTION_FILES_STATUS_OK
+        except:
+            print(f"Error generating zipfile: {traceback.format_exc()}")
+            zip_file = None
+            zip_file_status = EXECUTION_FILES_STATUS_ERROR
+        return zip_file, zip_file_status
+
     @staticmethod
     def get_solver_config(
         config, lib="pyomo", default_solver="cbc", remove_unknown=False