runnable 0.12.3__py3-none-any.whl → 0.14.0__py3-none-any.whl

runnable/__init__.py

@@ -34,14 +34,3 @@ from runnable.sdk import (  # noqa
 
 # Needed to disable ploomber telemetry
 os.environ["PLOOMBER_STATS_ENABLED"] = "false"
-
-## TODO: Summary should be a bit better for catalog.
-## If the execution fails, hint them about the retry executor.
-# Make the retry executor loose!
-
-# TODO: Think of model registry as a central place to store models.
-# TODO: Implement Sagemaker pipelines as a executor.
-
-
-# TODO: Think of way of generating dag hash without executor configuration
-# Try to get a release

runnable/catalog.py

@@ -11,6 +11,26 @@ from runnable.datastore import DataCatalog
 logger = logging.getLogger(defaults.LOGGER_NAME)
 
 
+def is_catalog_out_of_sync(
+    catalog, synced_catalogs=Optional[List[DataCatalog]]
+) -> bool:
+    """
+    Check if the catalog items are out of sync from already cataloged objects.
+    If they are, return False.
+    If the object does not exist or synced catalog does not exist, return True
+    """
+    if not synced_catalogs:
+        return True  # If nothing has been synced in the past
+
+    for synced_catalog in synced_catalogs:
+        if synced_catalog.catalog_relative_path == catalog.catalog_relative_path:
+            if synced_catalog.data_hash == catalog.data_hash:
+                return False
+            return True
+
+    return True  # The object does not exist, sync it
+
+
 # --8<-- [start:docs]
 
 
@@ -26,8 +46,7 @@ class BaseCatalog(ABC, BaseModel):
     model_config = ConfigDict(extra="forbid")
 
     @abstractmethod
-    def get_summary(self) -> Dict[str, Any]:
-        ...
+    def get_summary(self) -> Dict[str, Any]: ...
 
     @property
     def _context(self):
@@ -38,7 +57,9 @@ class BaseCatalog(ABC, BaseModel):
         return defaults.COMPUTE_DATA_FOLDER
 
     @abstractmethod
-    def get(self, name: str, run_id: str, compute_data_folder: str = "", **kwargs) -> List[DataCatalog]:
+    def get(
+        self, name: str, run_id: str, compute_data_folder: str = "", **kwargs
+    ) -> List[DataCatalog]:
         """
         Get the catalog item by 'name' for the 'run id' and store it in compute data folder.
 
@@ -119,7 +140,9 @@ class DoNothingCatalog(BaseCatalog):
     def get_summary(self) -> Dict[str, Any]:
         return {}
 
-    def get(self, name: str, run_id: str, compute_data_folder: str = "", **kwargs) -> List[DataCatalog]:
+    def get(
+        self, name: str, run_id: str, compute_data_folder: str = "", **kwargs
+    ) -> List[DataCatalog]:
         """
         Does nothing
         """
@@ -145,4 +168,3 @@ class DoNothingCatalog(BaseCatalog):
         Does nothing
         """
         logger.info("Using a do-nothing catalog, doing nothing while sync between runs")
-        ...

runnable/cli.py

@@ -22,9 +22,13 @@ def cli():
 
 
 @cli.command("execute", short_help="Execute/translate a pipeline")
-@click.option("-f", "--file", default="pipeline.yaml", help="The pipeline definition file", show_default=True)
+@click.argument("filename")
 @click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
+    "-c",
+    "--config-file",
+    default=None,
+    help="config file, in yaml, to be used for the run",
+    show_default=True,
 )
 @click.option(
     "-p",
@@ -41,8 +45,12 @@ def cli():
     type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
 )
 @click.option("--tag", default="", help="A tag attached to the run")
-@click.option("--run-id", help="An optional run_id, one would be generated if not provided")
-def execute(file, config_file, parameters_file, log_level, tag, run_id):  # pragma: no cover
+@click.option(
+    "--run-id", help="An optional run_id, one would be generated if not provided"
+)
+def execute(
+    filename, config_file, parameters_file, log_level, tag, run_id
+):  # pragma: no cover
     """
     Execute a pipeline
 
@@ -64,20 +72,35 @@ def execute(file, config_file, parameters_file, log_level, tag, run_id):  # prag
 
     entrypoints.execute(
         configuration_file=config_file,
-        pipeline_file=file,
+        pipeline_file=filename,
         tag=tag,
         run_id=run_id,
         parameters_file=parameters_file,
     )
 
 
-@cli.command("execute_single_node", short_help="Internal entry point to execute a single node", hidden=True)
+@cli.command(
+    "execute_single_node",
+    short_help="Internal entry point to execute a single node",
+    hidden=True,
+)
 @click.argument("run_id")
 @click.argument("step_name")
-@click.option("--map-variable", default="", help="The map variable dictionary in str", show_default=True)
-@click.option("-f", "--file", default="", help="The pipeline definition file", show_default=True)
 @click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
+    "--map-variable",
+    default="",
+    help="The map variable dictionary in str",
+    show_default=True,
+)
+@click.option(
+    "-f", "--file", default="", help="The pipeline definition file", show_default=True
+)
+@click.option(
+    "-c",
+    "--config-file",
+    default=None,
+    help="config file, in yaml, to be used for the run",
+    show_default=True,
 )
 @click.option(
     "-p",
@@ -94,7 +117,9 @@ def execute(file, config_file, parameters_file, log_level, tag, run_id):  # prag
     type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
 )
 @click.option("--tag", default="", help="A tag attached to the run")
-def execute_single_node(run_id, step_name, map_variable, file, config_file, parameters_file, log_level, tag):
+def execute_single_node(
+    run_id, step_name, map_variable, file, config_file, parameters_file, log_level, tag
+):
     """
     Internal entrypoint for runnable to execute a single node.
 
@@ -119,7 +144,11 @@ def execute_single_node(run_id, step_name, map_variable, file, config_file, para
 @click.argument("filename")
 @click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
 @click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
+    "-c",
+    "--config-file",
+    default=None,
+    help="config file, in yaml, to be used for the run",
+    show_default=True,
 )
 @click.option(
     "-p",
@@ -136,10 +165,20 @@ def execute_single_node(run_id, step_name, map_variable, file, config_file, para
     type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
 )
 @click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
-@click.option("--put-in-catalog", "-put", default=None, multiple=True, help="The data to put from the catalog")
-@click.option("--notebook-output-path", default="", help="The output path for the notebook")
+@click.option(
+    "--put-in-catalog",
+    "-put",
+    default=None,
+    multiple=True,
+    help="The data to put from the catalog",
+)
+@click.option(
+    "--notebook-output-path", default="", help="The output path for the notebook"
+)
 @click.option("--tag", help="A tag attached to the run")
-@click.option("--run-id", help="An optional run_id, one would be generated if not provided")
+@click.option(
+    "--run-id", help="An optional run_id, one would be generated if not provided"
+)
 def execute_notebook(
     filename,
     entrypoint,
@@ -159,7 +198,10 @@ def execute_notebook(
     The execution plan is unchained.
     """
     logger.setLevel(log_level)
-    catalog_config = {"compute_data_folder": data_folder, "put": list(put_in_catalog) if put_in_catalog else None}
+    catalog_config = {
+        "compute_data_folder": data_folder,
+        "put": list(put_in_catalog) if put_in_catalog else None,
+    }
     if not filename.endswith(".ipynb"):
         raise Exception("A notebook should always have ipynb as the extension")
 
@@ -179,7 +221,11 @@ def execute_notebook(
 @click.argument("command")
 @click.option("--entrypoint", default=defaults.ENTRYPOINT.USER.value, hidden=True)
 @click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
+    "-c",
+    "--config-file",
+    default=None,
+    help="config file, in yaml, to be used for the run",
+    show_default=True,
 )
 @click.option(
     "-p",
@@ -196,11 +242,27 @@ def execute_notebook(
     type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
 )
 @click.option("--data-folder", "-d", default="data/", help="The catalog data folder")
-@click.option("--put-in-catalog", "-put", default=None, multiple=True, help="The data to put from the catalog")
+@click.option(
+    "--put-in-catalog",
+    "-put",
+    default=None,
+    multiple=True,
+    help="The data to put from the catalog",
+)
 @click.option("--tag", help="A tag attached to the run")
-@click.option("--run-id", help="An optional run_id, one would be generated if not provided")
+@click.option(
+    "--run-id", help="An optional run_id, one would be generated if not provided"
+)
 def execute_function(
-    command, entrypoint, config_file, parameters_file, log_level, data_folder, put_in_catalog, tag, run_id
+    command,
+    entrypoint,
+    config_file,
+    parameters_file,
+    log_level,
+    data_folder,
+    put_in_catalog,
+    tag,
+    run_id,
 ):
     """
     External entry point to execute a python function in isolation.
@@ -209,7 +271,10 @@ def execute_function(
     The execution plan is unchained.
     """
     logger.setLevel(log_level)
-    catalog_config = {"compute_data_folder": data_folder, "put": list(put_in_catalog) if put_in_catalog else None}
+    catalog_config = {
+        "compute_data_folder": data_folder,
+        "put": list(put_in_catalog) if put_in_catalog else None,
+    }
     entrypoints.execute_function(
         entrypoint=entrypoint,
         command=command,
@@ -221,14 +286,35 @@ def execute_function(
     )
 
 
-@cli.command("fan", short_help="Internal entry point to fan in or out a composite node", hidden=True)
+@cli.command(
+    "fan",
+    short_help="Internal entry point to fan in or out a composite node",
+    hidden=True,
+)
 @click.argument("run_id")
 @click.argument("step_name")
-@click.option("-m", "--mode", help="fan in or fan out", required=True, type=click.Choice(["in", "out"]))
-@click.option("--map-variable", default="", help="The map variable dictionary in str", show_default=True)
-@click.option("-f", "--file", default="", help="The pipeline definition file", show_default=True)
 @click.option(
-    "-c", "--config-file", default=None, help="config file, in yaml, to be used for the run", show_default=True
+    "-m",
+    "--mode",
+    help="fan in or fan out",
+    required=True,
+    type=click.Choice(["in", "out"]),
+)
+@click.option(
+    "--map-variable",
+    default="",
+    help="The map variable dictionary in str",
+    show_default=True,
+)
+@click.option(
+    "-f", "--file", default="", help="The pipeline definition file", show_default=True
+)
+@click.option(
+    "-c",
+    "--config-file",
+    default=None,
+    help="config file, in yaml, to be used for the run",
+    show_default=True,
 )
 @click.option(
     "-p",
@@ -245,7 +331,17 @@ def execute_function(
     type=click.Choice(["INFO", "DEBUG", "WARNING", "ERROR", "FATAL"]),
 )
 @click.option("--tag", default="", help="A tag attached to the run")
-def fan(run_id, step_name, mode, map_variable, file, config_file, parameters_file, log_level, tag):
+def fan(
+    run_id,
+    step_name,
+    mode,
+    map_variable,
+    file,
+    config_file,
+    parameters_file,
+    log_level,
+    tag,
+):
     """
     Internal entrypoint for runnable to fan in or out a composite node.
 

runnable/datastore.py

@@ -18,7 +18,6 @@ from typing import (
 
 from pydantic import BaseModel, Field, computed_field
 
-import runnable.context as context
 from runnable import defaults, exceptions
 
 logger = logging.getLogger(defaults.LOGGER_NAME)
@@ -56,14 +55,11 @@ class DataCatalog(BaseModel, extra="allow"):
         return other.name == self.name
 
 
-"""
-The theory behind reduced:
-    parameters returned by steps in map node are only reduced by the end of the map step, fan-in.
-    If they are accessed within the map step, the value should be the value returned by the step in the map step.
-
-    Once the map state is complete, we can set the reduce to true and have the value as
-    the reduced value. Its either a list or a custom function return.
-"""
+# The theory behind reduced:
+#     parameters returned by steps in map node are only reduced by the end of the map step, fan-in.
+#     If they are accessed within the map step, the value should be the value returned by the step in the map step.
+#     Once the map state is complete, we can set the reduce to true and have the value as
+#     the reduced value. Its either a list or a custom function return.
 
 
 class JsonParameter(BaseModel):
@@ -125,7 +121,9 @@ class ObjectParameter(BaseModel):
         os.remove(self.file_name)  # Remove after loading
 
 
-Parameter = Annotated[Union[JsonParameter, ObjectParameter, MetricParameter], Field(discriminator="kind")]
+Parameter = Annotated[
+    Union[JsonParameter, ObjectParameter, MetricParameter], Field(discriminator="kind")
+]
 
 
 class StepAttempt(BaseModel):
@@ -157,8 +155,12 @@ class CodeIdentity(BaseModel, extra="allow"):
 
     code_identifier: Optional[str] = ""  # GIT sha code or docker image id
     code_identifier_type: Optional[str] = ""  # git or docker
-    code_identifier_dependable: Optional[bool] = False  # If git, checks if the tree is clean.
-    code_identifier_url: Optional[str] = ""  # The git remote url or docker repository url
+    code_identifier_dependable: Optional[bool] = (
+        False  # If git, checks if the tree is clean.
+    )
+    code_identifier_url: Optional[str] = (
+        ""  # The git remote url or docker repository url
+    )
     code_identifier_message: Optional[str] = ""  # Any optional message
 
 
@@ -185,18 +187,28 @@ class StepLog(BaseModel):
         summary: Dict[str, Any] = {}
 
         summary["Name"] = self.internal_name
-        summary["Input catalog content"] = [dc.name for dc in self.data_catalog if dc.stage == "get"]
+        summary["Input catalog content"] = [
+            dc.name for dc in self.data_catalog if dc.stage == "get"
+        ]
         summary["Available parameters"] = [
-            (p, v.description) for attempt in self.attempts for p, v in attempt.input_parameters.items()
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.input_parameters.items()
         ]
 
-        summary["Output catalog content"] = [dc.name for dc in self.data_catalog if dc.stage == "put"]
+        summary["Output catalog content"] = [
+            dc.name for dc in self.data_catalog if dc.stage == "put"
+        ]
         summary["Output parameters"] = [
-            (p, v.description) for attempt in self.attempts for p, v in attempt.output_parameters.items()
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.output_parameters.items()
         ]
 
         summary["Metrics"] = [
-            (p, v.description) for attempt in self.attempts for p, v in attempt.user_defined_metrics.items()
+            (p, v.description)
+            for attempt in self.attempts
+            for p, v in attempt.user_defined_metrics.items()
         ]
 
         cis = []
@@ -312,10 +324,18 @@ class RunLog(BaseModel):
         summary["Catalog Location"] = _context.catalog_handler.get_summary()
         summary["Full Run log present at: "] = _context.run_log_store.get_summary()
 
-        run_log = _context.run_log_store.get_run_log_by_id(run_id=_context.run_id, full=True)
+        run_log = _context.run_log_store.get_run_log_by_id(
+            run_id=_context.run_id, full=True
+        )
 
-        summary["Final Parameters"] = {p: v.description for p, v in run_log.parameters.items()}
-        summary["Collected metrics"] = {p: v.description for p, v in run_log.parameters.items() if v.kind == "metric"}
+        summary["Final Parameters"] = {
+            p: v.description for p, v in run_log.parameters.items()
+        }
+        summary["Collected metrics"] = {
+            p: v.description
+            for p, v in run_log.parameters.items()
+            if v.kind == "metric"
+        }
 
         return summary
 
@@ -338,7 +358,9 @@ class RunLog(BaseModel):
 
         return list(set(data_catalogs))
 
-    def search_branch_by_internal_name(self, i_name: str) -> Tuple[Union[BranchLog, RunLog], Union[StepLog, None]]:
+    def search_branch_by_internal_name(
+        self, i_name: str
+    ) -> Tuple[Union[BranchLog, RunLog], Union[StepLog, None]]:
         """
         Given a branch internal name, search for it in the run log.
 
@@ -385,7 +407,9 @@ class RunLog(BaseModel):
 
         raise exceptions.BranchLogNotFoundError(self.run_id, i_name)
 
-    def search_step_by_internal_name(self, i_name: str) -> Tuple[StepLog, Union[BranchLog, None]]:
+    def search_step_by_internal_name(
+        self, i_name: str
+    ) -> Tuple[StepLog, Union[BranchLog, None]]:
         """
         Given a steps internal name, search for the step name.
 
@@ -415,7 +439,9 @@ class RunLog(BaseModel):
                 # Its odd, so we are in brach name
                 current_branch = current_step.branches[".".join(dot_path[: i + 1])]  # type: ignore
                 current_steps = current_branch.steps
-                logger.debug(f"Finding step log for {i_name} in branch: {current_branch}")
+                logger.debug(
+                    f"Finding step log for {i_name} in branch: {current_branch}"
+                )
             else:
                 # Its even, so we are in step, we start here!
                 current_step = current_steps[".".join(dot_path[: i + 1])]
@@ -428,10 +454,6 @@ class RunLog(BaseModel):
         raise exceptions.StepLogNotFoundError(self.run_id, i_name)
 
 
-# All outside modules should interact with dataclasses using the RunLogStore to promote extensibility
-# If you want to customize dataclass, extend BaseRunLogStore and implement the methods as per the specification
-
-
 class BaseRunLogStore(ABC, BaseModel):
     """
     The base class of a Run Log Store with many common methods implemented.
@@ -441,8 +463,7 @@ class BaseRunLogStore(ABC, BaseModel):
     service_type: str = "run_log_store"
 
     @abstractmethod
-    def get_summary(self) -> Dict[str, Any]:
-        ...
+    def get_summary(self) -> Dict[str, Any]: ...
 
     @property
     def _context(self):
@@ -629,7 +650,9 @@ class BaseRunLogStore(ABC, BaseModel):
             RunLogNotFoundError: If the run log for run_id is not found in the datastore
             StepLogNotFoundError: If the step log for internal_name is not found in the datastore for run_id
         """
-        logger.info(f"{self.service_name} Getting the step log: {internal_name} of {run_id}")
+        logger.info(
+            f"{self.service_name} Getting the step log: {internal_name} of {run_id}"
+        )
         run_log = self.get_run_log_by_id(run_id=run_id)
         step_log, _ = run_log.search_step_by_internal_name(internal_name)
         return step_log
@@ -675,10 +698,14 @@ class BaseRunLogStore(ABC, BaseModel):
             BranchLog: Uncommitted and initialized with defaults BranchLog object
         """
         # Create a new BranchLog
-        logger.info(f"{self.service_name} Creating a Branch Log : {internal_branch_name}")
+        logger.info(
+            f"{self.service_name} Creating a Branch Log : {internal_branch_name}"
+        )
         return BranchLog(internal_name=internal_branch_name, status=defaults.CREATED)
 
-    def get_branch_log(self, internal_branch_name: str, run_id: str, **kwargs) -> Union[BranchLog, RunLog]:
+    def get_branch_log(
+        self, internal_branch_name: str, run_id: str, **kwargs
+    ) -> Union[BranchLog, RunLog]:
         """
         Returns the branch log by the internal branch name for the run id
 
@@ -697,7 +724,9 @@ class BaseRunLogStore(ABC, BaseModel):
         branch, _ = run_log.search_branch_by_internal_name(internal_branch_name)
         return branch
 
-    def add_branch_log(self, branch_log: Union[BranchLog, RunLog], run_id: str, **kwargs):
+    def add_branch_log(
+        self, branch_log: Union[BranchLog, RunLog], run_id: str, **kwargs
+    ):
         """
         The method should:
         # Get the run log
@@ -775,7 +804,9 @@ class BufferRunLogstore(BaseRunLogStore):
     """
 
     service_name: str = "buffered"
-    run_log: Optional[RunLog] = Field(default=None, exclude=True)  # For a buffered Run Log, this is the database
+    run_log: Optional[RunLog] = Field(
+        default=None, exclude=True
+    )  # For a buffered Run Log, this is the database
 
     def get_summary(self) -> Dict[str, Any]:
         summary = {"Type": self.service_name, "Location": "Not persisted"}
@@ -826,5 +857,10 @@ class BufferRunLogstore(BaseRunLogStore):
         # Puts the run log in the db
         # Raises Exception if not found
         """
-        logger.info(f"{self.service_name} Putting the run log in the DB: {run_log.run_id}")
+        logger.info(
+            f"{self.service_name} Putting the run log in the DB: {run_log.run_id}"
+        )
         self.run_log = run_log
+
+
+import runnable.context as context  # noqa: F401, E402

runnable/defaults.py

@@ -33,7 +33,6 @@ class RunnableConfig(TypedDict, total=False):
     secrets: Optional[ServiceConfig]
     catalog: Optional[ServiceConfig]
     executor: Optional[ServiceConfig]
-    experiment_tracker: Optional[ServiceConfig]
     pickler: Optional[ServiceConfig]