runnable 0.50.0__py3-none-any.whl

extensions/secrets/dotenv.py

@@ -0,0 +1,62 @@
+import logging
+
+from dotenv import dotenv_values
+
+from runnable import defaults, exceptions
+from runnable.secrets import BaseSecrets
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+class DotEnvSecrets(BaseSecrets):
+    """
+    A secret manager which uses .env files for secrets.
+
+    We recommend this secrets manager only for local development and should not be used for anything close to
+    production.
+    """
+
+    service_name: str = "dotenv"
+    location: str = defaults.DOTENV_FILE_LOCATION
+    secrets: dict = {}
+
+    @property
+    def secrets_location(self):
+        """
+        Return the location of the .env file.
+        If the user has not over-ridden it, it defaults to .env file in the project root.
+
+        Returns:
+            str: The location of the secrets file
+        """
+        return self.location
+
+    def _load_secrets(self):
+        """
+        Use dotenv to load the secrets
+        """
+        self.secrets = dotenv_values(self.secrets_location)
+
+    def get(self, name: str = "") -> str:
+        """
+        Get a secret of name from the secrets file.
+
+
+        Args:
+            name (str): The name of the secret to retrieve
+
+        Raises:
+            Exception: If the secret by the name is not found.
+
+        Returns:
+            str: The value of the secret
+        """
+        if not self.secrets:
+            self._load_secrets()
+
+        if name in self.secrets:
+            return self.secrets[name]
+
+        raise exceptions.SecretNotFoundError(
+            secret_name=name, secret_setting=self.secrets_location
+        )

extensions/secrets/pyproject.toml

@@ -0,0 +1,15 @@
+[project]
+name = "secrets"
+version = "0.0.0"
+description = "Extension to manage secrets"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

runnable/__init__.py

@@ -0,0 +1,108 @@
+# ruff: noqa
+
+import os
+
+from rich.console import Console
+
+console = Console(record=True)
+console.print(":runner: Lets go!!")
+
+task_console = Console(record=True)
+
+
+def _configure_telemetry():
+    """
+    Auto-configure logfire/OpenTelemetry from environment variables.
+
+    This runs at import time to ensure telemetry is configured before
+    any task execution, especially for containerized execution where
+    the entrypoint is `runnable execute_single_node ...`.
+
+    Environment variables:
+        RUNNABLE_TELEMETRY_CONSOLE: Set to "true" for console output
+        OTEL_EXPORTER_OTLP_ENDPOINT: OTLP collector endpoint (e.g., http://localhost:4317)
+        LOGFIRE_TOKEN: Logfire cloud token (enables send_to_logfire)
+    """
+    import logfire_api as logfire
+
+    console_enabled = os.environ.get("RUNNABLE_TELEMETRY_CONSOLE", "").lower() == "true"
+    otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "")
+    logfire_token = os.environ.get("LOGFIRE_TOKEN", "")
+
+    # Skip if no telemetry config is set
+    if not (console_enabled or otlp_endpoint or logfire_token):
+        return
+
+    try:
+        import logfire
+
+        config_kwargs = {
+            "send_to_logfire": bool(logfire_token),
+        }
+
+        if console_enabled:
+            config_kwargs["console"] = logfire.ConsoleOptions(
+                colors="auto",
+                span_style="indented",
+                verbose=True,
+            )
+
+        if otlp_endpoint:
+            try:
+                from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+                    OTLPSpanExporter,
+                )
+                from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+                config_kwargs["additional_span_processors"] = [
+                    BatchSpanProcessor(OTLPSpanExporter(endpoint=otlp_endpoint))
+                ]
+            except ImportError:
+                pass  # OTLP exporter not installed
+
+        logfire.configure(**config_kwargs)  # ty: ignore
+
+    except ImportError:
+        pass  # logfire not installed, telemetry is no-op
+
+
+_configure_telemetry()
+
+
+from runnable.sdk import (  # noqa;
+    AsyncPipeline,
+    AsyncPythonTask,
+    Catalog,
+    Conditional,
+    Fail,
+    Loop,
+    Map,
+    NotebookJob,
+    NotebookTask,
+    Parallel,
+    Pipeline,
+    PythonJob,
+    PythonTask,
+    ShellJob,
+    ShellTask,
+    Stub,
+    Success,
+    json,
+    metric,
+    pickled,
+)
+from runnable.telemetry import (  # noqa;
+    OTEL_AVAILABLE,
+    get_stream_queue,
+    set_stream_queue,
+    truncate_value,
+)
+
+# Conditionally export StreamingSpanProcessor
+if OTEL_AVAILABLE:
+    from runnable.telemetry import StreamingSpanProcessor  # noqa;
+else:
+    StreamingSpanProcessor = None  # type: ignore
+
+# Needed to disable ploomber telemetry
+os.environ["PLOOMBER_STATS_ENABLED"] = "false"

runnable/catalog.py

@@ -0,0 +1,141 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+from pydantic import BaseModel, ConfigDict, Field
+
+import runnable.context as context
+from runnable import defaults
+from runnable.datastore import DataCatalog
+
+logger = logging.getLogger(defaults.LOGGER_NAME)
+
+
+# --8<-- [start:docs]
+
+
+class BaseCatalog(ABC, BaseModel):
+    """
+    Base Catalog class definition.
+
+    All implementations of the catalog handler should inherit and extend this class.
+    """
+
+    service_name: str = ""
+    service_type: str = "catalog"
+
+    compute_data_folder: str = Field(default=defaults.COMPUTE_DATA_FOLDER)
+
+    model_config = ConfigDict(extra="forbid")
+
+    @abstractmethod
+    def get_summary(self) -> Dict[str, Any]: ...
+
+    @property
+    def _context(self):
+        current_context = context.get_run_context()
+        if current_context is None:
+            raise RuntimeError("No run context available")
+        return current_context
+
+    @abstractmethod
+    def get(self, name: str) -> List[DataCatalog]:
+        """
+        Get the catalog item by 'name' for the 'run id' and store it in compute data folder.
+
+        The catalog location should have been created before you can get from it.
+
+        Args:
+            name (str): The name of the catalog item
+            run_id (str): The run_id of the run.
+            compute_data_folder (str, optional): The compute data folder. Defaults to runnable default (data/)
+
+        Raises:
+            NotImplementedError: Base class, hence not implemented
+
+        Returns:
+            List(object) : A list of catalog objects
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def put(
+        self, name: str, allow_file_not_found_exc: bool = False, store_copy: bool = True
+    ) -> List[DataCatalog]:
+        """
+        Put the file by 'name' from the 'compute_data_folder' in the catalog for the run_id.
+
+        If previous syncing has happened and the file has not been changed, we do not sync again.
+
+        Args:
+            name (str): The name of the catalog item.
+            run_id (str): The run_id of the run.
+            compute_data_folder (str, optional): The compute data folder. Defaults to runnable default (data/)
+            synced_catalogs (dict, optional): Any previously synced catalogs. Defaults to None.
+
+        Raises:
+            NotImplementedError: Base class, hence not implemented
+
+        Returns:
+            List(object) : A list of catalog objects
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def sync_between_runs(self, previous_run_id: str, run_id: str):
+        """
+        Given run_id of a previous run, sync them to the catalog of the run given by run_id
+
+        Args:
+            previous_run_id (str): The run id of the previous run
+            run_id (str): The run_id to which the data catalogs should be synced to.
+
+        Raises:
+            NotImplementedError: Base class, hence not implemented
+        """
+        raise NotImplementedError
+
+
+# --8<-- [end:docs]
+
+
+class DoNothingCatalog(BaseCatalog):
+    """
+    A Catalog handler that does nothing.
+
+    Example config:
+
+    catalog:
+      type: do-nothing
+
+    """
+
+    service_name: str = "do-nothing"
+
+    def get_summary(self) -> Dict[str, Any]:
+        return {}
+
+    def get(self, name: str) -> List[DataCatalog]:
+        """
+        Does nothing
+        """
+        logger.info("Using a do-nothing catalog, doing nothing in get")
+        return []
+
+    def put(
+        self,
+        name: str,
+        allow_file_not_found_exc: bool = False,
+        store_copy: bool = True,
+    ) -> List[DataCatalog]:
+        """
+        Does nothing
+        """
+        logger.info("Using a do-nothing catalog, doing nothing in put")
+        return []
+
+    def sync_between_runs(self, previous_run_id: str, run_id: str):
+        """
+        Does nothing
+        """
+        logger.info("Using a do-nothing catalog, doing nothing while sync between runs")