beamflow-lib 0.3.0__tar.gz

beamflow_lib-0.3.0/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 2.3
+Name: beamflow-lib
+Version: 0.3.0
+Summary: Shared domain logic and utilities for the Beamflow managed platform.
+Author: juraj.bezdek@gmail.com
+Author-email: juraj.bezdek@gmail.com
+Requires-Python: >=3.11
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: PyJWT (>=2.8.0)
+Requires-Dist: beamflow-clients (>=0.3.0,<0.4.0)
+Requires-Dist: cryptography (>=42.0.0)
+Requires-Dist: email-validator (>=2.0.0)
+Requires-Dist: google-cloud-build (>=3.23.0)
+Requires-Dist: google-cloud-firestore (>=2.16.0)
+Requires-Dist: google-cloud-scheduler (>=2.13.0)
+Requires-Dist: google-cloud-storage (>=2.14.0)
+Requires-Dist: google-cloud-tasks (>=2.16.0)
+Requires-Dist: httpx (>=0.27.0)
+Requires-Dist: opentelemetry-api (>=1.24.0)
+Requires-Dist: opentelemetry-exporter-otlp (>=1.24.0)
+Requires-Dist: opentelemetry-sdk (>=1.24.0)
+Requires-Dist: pydantic (>=2.10.0)
+Requires-Dist: pydantic-settings (>=2.2.1)
+Requires-Dist: python-dotenv (>=1.0.1)
+Requires-Dist: pyyaml (>=6.0.1)
+Requires-Dist: redis (>=5.0.3)

beamflow_lib-0.3.0/beamflow_lib/__init__.py

@@ -0,0 +1,21 @@
+from .observability.logging import logger
+from .context import integration_context, get_context
+from .decorators import integration_task, integration_step
+from .pipelines.ingress import ingress
+from .pipelines.records_feed import RecordsFeed
+from .pipelines.records_model import RecordData
+from .pipelines.consumer import feed_consumer, ConsumerRunner
+
+__all__ = [
+    "logger", 
+    "integration_context", 
+    "get_context", 
+    "integration_task", 
+    "integration_step",
+    "ingress",
+    "RecordsFeed",
+    "RecordData",
+    "feed_consumer",
+    "ConsumerRunner"
+]
+

beamflow_lib-0.3.0/beamflow_lib/config/env_loader.py

@@ -0,0 +1,127 @@
+import yaml
+from typing import Any, Dict, Optional
+from pathlib import Path
+from .runtime_config import RuntimeConfig, ClientsConfigPointer, BackendConfig, WebhooksConfig
+from .observability_config import ObservabilityConfig
+from beamflow_clients.config import ClientSettings
+
+def load_env_yaml(path: str | Path) -> Dict[str, Any]:
+    """Load an environment YAML file."""
+    with open(path, "r") as f:
+        data = yaml.safe_load(f)
+        return data if isinstance(data, dict) else {}
+
+def _load_data_from_dir(path: Path) -> tuple[Optional[Dict[str, Any]], Optional[Dict[str, Any]], Dict[str, ClientSettings]]:
+    """Helper to load config components from a single directory."""
+    # 1. Observability
+    obs_file = path / "observability.yaml"
+    observability = None
+    if obs_file.exists():
+        observability = load_env_yaml(obs_file)
+
+    # 2. Backend
+    backend_file = path / "backend.yaml"
+    backend = None
+    if backend_file.exists():
+        backend = load_env_yaml(backend_file)
+
+    # 3. Clients
+    clients_file = path / "clients.yaml"
+    clients: Dict[str, ClientSettings] = {}
+    
+    if clients_file.exists():
+        clients_pointer_data = load_env_yaml(clients_file)
+        pointer = ClientsConfigPointer.model_validate(clients_pointer_data)
+        
+        clients_dir = Path(pointer.path)
+        if not clients_dir.is_absolute():
+            clients_dir = (path / clients_dir).resolve()
+        
+        if not clients_dir.is_dir():
+             raise ValueError(f"Clients directory {clients_dir} (from clients.yaml) does not exist")
+
+        pattern = pointer.pattern
+        recursive = pointer.recursive
+        
+        search_pattern = f"**/{pattern}" if recursive else pattern
+        client_files = list(clients_dir.glob(search_pattern))
+        
+        for cf in client_files:
+            if not cf.is_file():
+                continue
+            
+            client_data = load_env_yaml(cf)
+            client_id = client_data.get("client_id") or client_data.get("clientId") or cf.stem
+            
+            if client_id in clients:
+                raise ValueError(f"Duplicate client_id '{client_id}' found in {cf}")
+
+            if "client_id" not in client_data:
+                client_data["client_id"] = client_id
+                
+            settings = ClientSettings.model_validate(client_data)
+            
+            # Known keys to exclude from extra
+            settings_fields = set(ClientSettings.model_fields.keys())
+            for field in ClientSettings.model_fields.values():
+                if field.alias:
+                    settings_fields.add(field.alias)
+            
+            known_keys = settings_fields | {"client_id", "clientId"}
+            extra = {k: v for k, v in client_data.items() if k not in known_keys}
+            settings.extra.update(extra)
+            
+            clients[client_id] = settings
+    return observability, backend, clients
+
+def load_config_dir(config_dir: str | Path, environment: str) -> RuntimeConfig:
+    """
+    Loads unified configuration from a directory.
+    - {config_dir}/shared/ - shared base config
+    - {config_dir}/{environment}/ - environment-specific overrides
+    """
+
+    print("Loading config for environment: ", environment)
+    config_path = Path(config_dir)
+    if not config_path.is_dir():
+        raise ValueError(f"Config directory {config_dir} does not exist or is not a directory")
+
+    shared_path = config_path / "shared"
+    env_path = config_path / environment
+
+    # Load shared config
+    obs_shared, backend_shared, clients_shared = None, None, {}
+    if shared_path.is_dir():
+        obs_shared, backend_shared, clients_shared = _load_data_from_dir(shared_path)
+
+    # Load env config
+    obs_env, backend_env, clients_env = None, None, {}
+    if env_path.is_dir():
+        obs_env, backend_env, clients_env = _load_data_from_dir(env_path)
+    elif environment != "shared":
+        raise ValueError(f"Environment directory {env_path} does not exist")
+
+    # Merge
+    # Observability: Env wins
+    observability_data = obs_env or obs_shared
+    observability = ObservabilityConfig.model_validate(observability_data) if observability_data else None
+
+    # Backend: Env wins
+    import os
+    backend_yaml_env = backend_env or backend_shared or {}
+    
+    backend_data = backend_yaml_env.get("backend")
+    backend_config = BackendConfig.model_validate(backend_data) if backend_data else BackendConfig()
+    
+    webhooks_data = backend_yaml_env.get("webhooks")
+    webhooks_config = WebhooksConfig.model_validate(webhooks_data) if webhooks_data else WebhooksConfig()
+    
+    # Clients: Merge dicts, Env wins on collision
+    clients = {**clients_shared, **clients_env}
+
+    return RuntimeConfig(
+        observability=observability,
+        backend=backend_config,
+        webhooks=webhooks_config,
+        clients=clients
+    )

beamflow_lib-0.3.0/beamflow_lib/config/observability_config.py

@@ -0,0 +1,37 @@
+from enum import Enum
+from typing import Optional
+from pydantic import BaseModel, ConfigDict, Field
+
+class DurabilityMode(str, Enum):
+    IMMEDIATE = "immediate"
+    EVENTUAL = "eventual"
+
+class StoreType(str, Enum):
+    GCP = "gcp"
+    LOCAL = "local"
+
+class LoggingConfig(BaseModel):
+    enabled: bool = True
+    min_level: str = "INFO"
+    include_prefixes: list[str] = Field(default_factory=list, alias="includePrefixes")
+    exclude_prefixes: list[str] = Field(default_factory=list, alias="excludePrefixes")
+    filter_fn: Optional[str] = Field(None, alias="filterFn")
+
+    model_config = ConfigDict(populate_by_name=True)
+
+class ObservabilityConfig(BaseModel):
+    durability: DurabilityMode = DurabilityMode.EVENTUAL
+    store_type: StoreType = StoreType.LOCAL
+    
+    # Local settings
+    local_store_path: str = Field("logs", alias="localStorePath")
+
+    # GCP settings
+    gcp_project_id: Optional[str] = Field(None, alias="gcpProjectId")
+    gcp_dataset_id: Optional[str] = Field(None, alias="gcpDatasetId")
+    gcp_bucket_name: Optional[str] = Field(None, alias="gcpBucketName")
+
+    # Logging settings
+    logging: LoggingConfig = Field(default_factory=LoggingConfig)
+
+    model_config = ConfigDict(populate_by_name=True)

beamflow_lib-0.3.0/beamflow_lib/config/runtime_config.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+from enum import Enum
+from typing import Dict, Any, Optional, List
+from pydantic import BaseModel, Field, ConfigDict
+from .observability_config import ObservabilityConfig
+from beamflow_clients.config import ClientSettings
+
+
+class ClientsConfigPointer(BaseModel):
+    path: str
+    pattern: str = "*.yaml"
+    recursive: bool = False
+
+class BackendType(str, Enum):
+    """Task queue backend types."""
+    ASYNC = "async_backend"      # In-process async execution
+    DRAMATIQ = "dramatiq"        # Dramatiq queue
+    MANAGED = "managed"          # Managed platform (Cloud Tasks via API)
+
+class DramatiqConfig(BaseModel):
+    """Dramatiq backend configuration."""
+    redis_url: Optional[str] = None # Redis connection string, overrides REDIS_URL env var if set
+    
+    model_config = ConfigDict(populate_by_name=True)
+
+class BackendConfig(BaseModel):
+    """Configuration for task backend."""
+    type: BackendType = BackendType.DRAMATIQ
+    dramatiq: Optional[DramatiqConfig] = None
+    
+    model_config = ConfigDict(populate_by_name=True)
+
+class WebhooksConfig(BaseModel):
+    """Configuration for webhook ingress."""
+    prefix: str = "/webhooks"
+    
+
+    model_config = ConfigDict(populate_by_name=True)
+
+class RuntimeConfig(BaseModel):
+    observability: Optional[ObservabilityConfig] = None
+    backend: BackendConfig = Field(default_factory=BackendConfig)
+    clients: Dict[str, ClientSettings] = Field(default_factory=dict)
+    webhooks: WebhooksConfig = Field(default_factory=WebhooksConfig)
+
+
+    model_config = ConfigDict(populate_by_name=True)
+
+class RuntimeConfigRegistry:
+    """
+    A registry for the runtime configuration.
+    """
+    def __init__(self, config: RuntimeConfig):
+        self._config = config
+        self._clients = config.clients
+
+    @property
+    def observability(self) -> Optional[ObservabilityConfig]:
+        return self._config.observability
+
+    @property
+    def backend(self) -> BackendConfig:
+        return self._config.backend
+
+    def get_client(self, client_id: str) -> Optional[ClientSettings]:
+        return self._clients.get(client_id)
+
+    def __getitem__(self, key: str) -> ClientSettings:
+        if key not in self._clients:
+            raise KeyError(f"Client '{key}' not found.")
+        return self._clients[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self._clients
+
+    @property
+    def clients(self) -> Dict[str, ClientSettings]:
+        return self._clients

beamflow_lib-0.3.0/beamflow_lib/context.py

@@ -0,0 +1,141 @@
+from dataclasses import dataclass, field
+from typing import Any, Mapping, Optional
+import contextvars
+import uuid
+
+@dataclass(frozen=True)
+class IntegrationContext:
+    integration: str
+    integration_pipeline: str
+    run_id: str
+    traceparent: Optional[str] = None
+    tracestate: Optional[str] = None
+    baggage: Mapping[str, str] = field(default_factory=dict)
+    tags: Mapping[str, Any] = field(default_factory=dict)
+    tenant_id: str = "default"
+    current_record_key: Optional[str] = None
+
+
+
+    @property
+    def corelation(self) -> Any:
+        # Import inside to avoid circular dependencies
+        from .observability.model import Correlation
+        
+        trace_id = None
+        span_id = None
+        if self.traceparent:
+            parts = self.traceparent.split("-")
+            if len(parts) >= 3:
+                trace_id = parts[1]
+                span_id = parts[2]
+
+        return Correlation(
+            integration=self.integration,
+            integration_pipeline=self.integration_pipeline,
+            run_id=self.run_id,
+            trace_id=self.trace_id,
+            span_id=self.span_id,
+            tags={k: str(v) for k, v in self.tags.items()}
+        )
+
+    @property
+    def trace_id(self) -> Optional[str]:
+        if self.traceparent:
+            parts = self.traceparent.split("-")
+            if len(parts) >= 2:
+                return parts[1]
+        return None
+
+    @property
+    def span_id(self) -> Optional[str]:
+        if self.traceparent:
+            parts = self.traceparent.split("-")
+            if len(parts) >= 3:
+                return parts[2]
+        return None
+
+
+
+_current_ctx: contextvars.ContextVar[Optional[IntegrationContext]] = contextvars.ContextVar(
+    "framework.integration_context", default=None
+)
+
+def current_context() -> Optional[IntegrationContext]:
+    """Return the current integration context."""
+    return _current_ctx.get()
+
+get_context = current_context
+
+def set_context(ctx: IntegrationContext) -> Any:
+    """Set the current integration context. Returns a token for resetting."""
+    return _current_ctx.set(ctx)
+
+def reset_context(token: Any):
+    """Reset the integration context to its previous state."""
+    _current_ctx.reset(token)
+
+class integration_context:
+    """
+    Context manager for setting/creating an IntegrationContext.
+    If no context exists, it creates one with a new run_id.
+    """
+    def __init__(
+        self,
+        integration: Optional[str] = None,
+        integration_pipeline: Optional[str] = None,
+        run_id: Optional[str] = None,
+        tenant_id: Optional[str] = None,
+        current_record_key: Optional[str] = None,
+
+        **kwargs
+
+    ):
+        self.overrides = {
+            "integration": integration,
+            "integration_pipeline": integration_pipeline,
+            "run_id": run_id,
+            "tenant_id": tenant_id,
+            **kwargs
+        }
+
+        self.token = None
+
+    def __enter__(self) -> IntegrationContext:
+        parent = current_context()
+        if parent:
+            # Reuse parent values if not overridden
+            merged = {
+                "integration": self.overrides.get("integration") or parent.integration,
+                "integration_pipeline": self.overrides.get("integration_pipeline") or parent.integration_pipeline,
+                "run_id": self.overrides.get("run_id") or parent.run_id,
+                "tenant_id": self.overrides.get("tenant_id") or parent.tenant_id,
+                "traceparent": parent.traceparent,
+                "tracestate": parent.tracestate,
+                "baggage": {**parent.baggage, **(self.overrides.get("baggage") or {})},
+                "tags": {**parent.tags, **(self.overrides.get("tags") or {})},
+                "current_record_key": self.overrides.get("current_record_key") or parent.current_record_key
+            }
+
+
+        else:
+            # New root context
+            merged = {
+                "integration": self.overrides.get("integration") or "unknown",
+                "integration_pipeline": self.overrides.get("integration_pipeline") or "unknown",
+                "run_id": self.overrides.get("run_id") or str(uuid.uuid4()),
+                "tenant_id": self.overrides.get("tenant_id") or "default",
+                "baggage": self.overrides.get("baggage") or {},
+                "tags": self.overrides.get("tags") or {},
+                "current_record_key": self.overrides.get("current_record_key")
+            }
+
+
+        
+        ctx = IntegrationContext(**merged)
+        self.token = set_context(ctx)
+        return ctx
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.token:
+            reset_context(self.token)