genblaze-core 0.1.0__py3-none-any.whl

genblaze_core/__init__.py

@@ -0,0 +1,110 @@
+"""genblaze-core — orchestration framework for media generation."""
+
+from genblaze_core._version import __version__
+
+_LAZY_IMPORTS: dict[str, tuple[str, str]] = {
+    # pipeline (primary entry point)
+    "Pipeline": ("genblaze_core.pipeline.pipeline", "Pipeline"),
+    "PipelineResult": ("genblaze_core.pipeline.result", "PipelineResult"),
+    "StepCompleteEvent": ("genblaze_core.pipeline.result", "StepCompleteEvent"),
+    # pipeline cache
+    "StepCache": ("genblaze_core.pipeline.cache", "StepCache"),
+    # pipeline templates
+    "PipelineTemplate": ("genblaze_core.pipeline.template", "PipelineTemplate"),
+    "StepTemplate": ("genblaze_core.pipeline.template", "StepTemplate"),
+    # moderation
+    "ModerationHook": ("genblaze_core.pipeline.moderation", "ModerationHook"),
+    "ModerationResult": ("genblaze_core.pipeline.moderation", "ModerationResult"),
+    # builders
+    "RunBuilder": ("genblaze_core.builders.run_builder", "RunBuilder"),
+    "StepBuilder": ("genblaze_core.builders.step_builder", "StepBuilder"),
+    # providers
+    "BaseProvider": ("genblaze_core.providers.base", "BaseProvider"),
+    "ProviderCapabilities": ("genblaze_core.providers.base", "ProviderCapabilities"),
+    "SubmitResult": ("genblaze_core.providers.base", "SubmitResult"),
+    "SyncProvider": ("genblaze_core.providers.base", "SyncProvider"),
+    "validate_asset_url": ("genblaze_core.providers.base", "validate_asset_url"),
+    "validate_chain_input_url": ("genblaze_core.providers.base", "validate_chain_input_url"),
+    "FFmpegCompositor": ("genblaze_core.providers.compositor", "FFmpegCompositor"),
+    "FFmpegTransform": ("genblaze_core.providers.transform", "FFmpegTransform"),
+    "ProgressEvent": ("genblaze_core.providers.progress", "ProgressEvent"),
+    # observability
+    "StreamEvent": ("genblaze_core.observability.events", "StreamEvent"),
+    "Tracer": ("genblaze_core.observability.tracer", "Tracer"),
+    "NoOpTracer": ("genblaze_core.observability.tracer", "NoOpTracer"),
+    "LoggingTracer": ("genblaze_core.observability.tracer", "LoggingTracer"),
+    "OTelTracer": ("genblaze_core.observability.tracer", "OTelTracer"),
+    "CompositeTracer": ("genblaze_core.observability.tracer", "CompositeTracer"),
+    "StructuredLogger": ("genblaze_core.observability.logger", "StructuredLogger"),
+    # agents
+    "AgentLoop": ("genblaze_core.agents.loop", "AgentLoop"),
+    "AgentContext": ("genblaze_core.agents.loop", "AgentContext"),
+    "AgentIteration": ("genblaze_core.agents.loop", "AgentIteration"),
+    "AgentResult": ("genblaze_core.agents.loop", "AgentResult"),
+    "Evaluator": ("genblaze_core.agents.evaluator", "Evaluator"),
+    "EvaluationResult": ("genblaze_core.agents.evaluator", "EvaluationResult"),
+    "CallableEvaluator": ("genblaze_core.agents.evaluator", "CallableEvaluator"),
+    "ThresholdEvaluator": ("genblaze_core.agents.evaluator", "ThresholdEvaluator"),
+    # models
+    "Manifest": ("genblaze_core.models.manifest", "Manifest"),
+    "Run": ("genblaze_core.models.run", "Run"),
+    "Step": ("genblaze_core.models.step", "Step"),
+    "Asset": ("genblaze_core.models.asset", "Asset"),
+    "AudioMetadata": ("genblaze_core.models.asset", "AudioMetadata"),
+    "Track": ("genblaze_core.models.asset", "Track"),
+    "VideoMetadata": ("genblaze_core.models.asset", "VideoMetadata"),
+    "WordTiming": ("genblaze_core.models.asset", "WordTiming"),
+    # enums
+    "Modality": ("genblaze_core.models.enums", "Modality"),
+    "StepType": ("genblaze_core.models.enums", "StepType"),
+    "RunStatus": ("genblaze_core.models.enums", "RunStatus"),
+    "StepStatus": ("genblaze_core.models.enums", "StepStatus"),
+    "PromptVisibility": ("genblaze_core.models.enums", "PromptVisibility"),
+    "ProviderErrorCode": ("genblaze_core.models.enums", "ProviderErrorCode"),
+    "EmbedPolicy": ("genblaze_core.models.policy", "EmbedPolicy"),
+    "PromptTemplate": ("genblaze_core.models.prompt_template", "PromptTemplate"),
+    # provider discovery
+    "discover_providers": ("genblaze_core.providers.registry", "discover_providers"),
+    # sinks
+    "BaseSink": ("genblaze_core.sinks.base", "BaseSink"),
+    "ParquetSink": ("genblaze_core.sinks.parquet", "ParquetSink"),
+    # storage
+    "StorageBackend": ("genblaze_core.storage.base", "StorageBackend"),
+    "ObjectStorageSink": ("genblaze_core.storage.sink", "ObjectStorageSink"),
+    "AssetTransfer": ("genblaze_core.storage.transfer", "AssetTransfer"),
+    "KeyStrategy": ("genblaze_core.storage.base", "KeyStrategy"),
+    "ObjectLockConfig": ("genblaze_core.storage.base", "ObjectLockConfig"),
+    # testing
+    "MockProvider": ("genblaze_core.testing", "MockProvider"),
+    "MockVideoProvider": ("genblaze_core.testing", "MockVideoProvider"),
+    "MockAudioProvider": ("genblaze_core.testing", "MockAudioProvider"),
+    "ProviderComplianceTests": ("genblaze_core.testing", "ProviderComplianceTests"),
+    # exceptions
+    "GenblazeError": ("genblaze_core.exceptions", "GenblazeError"),
+    "PipelineTimeoutError": ("genblaze_core.exceptions", "PipelineTimeoutError"),
+    "EmbeddingError": ("genblaze_core.exceptions", "EmbeddingError"),
+    "StorageError": ("genblaze_core.exceptions", "StorageError"),
+    "ProviderError": ("genblaze_core.exceptions", "ProviderError"),
+    "ManifestError": ("genblaze_core.exceptions", "ManifestError"),
+    "SinkError": ("genblaze_core.exceptions", "SinkError"),
+    "WebhookError": ("genblaze_core.exceptions", "WebhookError"),
+    # webhooks
+    "WebhookNotifier": ("genblaze_core.webhooks.notifier", "WebhookNotifier"),
+    "WebhookConfig": ("genblaze_core.webhooks.notifier", "WebhookConfig"),
+    "WebhookEvent": ("genblaze_core.webhooks.notifier", "WebhookEvent"),
+    "WebhookSink": ("genblaze_core.webhooks.sink", "WebhookSink"),
+}
+
+__all__ = [*_LAZY_IMPORTS.keys(), "__version__"]
+
+
+def __getattr__(name: str):
+    if name in _LAZY_IMPORTS:
+        module_path, attr = _LAZY_IMPORTS[name]
+        import importlib
+
+        mod = importlib.import_module(module_path)
+        val = getattr(mod, attr)
+        globals()[name] = val
+        return val
+    raise AttributeError(f"module 'genblaze_core' has no attribute {name!r}")

genblaze_core/_utils.py

@@ -0,0 +1,158 @@
+"""Internal utilities."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import ipaddress
+import random
+import re
+import socket
+import tempfile
+import uuid
+from collections.abc import Coroutine
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse as _urlparse
+
+
+def new_id() -> str:
+    """Generate a new UUID4 string."""
+    return str(uuid.uuid4())
+
+
+def utc_now() -> datetime:
+    """Return current UTC datetime."""
+    return datetime.now(UTC)
+
+
+def compute_sha256(data: bytes) -> str:
+    """Compute SHA-256 hex digest of raw bytes."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def _run_async(coro: Coroutine) -> Any:
+    """Run an async coroutine from sync code safely.
+
+    If there's already a running event loop (e.g. inside Jupyter or an async provider
+    called from BaseProvider.invoke), runs in a new thread to avoid RuntimeError.
+    Otherwise uses asyncio.run().
+    """
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = None
+
+    if loop is not None and loop.is_running():
+        # Already inside an event loop — run in a separate thread
+        with ThreadPoolExecutor(max_workers=1) as pool:
+            return pool.submit(asyncio.run, coro).result()
+    return asyncio.run(coro)
+
+
+# ---------------------------------------------------------------------------
+# SSRF protection — shared by storage/transfer.py and webhooks/notifier.py
+# ---------------------------------------------------------------------------
+
+# Private/reserved IP ranges blocked to prevent SSRF
+BLOCKED_NETWORKS = [
+    ipaddress.ip_network("0.0.0.0/8"),  # "This host" — resolves to loopback on Linux
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("169.254.0.0/16"),  # Link-local / IMDS
+    ipaddress.ip_network("100.64.0.0/10"),  # Carrier-grade NAT
+    ipaddress.ip_network("::1/128"),  # IPv6 loopback
+    ipaddress.ip_network("fc00::/7"),  # IPv6 unique local
+    ipaddress.ip_network("fe80::/10"),  # IPv6 link-local
+]
+
+
+# Allowed parent directories for file:// inputs (shared by storage + ffmpeg providers)
+# Deduplicate: /tmp and gettempdir() may resolve to the same or different paths
+ALLOWED_FILE_ROOTS: tuple[Path, ...] = tuple(
+    {Path(tempfile.gettempdir()).resolve(), Path("/tmp").resolve()}  # noqa: S108
+)
+
+
+def check_ssrf(url: str, *, exc_type: type[Exception] = ValueError) -> None:
+    """Reject non-HTTPS URLs and hostnames resolving to private IP ranges.
+
+    Shared SSRF guard used by storage transfers and webhook dispatch.
+    Callers pass their domain-specific exception type via ``exc_type``.
+    """
+    parsed = _urlparse(url)
+    if parsed.scheme not in ("https",):
+        raise exc_type(f"Only HTTPS URLs are allowed, got: {parsed.scheme}://")
+
+    host = parsed.hostname or ""
+    if host.lower() == "localhost":
+        raise exc_type(f"Private/loopback URLs are not allowed: {host}")
+
+    try:
+        addrinfos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
+    except socket.gaierror as exc:
+        raise exc_type(f"Cannot resolve hostname: {host}") from exc
+
+    for _, _, _, _, sockaddr in addrinfos:
+        try:
+            ip = ipaddress.ip_address(str(sockaddr[0]))
+        except ValueError:
+            continue
+        if any(ip in net for net in BLOCKED_NETWORKS):
+            raise exc_type(f"Private/loopback URLs are not allowed: {host}")
+
+
+# ---------------------------------------------------------------------------
+# Manifest size cap — bounds the JSON payload accepted from disk/media
+# ---------------------------------------------------------------------------
+# Real manifests are O(KB). 16 MiB is generous and bounds OOM blast from
+# malicious media or sidecars that declare absurd payload sizes.
+MAX_MANIFEST_BYTES = 16 * 1024 * 1024
+
+
+# ---------------------------------------------------------------------------
+# Credential pattern detection — used by error sanitization (providers/base.py)
+# AND by Pipeline.step build-time rejection of secret-shaped params values.
+# Centralized here so both call sites share one regex of record.
+# ---------------------------------------------------------------------------
+_SECRET_PATTERNS = re.compile(
+    r"(r8_[A-Za-z0-9]{20,})"  # Replicate tokens
+    r"|(sk-ant-[A-Za-z0-9\-]{20,})"  # Anthropic API keys (before generic sk-)
+    r"|(sk-[A-Za-z0-9]{20,})"  # OpenAI-style keys
+    r"|(AIza[A-Za-z0-9_\-]{30,})"  # Google API keys
+    r"|(AKIA[A-Z0-9]{16})"  # AWS access key IDs
+    r"|(Bearer\s+[A-Za-z0-9._\-]{20,})"  # Bearer tokens
+    r"|(Token\s+[A-Za-z0-9._\-]{20,})"  # Token auth headers
+    r"|(\bapi[_-]key[=:]\s*[A-Za-z0-9._\-]{20,})",  # api_key=... / api-key:...
+    re.IGNORECASE,
+)
+
+
+def jittered_backoff(attempt: int) -> float:
+    """Compute exponential backoff with jitter to avoid thundering herd.
+
+    Starts at 1s, doubles per attempt, capped at 30s base with up to 25% jitter.
+    """
+    base = min(2**attempt, 30)
+    return base * (1 + random.uniform(0, 0.25))  # noqa: S311 — jitter, not crypto
+
+
+def probe_audio_duration(path: str | Any) -> float | None:
+    """Try to read audio duration from a file using mutagen (optional dep).
+
+    Returns duration in seconds, or None if mutagen is not installed or
+    the file format is not recognized.
+    """
+    try:
+        from mutagen import File as MutagenFile
+
+        audio = MutagenFile(str(path))
+        if audio is not None and audio.info is not None:
+            return audio.info.length
+    except Exception:  # noqa: S110 — mutagen is optional, fail gracefully
+        pass
+    return None

genblaze_core/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

genblaze_core/agents/__init__.py

@@ -0,0 +1,20 @@
+"""Agent / reasoning layer — evaluate outputs, refine prompts, retry."""
+
+from genblaze_core.agents.evaluator import (
+    CallableEvaluator,
+    EvaluationResult,
+    Evaluator,
+    ThresholdEvaluator,
+)
+from genblaze_core.agents.loop import AgentContext, AgentIteration, AgentLoop, AgentResult
+
+__all__ = [
+    "AgentContext",
+    "AgentIteration",
+    "AgentLoop",
+    "AgentResult",
+    "CallableEvaluator",
+    "EvaluationResult",
+    "Evaluator",
+    "ThresholdEvaluator",
+]

genblaze_core/agents/evaluator.py

@@ -0,0 +1,100 @@
+"""Evaluator — pluggable quality-gate for pipeline outputs.
+
+An ``Evaluator`` inspects a :class:`PipelineResult` and returns an
+:class:`EvaluationResult` describing whether the output meets quality
+requirements. Evaluators can be simple threshold checks, LLM judges,
+vision-model classifiers, or composed chains.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from genblaze_core.pipeline.result import PipelineResult
+
+
+@dataclass
+class EvaluationResult:
+    """Outcome of evaluating a pipeline result.
+
+    Attributes:
+        passed: Whether the output meets quality requirements.
+        score: Optional numeric score (typically 0.0–1.0). None if not applicable.
+        feedback: Human-readable explanation to feed back into a refinement step.
+        metadata: Arbitrary structured data (e.g. per-dimension scores, detected issues).
+    """
+
+    passed: bool
+    score: float | None = None
+    feedback: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class Evaluator(ABC):
+    """Abstract evaluator. Subclasses implement at least ``evaluate``."""
+
+    @abstractmethod
+    def evaluate(self, result: PipelineResult) -> EvaluationResult:
+        """Judge a completed pipeline result. Must return an EvaluationResult."""
+
+    async def aevaluate(self, result: PipelineResult) -> EvaluationResult:
+        """Async variant. Default runs ``evaluate`` in a thread."""
+        return await asyncio.to_thread(self.evaluate, result)
+
+
+class CallableEvaluator(Evaluator):
+    """Wraps a plain callable as an :class:`Evaluator`.
+
+    The callable receives the :class:`PipelineResult` and returns either
+    an :class:`EvaluationResult` or a bool (passed/not).
+    """
+
+    def __init__(
+        self,
+        fn: Callable[[PipelineResult], EvaluationResult | bool],
+    ) -> None:
+        self._fn = fn
+
+    def evaluate(self, result: PipelineResult) -> EvaluationResult:
+        out = self._fn(result)
+        if isinstance(out, EvaluationResult):
+            return out
+        return EvaluationResult(passed=bool(out))
+
+
+class ThresholdEvaluator(Evaluator):
+    """Pass/fail based on a numeric score crossing a threshold.
+
+    ``score_fn`` receives the :class:`PipelineResult` and returns a float.
+    Useful for wrapping a vision or text quality model that emits a score.
+    """
+
+    def __init__(
+        self,
+        score_fn: Callable[[PipelineResult], float],
+        threshold: float,
+        *,
+        higher_is_better: bool = True,
+        feedback_fn: Callable[[PipelineResult, float], str] | None = None,
+    ) -> None:
+        self._score_fn = score_fn
+        self._threshold = threshold
+        self._higher_is_better = higher_is_better
+        self._feedback_fn = feedback_fn
+
+    def evaluate(self, result: PipelineResult) -> EvaluationResult:
+        score = float(self._score_fn(result))
+        passed = score >= self._threshold if self._higher_is_better else score <= self._threshold
+        feedback = self._feedback_fn(result, score) if self._feedback_fn else None
+        return EvaluationResult(
+            passed=passed,
+            score=score,
+            feedback=feedback,
+            metadata={"threshold": self._threshold, "higher_is_better": self._higher_is_better},
+        )