keboola-cli 0.63.4__py3-none-any.whl

keboola_agent_cli/stream_client.py

@@ -0,0 +1,217 @@
+"""Keboola Stream (Data Streams) API client with retry, timeouts, token masking.
+
+This module talks to the Keboola **Stream control-plane API** for managing
+Data Streams sources and sinks (list / create / detail / delete). The base URL
+is derived from the Storage API stack URL by replacing 'connection.' with
+'stream.' in the hostname (the same scheme used for 'ai.'/'queue.'), and the
+request is authenticated with the per-project Storage API token
+(``X-StorageApi-Token``) -- no manage token is involved.
+
+Important: the OTLP *ingestion* endpoint (``stream-in.<region>/otlp/...``) is a
+separate data-plane host and is NOT derived here -- it is returned by the API in
+the source's ``otlp.url`` field. This client only speaks to the control plane.
+
+Source-create and source-delete are asynchronous: the API returns a ``Task``
+(202 Accepted); :meth:`wait_for_task` polls ``GET /v1/tasks/{taskId}`` until the
+task reports ``isFinished``.
+
+Inherits shared retry/error logic from :class:`BaseHttpClient`.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+from urllib.parse import quote, urlparse
+
+from .constants import STREAM_API_TIMEOUT, STREAM_TASK_POLL_INTERVAL, STREAM_TASK_TIMEOUT
+from .errors import ErrorCode, KeboolaApiError
+from .http_base import BaseHttpClient
+
+logger = logging.getLogger(__name__)
+
+
+class StreamClient(BaseHttpClient):
+    """HTTP client for the Keboola Stream (Data Streams) control-plane API.
+
+    Provides source CRUD, sink listing, and async-task polling, with the
+    retry/backoff (429/5xx), timeouts, and token masking inherited from
+    :class:`BaseHttpClient`.
+    """
+
+    def __init__(self, stack_url: str, token: str) -> None:
+        self._stack_url = stack_url.rstrip("/")
+        stream_base_url = self._derive_service_url(self._stack_url, "stream")
+        headers = {
+            "X-StorageApi-Token": token,
+            "Content-Type": "application/json",
+        }
+        super().__init__(
+            base_url=stream_base_url,
+            token=token,
+            headers=headers,
+            timeout=STREAM_API_TIMEOUT,
+        )
+
+    def __enter__(self) -> StreamClient:
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        self.close()
+
+    # ------------------------------------------------------------------
+    # Sources
+    # ------------------------------------------------------------------
+
+    def list_sources(self, branch_id: str) -> dict[str, Any]:
+        """List sources in a branch (``GET /v1/branches/{branch}/sources``)."""
+        path = f"/v1/branches/{quote(branch_id, safe='')}/sources"
+        response = self._do_request("GET", path)
+        return response.json()
+
+    def get_source(self, branch_id: str, source_id: str) -> dict[str, Any]:
+        """Fetch one source (``GET /v1/branches/{branch}/sources/{id}``)."""
+        path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}"
+        response = self._do_request("GET", path)
+        return response.json()
+
+    def create_source(
+        self,
+        branch_id: str,
+        name: str,
+        source_type: str,
+        source_id: str | None = None,
+        description: str | None = None,
+    ) -> dict[str, Any]:
+        """Create a source. Returns the async ``Task`` (poll with wait_for_task)."""
+        path = f"/v1/branches/{quote(branch_id, safe='')}/sources"
+        payload: dict[str, Any] = {"name": name, "type": source_type}
+        if source_id is not None:
+            payload["sourceId"] = source_id
+        if description is not None:
+            payload["description"] = description
+        response = self._do_request("POST", path, json=payload)
+        return response.json()
+
+    def delete_source(self, branch_id: str, source_id: str) -> dict[str, Any]:
+        """Delete a source. Returns the async ``Task`` (poll with wait_for_task)."""
+        path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}"
+        response = self._do_request("DELETE", path)
+        return response.json()
+
+    # ------------------------------------------------------------------
+    # Sinks
+    # ------------------------------------------------------------------
+
+    def list_sinks(self, branch_id: str, source_id: str) -> dict[str, Any]:
+        """List a source's sinks (``GET .../sources/{id}/sinks``)."""
+        path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}/sinks"
+        response = self._do_request("GET", path)
+        return response.json()
+
+    def create_sink(
+        self,
+        branch_id: str,
+        source_id: str,
+        *,
+        name: str,
+        table_id: str,
+        columns: list[dict[str, Any]],
+        allowed_signals: list[str] | None = None,
+        sink_id: str | None = None,
+    ) -> dict[str, Any]:
+        """Create a table sink on a source. Returns the async ``Task``.
+
+        ``columns`` is the table mapping column list (see the Stream API
+        ``TableColumn`` schema). ``allowed_signals`` restricts which OTLP signals
+        route to this sink (logs/metrics/traces); omit to accept all.
+        """
+        path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}/sinks"
+        payload: dict[str, Any] = {
+            "type": "table",
+            "name": name,
+            "table": {
+                "type": "keboola",
+                "tableId": table_id,
+                "mapping": {"columns": columns},
+            },
+        }
+        if allowed_signals is not None:
+            payload["allowedSignals"] = allowed_signals
+        if sink_id is not None:
+            payload["sinkId"] = sink_id
+        response = self._do_request("POST", path, json=payload)
+        return response.json()
+
+    # ------------------------------------------------------------------
+    # Tasks (async create/delete)
+    # ------------------------------------------------------------------
+
+    def get_task(self, task_id: str) -> dict[str, Any]:
+        """Fetch a task by id (``GET /v1/tasks/{taskId}``)."""
+        path = f"/v1/tasks/{quote(task_id, safe='')}"
+        response = self._do_request("GET", path)
+        return response.json()
+
+    def wait_for_task(
+        self,
+        task: dict[str, Any],
+        timeout: float = STREAM_TASK_TIMEOUT,
+        poll_interval: float = STREAM_TASK_POLL_INTERVAL,
+    ) -> dict[str, Any]:
+        """Poll a ``Task`` to completion and return the finished task.
+
+        Accepts the Task dict returned by :meth:`create_source` /
+        :meth:`delete_source`. Polls its canonical poll URL (the task's ``url``
+        reduced to a path, falling back to ``/v1/tasks/{taskId}``) until
+        ``isFinished`` is true, then raises :class:`KeboolaApiError` if the task
+        failed.
+        """
+        if task.get("isFinished"):
+            return self._check_task_result(task)
+
+        poll_path = self._task_poll_path(task)
+        deadline = time.monotonic() + timeout
+        latest = task
+        while time.monotonic() < deadline:
+            time.sleep(poll_interval)
+            response = self._do_request("GET", poll_path)
+            latest = response.json()
+            if latest.get("isFinished"):
+                return self._check_task_result(latest)
+
+        raise KeboolaApiError(
+            message=(
+                f"Stream task '{latest.get('taskId', '?')}' did not finish within "
+                f"{timeout:.0f}s (last status: {latest.get('status', 'unknown')})"
+            ),
+            error_code=ErrorCode.TIMEOUT,
+            retryable=True,
+        )
+
+    def _task_poll_path(self, task: dict[str, Any]) -> str:
+        """Resolve the path to poll for ``task``.
+
+        Prefers the task's ``url`` field reduced to a path relative to the
+        Stream base; falls back to ``/v1/tasks/{taskId}``.
+        """
+        url = task.get("url")
+        if isinstance(url, str) and url:
+            parsed = urlparse(url)
+            if parsed.path:
+                return parsed.path
+        task_id = task.get("taskId", "")
+        return f"/v1/tasks/{quote(str(task_id), safe='')}"
+
+    @staticmethod
+    def _check_task_result(task: dict[str, Any]) -> dict[str, Any]:
+        """Return a finished task, raising if it ended in error."""
+        error = task.get("error")
+        status = task.get("status")
+        if error or status == "error":
+            raise KeboolaApiError(
+                message=f"Stream task failed: {error or status}",
+                error_code=ErrorCode.API_ERROR,
+            )
+        return task

keboola_agent_cli/sync/__init__.py

@@ -0,0 +1 @@
+"""Sync package: filesystem serialization for Keboola configurations."""

keboola_agent_cli/sync/branch_mapping.py

@@ -0,0 +1,174 @@
+"""Branch mapping for git-to-Keboola branch mapping.
+
+Manages .keboola/branch-mapping.json which maps git branch names
+to Keboola development branch IDs.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from ..constants import BRANCH_MAPPING_FILENAME, KEBOOLA_DIR_NAME
+from ..errors import ConfigError
+
+
+def _coerce_keboola_id(raw: Any) -> int | None:
+    """Coerce a raw ``id`` field from JSON to ``int | None``.
+
+    Older kbagent versions (<= 0.30.3) wrote branch IDs as strings (e.g.
+    ``"99999"``) due to issue #267. ``None`` means production. Empty
+    string is also treated as production for legacy tolerance.
+
+    Raises ``ValueError`` with a descriptive message if *raw* is neither
+    None, empty, nor parseable as an int (e.g. a hand-edited
+    ``branch-mapping.json`` containing ``"id": "not-a-number"``). The
+    caller (typically ``BranchMapping.from_dict``) should let this
+    bubble up to ``load_branch_mapping`` which converts it to a
+    ConfigError surface (issue #269 sec-20).
+    """
+    if raw is None or raw == "":
+        return None
+    try:
+        return int(raw)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(
+            f"Invalid branch ID in branch-mapping.json: {raw!r}. "
+            f"Expected null or an integer; got {type(raw).__name__}."
+        ) from exc
+
+
+class BranchMappingEntry:
+    """A single git branch -> Keboola branch mapping."""
+
+    def __init__(self, keboola_id: int | None, name: str):
+        self.keboola_id = keboola_id  # None = production
+        self.name = name
+
+    def is_production(self) -> bool:
+        return self.keboola_id is None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"id": self.keboola_id, "name": self.name}
+
+
+class BranchMapping:
+    """Manages git-to-Keboola branch mappings."""
+
+    def __init__(self) -> None:
+        self.version: int = 1
+        self.mappings: dict[str, BranchMappingEntry] = {}
+
+    def get(self, git_branch: str) -> BranchMappingEntry | None:
+        return self.mappings.get(git_branch)
+
+    def set(self, git_branch: str, keboola_id: int | None, name: str) -> None:
+        self.mappings[git_branch] = BranchMappingEntry(keboola_id, name)
+
+    def remove(self, git_branch: str) -> bool:
+        if git_branch in self.mappings:
+            del self.mappings[git_branch]
+            return True
+        return False
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "version": self.version,
+            "mappings": {k: v.to_dict() for k, v in self.mappings.items()},
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> BranchMapping:
+        mapping = cls()
+        mapping.version = data.get("version", 1)
+        for git_branch, entry in data.get("mappings", {}).items():
+            mapping.mappings[git_branch] = BranchMappingEntry(
+                keboola_id=_coerce_keboola_id(entry.get("id")),
+                name=entry.get("name", ""),
+            )
+        return mapping
+
+
+def load_branch_mapping(project_root: Path) -> BranchMapping:
+    """Load .keboola/branch-mapping.json.
+
+    Raises:
+        FileNotFoundError: If the mapping file does not exist.
+        ConfigError: If the JSON cannot be parsed or contains a malformed
+            branch ID. The descriptive message names the offending file
+            so the user can find and fix it. Surfacing this as ConfigError
+            (rather than raw ValueError) lets CLI commands catch it via
+            the existing ``except ConfigError`` handler and emit a clean
+            JSON error envelope with exit code 5 instead of a Python
+            traceback (issue #269 sec-20 + smoke-test follow-up).
+    """
+    path = project_root / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME
+    if not path.exists():
+        raise FileNotFoundError(f"Branch mapping not found at {path}")
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+        return BranchMapping.from_dict(data)
+    except ValueError as exc:
+        # _coerce_keboola_id and json.loads both raise ValueError on
+        # malformed input; wrap with path context and convert to
+        # ConfigError so the CLI surfaces a clean error envelope.
+        raise ConfigError(f"Failed to parse {path}: {exc}") from exc
+
+
+def save_branch_mapping(project_root: Path, mapping: BranchMapping) -> None:
+    """Save branch mapping to .keboola/branch-mapping.json."""
+    path = project_root / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(mapping.to_dict(), indent=4, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
+
+
+def find_sync_workspace(start: Path | None = None) -> Path | None:
+    """Locate the nearest enclosing sync workspace.
+
+    Walks up from *start* (or the current working directory) and returns
+    the first directory that contains a ``.keboola/branch-mapping.json``
+    file, or ``None`` if none is found before the filesystem root.
+    """
+    cursor = (start or Path.cwd()).resolve()
+    for candidate in [cursor, *cursor.parents]:
+        if (candidate / KEBOOLA_DIR_NAME / BRANCH_MAPPING_FILENAME).exists():
+            return candidate
+    return None
+
+
+def cleanup_branch_id_from_mapping(branch_id: int) -> dict[str, Any] | None:
+    """Remove every git-branch entry that maps to *branch_id* from the
+    nearest enclosing sync workspace, if one exists.
+
+    Designed to be a best-effort cleanup hook for ``branch delete`` and
+    ``branch merge``: locates ``.keboola/branch-mapping.json`` via
+    :func:`find_sync_workspace`, removes any entries whose ``keboola_id``
+    equals *branch_id*, and persists the change. Returns a dict
+    describing what was unlinked, or ``None`` if no workspace was found
+    or no entry referenced the branch (no-op).
+    """
+    project_root = find_sync_workspace()
+    if project_root is None:
+        return None
+    try:
+        mapping = load_branch_mapping(project_root)
+    except (FileNotFoundError, ConfigError, ValueError):
+        # ValueError preserved for backward compat with callers that may
+        # still trigger it via custom paths; ConfigError is the new shape
+        # raised by load_branch_mapping itself.
+        return None
+
+    removed: list[str] = []
+    for git_branch, entry in list(mapping.mappings.items()):
+        if entry.keboola_id == branch_id:
+            mapping.remove(git_branch)
+            removed.append(git_branch)
+
+    if not removed:
+        return None
+    save_branch_mapping(project_root, mapping)
+    return {"project_root": str(project_root), "git_branches_unlinked": removed}

keboola_agent_cli/sync/clone.py

@@ -0,0 +1,211 @@
+"""Pure helpers for the ``sync clone`` composite (#426).
+
+A clone copies a *reference* synced project tree into a fresh target, applies
+declarative overrides, and re-points the manifest so a subsequent ``sync push``
+CREATEs everything fresh in the target project. Cloning into a **fresh** target
+project needs no id surgery: the reference's config ids do not exist in the
+target remote, so the diff classifies every config as ``added`` and the push
+assigns new ULIDs -- and because ``created_id_map`` is keyed by the reference id
+(the manifest entry's id before writeback), the Phase-C variable links and the
+Phase-D flow task ``configId``s remap reference->ULID automatically.
+
+These functions are deliberately side-effecting but **pure of API calls**: they
+only touch the on-disk tree + the in-memory manifest, so they are unit-testable
+without a client. ``SyncService.clone_project`` orchestrates them and drives the
+diff/push.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from ..constants import CONFIG_FILENAME
+
+# Duplicated here (not imported from sync_service) to avoid a circular import --
+# clone.py is imported *by* sync_service.
+VARIABLES_COMPONENT_ID = "keboola.variables"
+
+# Default branch dir name when a config's branch is not listed in the manifest
+# (non-git-branching projects pull a single "main/" tree).
+_DEFAULT_BRANCH_DIR = "main"
+
+
+def branch_path_map(manifest: Any) -> dict[int, str]:
+    """Map ``branch_id -> on-disk branch dir name`` from the manifest branches."""
+    return {b.id: b.path for b in manifest.branches}
+
+
+def _read_yaml(path: Path) -> dict[str, Any]:
+    loaded = yaml.safe_load(path.read_text(encoding="utf-8"))
+    return loaded if isinstance(loaded, dict) else {}
+
+
+def _write_yaml(path: Path, data: dict[str, Any]) -> None:
+    path.write_text(
+        yaml.dump(data, sort_keys=False, allow_unicode=True, default_flow_style=False),
+        encoding="utf-8",
+    )
+
+
+def copy_reference_tree(source_dir: Path, target_dir: Path) -> None:
+    """Copy a reference project tree to a fresh target dir (incl. code files).
+
+    Raises:
+        FileExistsError: if ``target_dir`` already exists (the caller decides
+            whether an existing target means "already cloned, just push").
+    """
+    if target_dir.exists():
+        raise FileExistsError(f"Target directory already exists: {target_dir}")
+    shutil.copytree(source_dir, target_dir)
+
+
+def repoint_manifest_project(manifest: Any, *, project_id: int, api_host: str) -> None:
+    """Re-point the manifest's project block at the clone's target project."""
+    manifest.project.id = project_id
+    manifest.project.api_host = api_host
+
+
+def _config_dir(target_dir: Path, branch_map: dict[int, str], branch_id: int, path: str) -> Path:
+    return target_dir / branch_map.get(branch_id, _DEFAULT_BRANCH_DIR) / path
+
+
+def _remap_bucket_in_table_id(table_id: Any, bucket_map: dict[str, str]) -> Any:
+    """Rewrite the bucket prefix of a storage table id via ``bucket_map``.
+
+    A table id is ``<stage>.<bucket>.<table>`` whose bucket id is
+    ``<stage>.<bucket>``. A bucket-level reference (``in.c-foo``) is mapped
+    whole; a table reference (``in.c-foo.users``) has only its bucket prefix
+    swapped. Unmatched / non-string values pass through unchanged.
+    """
+    if not isinstance(table_id, str):
+        return table_id
+    for old, new in bucket_map.items():
+        if table_id == old:
+            return new
+        if table_id.startswith(old + "."):
+            return new + table_id[len(old) :]
+    return table_id
+
+
+def _rewrite_buckets_in_config(config_dir: Path, bucket_map: dict[str, str]) -> int:
+    config_file = config_dir / CONFIG_FILENAME
+    if not config_file.exists():
+        return 0
+    data = _read_yaml(config_file)
+    rewrites = 0
+    for section, key in (("input", "source"), ("output", "destination")):
+        mapping = data.get(section)
+        if not isinstance(mapping, dict):
+            continue
+        for table in mapping.get("tables") or []:
+            if not isinstance(table, dict) or key not in table:
+                continue
+            new_value = _remap_bucket_in_table_id(table[key], bucket_map)
+            if new_value != table[key]:
+                table[key] = new_value
+                rewrites += 1
+    if rewrites:
+        _write_yaml(config_file, data)
+    return rewrites
+
+
+def apply_bucket_map(target_dir: Path, manifest: Any, bucket_map: dict[str, str]) -> int:
+    """Rewrite bucket ids in every config's storage input/output mappings.
+
+    Returns the number of table references rewritten. A no-op when
+    ``bucket_map`` is empty.
+    """
+    if not bucket_map:
+        return 0
+    branch_map = branch_path_map(manifest)
+    rewrites = 0
+    for cfg in manifest.configurations:
+        rewrites += _rewrite_buckets_in_config(
+            _config_dir(target_dir, branch_map, cfg.branch_id, cfg.path), bucket_map
+        )
+        for row in cfg.rows:
+            rewrites += _rewrite_buckets_in_config(
+                _config_dir(target_dir, branch_map, cfg.branch_id, row.path), bucket_map
+            )
+    return rewrites
+
+
+def _override_values_in_row(row_dir: Path, variable_values: dict[str, str]) -> int:
+    config_file = row_dir / CONFIG_FILENAME
+    if not config_file.exists():
+        return 0
+    data = _read_yaml(config_file)
+    # keboola.variables rows hoist their ``values`` array to the YAML top level.
+    values = data.get("values")
+    if not isinstance(values, list):
+        return 0
+    overridden = 0
+    for item in values:
+        if isinstance(item, dict) and item.get("name") in variable_values:
+            new_value = str(variable_values[item["name"]])
+            if item.get("value") != new_value:
+                item["value"] = new_value
+                overridden += 1
+    if overridden:
+        _write_yaml(config_file, data)
+    return overridden
+
+
+def apply_variable_values(target_dir: Path, manifest: Any, variable_values: dict[str, str]) -> int:
+    """Override ``keboola.variables`` row values by variable name.
+
+    Returns the number of values overridden. A no-op when ``variable_values``
+    is empty. Values are written as strings (the Keboola variables contract).
+    """
+    if not variable_values:
+        return 0
+    branch_map = branch_path_map(manifest)
+    overridden = 0
+    for cfg in manifest.configurations:
+        if cfg.component_id != VARIABLES_COMPONENT_ID:
+            continue
+        for row in cfg.rows:
+            overridden += _override_values_in_row(
+                _config_dir(target_dir, branch_map, cfg.branch_id, row.path), variable_values
+            )
+    return overridden
+
+
+def apply_instance_rename(target_dir: Path, manifest: Any, renames: dict[str, str]) -> int:
+    """Rename config-path prefixes on disk and in the manifest.
+
+    ``renames`` maps an old path prefix to a new one (e.g.
+    ``{"extractor/keboola.ex-http/Acme": "extractor/keboola.ex-http/Globex"}``).
+    For every config (and its rows) whose manifest ``path`` equals the prefix or
+    starts with ``prefix + "/"``, the on-disk subtree is moved once and the
+    manifest paths are rewritten. Returns the number of configs whose path
+    changed. A no-op when ``renames`` is empty.
+    """
+    if not renames:
+        return 0
+    branch_map = branch_path_map(manifest)
+    moved: set[tuple[str, str]] = set()
+    renamed = 0
+    for old, new in renames.items():
+        for cfg in manifest.configurations:
+            if not (cfg.path == old or cfg.path.startswith(old + "/")):
+                continue
+            branch_dir = branch_map.get(cfg.branch_id, _DEFAULT_BRANCH_DIR)
+            move_key = (branch_dir, old)
+            if move_key not in moved:
+                src = target_dir / branch_dir / old
+                dst = target_dir / branch_dir / new
+                if src.exists():
+                    dst.parent.mkdir(parents=True, exist_ok=True)
+                    shutil.move(str(src), str(dst))
+                moved.add(move_key)
+            cfg.path = new + cfg.path[len(old) :]
+            for row in cfg.rows:
+                if row.path == old or row.path.startswith(old + "/"):
+                    row.path = new + row.path[len(old) :]
+            renamed += 1
+    return renamed