keboola-cli 0.63.4__py3-none-any.whl

keboola_agent_cli/sync/manifest.py

@@ -0,0 +1,162 @@
+"""Pydantic v2 models for .keboola/manifest.json (v2, camelCase via aliases).
+
+Mirrors the manifest format used by the Keboola Go CLI so that
+directories written by kbagent are compatible with `kbc` tooling.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..constants import KEBOOLA_DIR_NAME, MANIFEST_FILENAME, MANIFEST_VERSION
+
+# ---------------------------------------------------------------------------
+# Sub-models
+# ---------------------------------------------------------------------------
+
+
+class ManifestProject(BaseModel):
+    """Project identification inside the manifest."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    id: int
+    api_host: str = Field(alias="apiHost")
+
+
+class ManifestGitBranching(BaseModel):
+    """Git branching settings."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    enabled: bool = False
+    default_branch: str = Field(default="main", alias="defaultBranch")
+
+
+class ManifestNaming(BaseModel):
+    """Naming templates that control the filesystem layout."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    branch: str = "{branch_name}"
+    config: str = "{component_type}/{component_id}/{config_name}"
+    config_row: str = Field(default="rows/{config_row_name}", alias="configRow")
+    scheduler_config: str = Field(default="schedules/{config_name}", alias="schedulerConfig")
+    shared_code_config: str = Field(
+        default="_shared/{target_component_id}", alias="sharedCodeConfig"
+    )
+    shared_code_config_row: str = Field(
+        default="codes/{config_row_name}", alias="sharedCodeConfigRow"
+    )
+    variables_config: str = Field(default="variables", alias="variablesConfig")
+    variables_values_row: str = Field(
+        default="values/{config_row_name}", alias="variablesValuesRow"
+    )
+    data_app_config: str = Field(default="app/{component_id}/{config_name}", alias="dataAppConfig")
+
+
+class ManifestBranch(BaseModel):
+    """A branch entry in the manifest."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    id: int
+    path: str
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class ManifestConfigRow(BaseModel):
+    """A single configuration row reference.
+
+    ``metadata`` mirrors :class:`ManifestConfiguration.metadata` and stores
+    pull-time hashes (``pull_hash``, ``pull_config_hash``) so the row-level
+    diff can distinguish local-changed, remote-changed, and conflict states.
+    Older manifests that lack the field load with an empty dict and upgrade
+    on the next successful pull.
+    """
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    id: str
+    path: str
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class ManifestConfiguration(BaseModel):
+    """A single configuration reference inside the manifest."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    branch_id: int = Field(alias="branchId")
+    component_id: str = Field(alias="componentId")
+    id: str
+    path: str
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    rows: list[ManifestConfigRow] = Field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Root manifest
+# ---------------------------------------------------------------------------
+
+
+class Manifest(BaseModel):
+    """Root model for .keboola/manifest.json (schema version 3)."""
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+    version: int = MANIFEST_VERSION
+    project: ManifestProject
+    allow_target_env: bool = Field(default=True, alias="allowTargetEnv")
+    git_branching: ManifestGitBranching = Field(
+        default_factory=ManifestGitBranching, alias="gitBranching"
+    )
+    sort_by: str = Field(default="id", alias="sortBy")
+    naming: ManifestNaming
+    allowed_branches: list[str] = Field(default_factory=list, alias="allowedBranches")
+    ignored_components: list[str] = Field(default_factory=list, alias="ignoredComponents")
+    branches: list[ManifestBranch] = Field(default_factory=list)
+    configurations: list[ManifestConfiguration] = Field(default_factory=list)
+
+
+# ---------------------------------------------------------------------------
+# Load / Save helpers
+# ---------------------------------------------------------------------------
+
+
+def load_manifest(project_root: Path) -> Manifest:
+    """Load .keboola/manifest.json from *project_root*.
+
+    Raises:
+        FileNotFoundError: if the manifest file does not exist.
+        ValueError: if the JSON cannot be parsed into a valid Manifest.
+    """
+    manifest_path = project_root / KEBOOLA_DIR_NAME / MANIFEST_FILENAME
+    if not manifest_path.exists():
+        raise FileNotFoundError(
+            f"Manifest not found at {manifest_path}. Is this a Keboola project directory?"
+        )
+
+    raw = json.loads(manifest_path.read_text(encoding="utf-8"))
+    return Manifest.model_validate(raw)
+
+
+def save_manifest(project_root: Path, manifest: Manifest) -> None:
+    """Save *manifest* to .keboola/manifest.json.
+
+    Uses ``by_alias=True`` so all keys are written in camelCase,
+    matching the format expected by the Go CLI.
+    """
+    keboola_dir = project_root / KEBOOLA_DIR_NAME
+    keboola_dir.mkdir(parents=True, exist_ok=True)
+
+    manifest_path = keboola_dir / MANIFEST_FILENAME
+    payload = manifest.model_dump(mode="json", by_alias=True)
+    manifest_path.write_text(
+        json.dumps(payload, indent=4, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )

keboola_agent_cli/sync/naming.py

@@ -0,0 +1,90 @@
+"""Path generation from naming templates.
+
+The naming templates (stored in ``manifest.json``) use placeholders like
+``{component_type}`` and ``{config_name}`` to produce deterministic
+filesystem paths for each configuration.
+"""
+
+from __future__ import annotations
+
+import re
+
+from ..constants import SANITIZE_NAME_MAX_LENGTH
+
+
+def config_path(
+    naming_template: str,
+    component_type: str,
+    component_id: str,
+    config_name: str,
+) -> str:
+    """Apply *naming_template* to generate a filesystem path for a configuration.
+
+    Example template: ``"{component_type}/{component_id}/{config_name}"``
+
+    All template inputs are passed through sanitizers so that an API-controlled
+    ``component_id`` like ``"../../etc"`` cannot escape the sync workspace
+    (issue #269 sec-01/sec-07). Legitimate component IDs (e.g.
+    ``keboola.python-transformation-v2``) preserve their dots and hyphens.
+    """
+    return naming_template.format(
+        component_type=sanitize_path_segment(component_type),
+        component_id=sanitize_path_segment(component_id),
+        config_name=sanitize_name(config_name),
+    )
+
+
+def config_row_path(naming_template: str, row_name: str) -> str:
+    """Apply *naming_template* to generate a path segment for a config row.
+
+    Example template: ``"rows/{config_row_name}"``
+    """
+    return naming_template.format(
+        config_row_name=sanitize_name(row_name),
+    )
+
+
+def sanitize_name(name: str) -> str:
+    """Sanitize *name* for use in filesystem paths.
+
+    Rules:
+    - Lowercase the string
+    - Replace spaces and non-alphanumeric characters (except hyphens)
+      with hyphens
+    - Collapse consecutive hyphens
+    - Strip leading and trailing hyphens
+    - Truncate to ``SANITIZE_NAME_MAX_LENGTH`` characters
+    """
+    result = name.lower()
+    # Replace anything that is not alphanumeric or a hyphen
+    result = re.sub(r"[^a-z0-9-]", "-", result)
+    # Collapse multiple hyphens
+    result = re.sub(r"-{2,}", "-", result)
+    # Strip leading/trailing hyphens
+    result = result.strip("-")
+    # Enforce max length
+    return result[:SANITIZE_NAME_MAX_LENGTH]
+
+
+def sanitize_path_segment(token: str) -> str:
+    """Sanitize an API-supplied token for use as a single path segment.
+
+    Stricter-than-``sanitize_name`` defense against path traversal: rejects
+    ``/``, ``\\``, parent-directory references (``..``), and other directory
+    separators while preserving the dots, hyphens, and underscores commonly
+    found in legitimate component IDs (e.g. ``keboola.ex-db-mysql``,
+    ``kds-team.app-custom-python``). Returns ``"_"`` if the input would
+    sanitize to empty so the resulting path always has a non-empty segment
+    (issue #269 sec-01).
+    """
+    # Replace path separators and whitespace with a single hyphen
+    result = re.sub(r"[/\\\s]", "-", token)
+    # Replace any run of 2+ dots (which would form parent refs) with
+    # a single underscore. Single dots are preserved for legitimate IDs.
+    result = re.sub(r"\.{2,}", "_", result)
+    # Strip leading dots that could be reintroduced as ``./...`` traversal
+    # if the template happens to put us at a directory boundary.
+    result = result.lstrip(".")
+    # Collapse repeated hyphens introduced by the substitutions
+    result = re.sub(r"-{2,}", "-", result).strip("-")
+    return result or "_"

keboola_agent_cli/sync/secrets.py

@@ -0,0 +1,62 @@
+"""Encrypted value detection for Keboola configurations.
+
+Keboola stores secrets as encrypted markers (e.g.
+``KBC::ProjectSecure::...``).  This module provides helpers to detect
+such values and locate them inside arbitrarily nested configuration dicts.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+ENCRYPTED_PREFIXES: tuple[str, ...] = (
+    "KBC::ProjectSecure::",
+    "KBC::ComponentSecure::",
+    "KBC::ConfigSecure::",
+    "KBC::ProjectWideSecure::",
+)
+
+
+def is_encrypted_value(value: Any) -> bool:
+    """Return ``True`` if *value* is a Keboola encrypted marker string."""
+    if not isinstance(value, str):
+        return False
+    return any(value.startswith(prefix) for prefix in ENCRYPTED_PREFIXES)
+
+
+def is_secret_key(key: str) -> bool:
+    """Return ``True`` if *key* indicates an encrypted field.
+
+    By Keboola convention, encrypted parameter keys start with ``#``.
+    """
+    return key.startswith("#")
+
+
+def find_encrypted_paths(obj: Any, prefix: str = "") -> list[str]:
+    """Walk *obj* recursively and return dot-separated paths of all encrypted values.
+
+    Both *encrypted marker values* and *secret keys* (starting with ``#``)
+    are reported.
+
+    Examples::
+
+        >>> find_encrypted_paths({"#token": "KBC::ProjectSecure::abc"})
+        ['#token']
+        >>> find_encrypted_paths({"a": {"#key": "val"}})
+        ['a.#key']
+    """
+    paths: list[str] = []
+
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            current = f"{prefix}.{key}" if prefix else key
+            if is_secret_key(key) or is_encrypted_value(value):
+                paths.append(current)
+            else:
+                paths.extend(find_encrypted_paths(value, prefix=current))
+    elif isinstance(obj, list):
+        for idx, item in enumerate(obj):
+            current = f"{prefix}[{idx}]"
+            paths.extend(find_encrypted_paths(item, prefix=current))
+
+    return paths

keboola_agent_cli/sync/sql_split.py

@@ -0,0 +1,134 @@
+"""Split and join SQL statements using a state machine.
+
+Splits on semicolons while respecting:
+- Single-quoted strings ('...')
+- Double-quoted identifiers ("...")
+- Dollar-quoted blocks ($$...$$)
+- Line comments (--, #, //)
+- Block comments (/* ... */)
+
+Compatible with the Keboola UI splitter and the old keboola-as-code CLI.
+"""
+
+from __future__ import annotations
+
+from enum import Enum, auto
+
+
+class _State(Enum):
+    NORMAL = auto()
+    SINGLE_QUOTE = auto()
+    DOUBLE_QUOTE = auto()
+    DOLLAR_QUOTE = auto()
+    LINE_COMMENT = auto()
+    BLOCK_COMMENT = auto()
+
+
+def split_statements(sql: str) -> list[str]:
+    """Split SQL text into individual statements on semicolons.
+
+    Returns a list of stripped, non-empty statements.  Trailing
+    semicolons are preserved on each statement.
+    """
+    sql = sql.rstrip()
+    if not sql:
+        return []
+
+    state = _State.NORMAL
+    statements: list[str] = []
+    buf: list[str] = []
+    i = 0
+    n = len(sql)
+
+    while i < n:
+        ch = sql[i]
+        nxt = sql[i + 1] if i + 1 < n else ""
+        two = ch + nxt
+
+        if state == _State.NORMAL:
+            if ch == ";":
+                buf.append(ch)
+                stmt = "".join(buf).strip()
+                if stmt.strip(";"):
+                    statements.append(stmt)
+                buf = []
+            elif ch == "'":
+                buf.append(ch)
+                state = _State.SINGLE_QUOTE
+            elif ch == '"':
+                buf.append(ch)
+                state = _State.DOUBLE_QUOTE
+            elif two == "$$":
+                buf.append(two)
+                state = _State.DOLLAR_QUOTE
+                i += 1
+            elif two in ("--", "//"):
+                buf.append(two)
+                state = _State.LINE_COMMENT
+                i += 1
+            elif ch == "#":
+                buf.append(ch)
+                state = _State.LINE_COMMENT
+            elif two == "/*":
+                buf.append(two)
+                state = _State.BLOCK_COMMENT
+                i += 1
+            else:
+                buf.append(ch)
+
+        elif state == _State.SINGLE_QUOTE:
+            if ch == "\\" and nxt:
+                buf.append(two)
+                i += 1
+            elif ch == "'":
+                buf.append(ch)
+                state = _State.NORMAL
+            else:
+                buf.append(ch)
+
+        elif state == _State.DOUBLE_QUOTE:
+            if ch == "\\" and nxt:
+                buf.append(two)
+                i += 1
+            elif ch == '"':
+                buf.append(ch)
+                state = _State.NORMAL
+            else:
+                buf.append(ch)
+
+        elif state == _State.DOLLAR_QUOTE:
+            if two == "$$":
+                buf.append(two)
+                state = _State.NORMAL
+                i += 1
+            else:
+                buf.append(ch)
+
+        elif state == _State.LINE_COMMENT:
+            buf.append(ch)
+            if ch == "\n":
+                state = _State.NORMAL
+
+        elif state == _State.BLOCK_COMMENT:
+            if two == "*/":
+                buf.append(two)
+                state = _State.NORMAL
+                i += 1
+            else:
+                buf.append(ch)
+
+        i += 1
+
+    # Remaining content without trailing semicolon
+    remaining = "".join(buf).strip()
+    if remaining:
+        statements.append(remaining)
+
+    return statements
+
+
+def join_statements(statements: list[str]) -> str:
+    """Join SQL statements with double newlines (matching Keboola convention)."""
+    if not statements:
+        return ""
+    return "\n\n".join(s.rstrip() for s in statements)