keboola-cli 0.63.4__py3-none-any.whl

keboola_agent_cli/services/_sync_bulk.py

@@ -0,0 +1,228 @@
+"""Multi-project (all-projects) pull/diff/push orchestration (from sync_service.py).
+
+Free functions taking the ``SyncService`` so the public ``pull_all`` / ``diff_all``
+/ ``push_all`` methods stay thin delegators. Each fans out across the registered
+projects in parallel, materialising one ``base_dir/<alias>/`` per project, and
+collects per-project results without letting one project's failure abort the rest.
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from ..constants import (
+    DEFAULT_JOBS_PER_CONFIG,
+    DEFAULT_MAX_SAMPLES,
+    DEFAULT_SAMPLE_LIMIT,
+    KEBOOLA_DIR_NAME,
+)
+from ..errors import SyncConflictError
+
+if TYPE_CHECKING:
+    from .sync_service import SyncService
+
+
+def pull_all(
+    service: SyncService,
+    base_dir: Path,
+    force: bool = False,
+    dry_run: bool = False,
+    job_limit: int = DEFAULT_JOBS_PER_CONFIG,
+    no_storage: bool = False,
+    no_jobs: bool = False,
+    with_samples: bool = False,
+    sample_limit: int = DEFAULT_SAMPLE_LIMIT,
+    max_samples: int = DEFAULT_MAX_SAMPLES,
+) -> dict[str, Any]:
+    """Pull all registered projects in parallel.
+
+    For each project, creates ``base_dir/<alias>/`` and initializes if no
+    manifest exists yet, then pulls. Returns a dict with per-project results and
+    a summary.
+    """
+    projects = service.resolve_projects(None)
+    results: dict[str, Any] = {}
+    success_count = 0
+    failed_count = 0
+
+    def _worker(alias: str) -> None:
+        nonlocal success_count, failed_count
+        project_root = base_dir / alias
+        manifest_path = project_root / KEBOOLA_DIR_NAME / "manifest.json"
+        try:
+            if not manifest_path.exists():
+                service.init_sync(alias, project_root)
+            result = service.pull(
+                alias,
+                project_root,
+                force=force,
+                dry_run=dry_run,
+                job_limit=job_limit,
+                no_storage=no_storage,
+                no_jobs=no_jobs,
+                with_samples=with_samples,
+                sample_limit=sample_limit,
+                max_samples=max_samples,
+            )
+            results[alias] = result
+            success_count += 1
+        except SyncConflictError as exc:
+            # Preserve the structured conflict so a programmatic / AI consumer of
+            # `--all-projects --json` can tell a merge conflict apart from any
+            # other error and read the conflicting configs -- the single-project
+            # path emits the same code + conflicts.
+            results[alias] = {
+                "error": exc.message,
+                "error_code": exc.error_code,
+                "conflicts": exc.conflicts,
+            }
+            failed_count += 1
+        except Exception as exc:
+            results[alias] = {"error": str(exc)}
+            failed_count += 1
+
+    max_workers = min(len(projects), service._resolve_max_workers()) if projects else 1
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(_worker, alias): alias for alias in projects}
+        for future in as_completed(futures):
+            # Exceptions are captured inside _worker; this catches truly
+            # unexpected failures (e.g. threading errors).
+            try:
+                future.result()
+            except Exception as exc:
+                alias = futures[future]
+                results[alias] = {"error": str(exc)}
+                failed_count += 1
+
+    total = len(projects)
+    return {
+        "projects": results,
+        "summary": {
+            "total": total,
+            "success": success_count,
+            "failed": failed_count,
+        },
+    }
+
+
+def diff_all(service: SyncService, base_dir: Path) -> dict[str, Any]:
+    """Diff all registered projects that have a local manifest.
+
+    Projects without an existing manifest are skipped. Returns a dict with
+    per-project diff results, a summary, and a skipped list.
+    """
+    projects = service.resolve_projects(None)
+    results: dict[str, Any] = {}
+    skipped: list[str] = []
+    success_count = 0
+    failed_count = 0
+
+    actionable: list[str] = []
+    for alias in projects:
+        manifest_path = base_dir / alias / KEBOOLA_DIR_NAME / "manifest.json"
+        if manifest_path.exists():
+            actionable.append(alias)
+        else:
+            skipped.append(alias)
+
+    def _worker(alias: str) -> None:
+        nonlocal success_count, failed_count
+        project_root = base_dir / alias
+        try:
+            results[alias] = service.diff(alias, project_root)
+            success_count += 1
+        except Exception as exc:
+            results[alias] = {"error": str(exc)}
+            failed_count += 1
+
+    max_workers = min(len(actionable), service._resolve_max_workers()) if actionable else 1
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(_worker, alias): alias for alias in actionable}
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception as exc:
+                alias = futures[future]
+                results[alias] = {"error": str(exc)}
+                failed_count += 1
+
+    total = len(projects)
+    return {
+        "projects": results,
+        "summary": {
+            "total": total,
+            "success": success_count,
+            "failed": failed_count,
+            "skipped": len(skipped),
+        },
+        "skipped": skipped,
+    }
+
+
+def push_all(
+    service: SyncService,
+    base_dir: Path,
+    dry_run: bool = False,
+    force: bool = False,
+    allow_plaintext_fallback: bool = False,
+) -> dict[str, Any]:
+    """Push all registered projects that have a local manifest.
+
+    Projects without an existing manifest are skipped. Returns a dict with
+    per-project push results, a summary, and a skipped list.
+    """
+    projects = service.resolve_projects(None)
+    results: dict[str, Any] = {}
+    skipped: list[str] = []
+    success_count = 0
+    failed_count = 0
+
+    actionable: list[str] = []
+    for alias in projects:
+        manifest_path = base_dir / alias / KEBOOLA_DIR_NAME / "manifest.json"
+        if manifest_path.exists():
+            actionable.append(alias)
+        else:
+            skipped.append(alias)
+
+    def _worker(alias: str) -> None:
+        nonlocal success_count, failed_count
+        project_root = base_dir / alias
+        try:
+            result = service.push(
+                alias,
+                project_root,
+                dry_run=dry_run,
+                force=force,
+                allow_plaintext_fallback=allow_plaintext_fallback,
+            )
+            results[alias] = result
+            success_count += 1
+        except Exception as exc:
+            results[alias] = {"error": str(exc)}
+            failed_count += 1
+
+    max_workers = min(len(actionable), service._resolve_max_workers()) if actionable else 1
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(_worker, alias): alias for alias in actionable}
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception as exc:
+                alias = futures[future]
+                results[alias] = {"error": str(exc)}
+                failed_count += 1
+
+    total = len(projects)
+    return {
+        "projects": results,
+        "summary": {
+            "total": total,
+            "success": success_count,
+            "failed": failed_count,
+            "skipped": len(skipped),
+        },
+        "skipped": skipped,
+    }

keboola_agent_cli/services/_sync_clone.py

@@ -0,0 +1,163 @@
+"""The ``sync clone`` composite orchestration (#426).
+
+Extracted from ``sync_service.py``. ``SyncService.clone_project`` is a thin
+delegator to :func:`clone_project` here; the pure, client-free override helpers
+live in ``..sync.clone``. This function only orchestrates: validate the
+reference, copy + re-point + parameterize the tree, run the fresh-target guard,
+and push (so push Phase C/D remaps the flow/variable links).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urlparse
+
+from ..errors import ConfigError
+from ..sync.clone import (
+    apply_bucket_map,
+    apply_instance_rename,
+    apply_variable_values,
+    copy_reference_tree,
+    repoint_manifest_project,
+)
+from ..sync.manifest import load_manifest, save_manifest
+
+if TYPE_CHECKING:
+    from .sync_service import SyncService
+
+
+def clone_project(
+    service: SyncService,
+    source: str | Path,
+    target_alias: str,
+    target_dir: str | Path,
+    *,
+    overrides: dict[str, Any] | None = None,
+    dry_run: bool = False,
+    branch_override: int | None = None,
+) -> dict[str, Any]:
+    """Clone a reference synced project into a fresh target project (#426).
+
+    Copies the reference tree at ``source`` into ``target_dir``, applies the
+    declarative ``overrides`` (``bucket_map``, ``variable_values``,
+    ``instance_rename``), re-points the manifest at ``target_alias``'s project,
+    and pushes -- so every config is CREATEd fresh and its flow task / variable
+    links are remapped reference->ULID by push Phase C/D.
+
+    Cloning into a fresh target needs no id surgery: the reference's config ids
+    do not exist in the target remote, so the diff classifies every config as
+    ``added`` and ``created_id_map`` (keyed by the reference id) drives the link
+    remaps.
+
+    Idempotent: re-running with an existing ``target_dir`` skips the copy +
+    overrides and just pushes, so a clean clone then reports ``no_changes`` /
+    ``created: 0``.
+
+    Args:
+        service: The owning :class:`SyncService` (for ``resolve_projects`` /
+            ``diff`` / ``push``).
+        source: A reference synced project dir (must contain
+            ``.keboola/manifest.json``).
+        target_alias: The project alias to clone INTO (must be a fresh project on
+            the first clone).
+        target_dir: Where to materialise the clone.
+        overrides: Optional dict with ``bucket_map`` (old->new bucket id),
+            ``variable_values`` (var name->value), and ``instance_rename`` (old
+            path prefix->new path prefix) keys.
+        dry_run: Apply overrides + report the diff without pushing.
+        branch_override: Optional target branch id.
+
+    Returns:
+        A dict matching ``CloneResult``: ``status``
+        (``cloned`` | ``no_changes`` | ``dry_run``), ``target_alias``,
+        ``target_dir``, ``created``, ``bucket_rewrites``, ``variable_overrides``,
+        ``renamed_instances``, ``flow_task_remaps``, ``push`` (the underlying
+        push result), ``errors``.
+
+    Raises:
+        ConfigError: ``source`` is not a synced project, ``target_alias`` is
+            unknown, or (first clone) the target project already contains the
+            reference's configs (not a fresh target).
+    """
+    overrides = overrides or {}
+    bucket_map = overrides.get("bucket_map") or {}
+    variable_values = overrides.get("variable_values") or {}
+    instance_rename = overrides.get("instance_rename") or {}
+    source_dir = Path(source)
+    target_path = Path(target_dir)
+
+    # Validate the reference is a synced project.
+    try:
+        load_manifest(source_dir)
+    except FileNotFoundError as exc:
+        raise ConfigError(
+            f"Source {source_dir} is not a synced project (no "
+            ".keboola/manifest.json). Run `kbagent sync pull` there first."
+        ) from exc
+
+    target_project = service.resolve_projects([target_alias])[target_alias]
+
+    bucket_rewrites = 0
+    variable_overrides = 0
+    renamed_instances = 0
+    already_cloned = target_path.exists()
+
+    if not already_cloned:
+        copy_reference_tree(source_dir, target_path)
+        manifest = load_manifest(target_path)
+        repoint_manifest_project(
+            manifest,
+            project_id=target_project.project_id or 0,
+            api_host=urlparse(target_project.stack_url).netloc,
+        )
+        bucket_rewrites = apply_bucket_map(target_path, manifest, bucket_map)
+        variable_overrides = apply_variable_values(target_path, manifest, variable_values)
+        renamed_instances = apply_instance_rename(target_path, manifest, instance_rename)
+        save_manifest(target_path, manifest)
+
+    override_counts = {
+        "bucket_rewrites": bucket_rewrites,
+        "variable_overrides": variable_overrides,
+        "renamed_instances": renamed_instances,
+    }
+
+    if dry_run:
+        diff_result = service.diff(target_alias, target_path, branch_override=branch_override)
+        return {
+            "status": "dry_run",
+            "target_alias": target_alias,
+            "target_dir": str(target_path),
+            "summary": diff_result["summary"],
+            **override_counts,
+        }
+
+    # Fresh-target guard: on a first clone every one of OUR configs must be a
+    # CREATE. A non-'added' change means a reference id already exists in the
+    # target -> not a fresh target; refuse rather than UPDATE a stranger's config.
+    if not already_cloned:
+        diff_result = service.diff(target_alias, target_path, branch_override=branch_override)
+        collisions = [
+            f"{c.get('component_id')}/{c.get('config_id')}"
+            for c in diff_result["changes"]
+            if c["change_type"] != "added"
+        ]
+        if collisions:
+            raise ConfigError(
+                "Clone requires a fresh target project, but these configs already "
+                f"exist there: {', '.join(collisions[:10])}. Use a new/empty target "
+                "project, or remove the existing configs first."
+            )
+
+    push_result = service.push(target_alias, target_path, branch_override=branch_override)
+    status = "no_changes" if push_result.get("status") == "no_changes" else "cloned"
+    return {
+        "status": status,
+        "target_alias": target_alias,
+        "target_dir": str(target_path),
+        "created": push_result.get("created", 0),
+        "flow_task_remaps": push_result.get("flow_task_remaps", 0),
+        "push": push_result,
+        "errors": push_result.get("errors", []),
+        **override_counts,
+    }

keboola_agent_cli/services/_sync_models.py

@@ -0,0 +1,97 @@
+"""Shared dataclasses + component-id constants for the sync service.
+
+Extracted from ``sync_service.py`` (which had grown to ~4000 lines, far past the
+1500-LOC ceiling) so the binding helpers in ``_sync_bindings.py`` can import them
+without a circular import. ``sync_service`` re-exports the names that external
+callers/tests rely on (e.g. ``CreatedConfig``), so the public surface is
+unchanged.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..sync.manifest import ManifestConfiguration
+
+# Sibling component that backs a transformation's variable links. A
+# transformation references it via ``configuration.variables_id`` (the config)
+# and ``configuration.variables_values_id`` (a row id).
+VARIABLES_COMPONENT_ID = "keboola.variables"
+
+# Conditional-flow component. A flow runs other configs via
+# ``configuration.tasks[].task.configId`` (job-type tasks); the Phase-D backfill
+# remaps those ids placeholder/source -> ULID after a fresh create (e.g. clone).
+FLOW_COMPONENT_ID = "keboola.flow"
+
+
+@dataclass
+class WritebackResult:
+    """Outcome of recording a freshly-created config in the manifest.
+
+    ``previous_id`` is the manifest entry's id **before** the placeholder ->
+    ULID overwrite (empty string when a brand-new entry was appended). The
+    create pass uses it to key ``created_id_map`` so row parents and
+    transformation variable links can be remapped placeholder -> ULID.
+    """
+
+    entry: ManifestConfiguration
+    previous_id: str
+
+
+@dataclass
+class CreatedConfig:
+    """A config created during a single ``push`` create pass.
+
+    Carries just enough to drive the Phase-C variable-link backfill: the
+    component id, the API-assigned ULID, and the on-disk directory holding
+    the (post-writeback) ``_config.yml``.
+    """
+
+    component_id: str
+    config_id: str
+    config_dir: Path
+
+
+@dataclass
+class VariableBindingResult:
+    """Outcome of the Phase-C variable-link backfill.
+
+    ``configs_rewritten`` counts transformations whose remote configuration +
+    local ``_configuration_extra`` were rebound to ULIDs (drives the
+    manifest-dirty flag). ``errors`` accumulates unresolved links so the push
+    envelope surfaces them instead of leaving a broken link silently.
+    """
+
+    errors: list[dict[str, str]] = field(default_factory=list)
+    configs_rewritten: int = 0
+
+
+@dataclass
+class FlowBindingResult:
+    """Outcome of the Phase-D flow-task-link backfill (#426).
+
+    ``configs_rewritten`` counts flows whose task ``configId``s were remapped to
+    ULIDs (drives the manifest-dirty flag); ``tasks_remapped`` is the total task
+    references rewritten; ``errors`` accumulates PUT failures so the push
+    envelope surfaces them.
+    """
+
+    errors: list[dict[str, str]] = field(default_factory=list)
+    configs_rewritten: int = 0
+    tasks_remapped: int = 0
+
+
+@dataclass
+class LocalConfigHashes:
+    """Hashes describing a config dir's on-disk state after a push.
+
+    ``file_hash`` is the ``_config.yml`` content hash, ``cfg_hash`` the
+    normalized config hash (see :func:`config_hash`), and ``extra_hashes``
+    maps each extracted code/companion file to its hash. Stored on the
+    manifest entry so the next ``sync diff`` recognises local == remote.
+    """
+
+    file_hash: str
+    cfg_hash: str
+    extra_hashes: dict[str, str] = field(default_factory=dict)