keboola-cli 0.80.2__tar.gz → 0.80.4__tar.gz

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: keboola-cli
-Version: 0.80.2
+Version: 0.80.4
 Summary: AI-friendly CLI for managing Keboola projects
 Author-email: Keboola <dev@keboola.com>
 License-Expression: MIT

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "keboola-cli"
-version = "0.80.2"
+version = "0.80.4"
 description = "AI-friendly CLI for managing Keboola projects"
 readme = "README.md"
 requires-python = ">=3.12"

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/src/keboola_agent_cli/changelog.py

@@ -24,6 +24,50 @@ from .constants import CHANGELOG_HEADLINE_MAX_CHARS
 
 # Ordered newest-first.  Each value is a list of brief one-line descriptions.
 CHANGELOG: dict[str, list[str]] = {
+    "0.80.4": [
+        "Fix: a sync manifest written on Windows was unusable by everyone else "
+        "on the team. `.keboola/manifest.json` is a **tracked** file -- sharing "
+        "the synced tree through git is the point of the sync workflow -- but "
+        "`sync push --adopt-existing` recorded config paths with the host's "
+        "separator, so a Windows machine committed "
+        "`extractor\\keboola.ex-http\\name`. The asymmetry is what hid it: "
+        "`Path()` on Windows accepts forward slashes, so POSIX-written "
+        "manifests work everywhere, while on POSIX a backslash path is a single "
+        "filename that matches nothing. Paths are now written with `as_posix()` "
+        "and normalised on load, so a manifest already committed by a Windows "
+        "kbagent repairs itself on the next read instead of needing a hand "
+        "edit. Found by running the full test suite on Windows rather than by a "
+        "report -- see the note below.",
+        "Note (tests): this is the first Windows-only defect the project found "
+        "itself. The full suite has never run on Windows in CI (only a narrow "
+        "slice does), and a real run was 55 failures -- unusable as a gate, so "
+        "nobody enabled it, so anything could hide in it. Working through those "
+        "failures to make gating possible is what surfaced this one, among "
+        "otherwise routine test-portability noise. The remaining failures are "
+        "POSIX-only permission assertions and unguarded `fcntl` imports.",
+    ],
+    "0.80.3": [
+        "Fix (#570): `kbagent lineage build` crashed with `UnicodeDecodeError` on "
+        "Windows when a transformation's `transform.sql` or `code.py` contained "
+        "non-ASCII text -- an accented word in a Czech comment was enough to abort "
+        "the whole graph. The reads used `Path.read_text()`, which decodes with "
+        "the platform default: cp1250 on a Czech box, where UTF-8's continuation "
+        "bytes are undefined. Keboola serves this content as UTF-8, so UTF-8 is "
+        "now explicit, which also makes a build reproducible across machines "
+        "instead of varying with the host locale. Files that genuinely hold "
+        "non-UTF-8 bytes -- a transformation edited locally and saved in the OS "
+        'codepage -- decode with `errors="replace"`: table identifiers are '
+        "ASCII, so a mangled comment costs nothing and never costs the graph. The "
+        "`PYTHONUTF8=1` workaround is no longer needed. Thanks to "
+        "@MichalProchazkaP3 for the report.",
+        "Fix (#570): the same unqualified `Path.read_text()` in "
+        "`dev-portal create --data` / `patch --value-file` and "
+        "`semantic-layer reference-data set --members-file` now reads UTF-8 too. "
+        "Those parse their input as data rather than scan it, so they stay "
+        "strict: a decode error there is honest and identical on every OS, "
+        "whereas the platform default silently produced different results per "
+        "machine.",
+    ],
     "0.80.2": [
         "Fix (#571): the deferred Windows self-update never applied. kbagent has "
         "not auto-updated on Windows since the deferred path landed in 0.78.0 -- "

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/src/keboola_agent_cli/commands/_semantic_layer_reference_data.py

@@ -111,7 +111,11 @@ def _print_reference_data_delete_result(console: Console, data: dict) -> None:
 def _load_members(formatter: Any, members_file: str) -> list[dict]:
     """Read a JSON array of member objects from a file or ``-`` (stdin)."""
     try:
-        raw = sys.stdin.read() if members_file == "-" else Path(members_file).read_text()
+        raw = (
+            sys.stdin.read()
+            if members_file == "-"
+            else Path(members_file).read_text(encoding="utf-8")
+        )
     except OSError as exc:
         formatter.error(
             message=f"Could not read members file {members_file!r}: {exc}",

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/src/keboola_agent_cli/commands/dev_portal.py

@@ -317,7 +317,7 @@ def _load_payload(data: str | None) -> dict:
         raise typer.BadParameter("--data is required")
     if data == "-":
         return json.loads(sys.stdin.read())
-    return json.loads(Path(data).read_text())
+    return json.loads(Path(data).read_text(encoding="utf-8"))
 
 
 def _render_pending(formatter: OutputFormatter, pending: PendingWrite) -> None:
@@ -435,7 +435,7 @@ def patch_cmd(
         payload = _load_payload(data)
     elif property_:
         if value_file:
-            raw = Path(value_file).read_text()
+            raw = Path(value_file).read_text(encoding="utf-8")
         elif value is not None:
             raw = value
         else:

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/src/keboola_agent_cli/services/deep_lineage_service.py

@@ -48,6 +48,23 @@ _QUALIFIED_3 = re.compile(r'"KBC_USE4_(\d+)"\s*\.\s*"([^"]+)"\s*\.\s*"([^"]+)"')
 _QUALIFIED_2 = re.compile(r'"([^"]+)"\s*\.\s*"([^"]+)"')
 
 
+def _read_source(path: Path) -> str:
+    """Read a transformation's code, whatever bytes the file actually holds.
+
+    ``Path.read_text()`` decodes with the platform default, which on a Czech or
+    Polish Windows box is cp1250 -- so a single accented character in a SQL
+    comment aborted the whole lineage build with ``UnicodeDecodeError`` (issue
+    #570). Keboola serves this content as UTF-8, so UTF-8 is the right guess
+    everywhere, and forcing it also makes a build reproducible across machines.
+
+    ``errors="replace"`` because the goal here is finding table references: a
+    file that genuinely holds non-UTF-8 bytes (a locally-edited transformation
+    saved in the OS codepage) should cost a mangled comment, never the entire
+    graph. Table identifiers are ASCII, so nothing load-bearing is lost.
+    """
+    return path.read_text(encoding="utf-8", errors="replace")
+
+
 def extract_sql_table_refs(sql: str, project_id: int) -> list[tuple[int, str, str]]:
     """Extract table references from Snowflake SQL using a state machine.
 
@@ -632,10 +649,10 @@ class DeepLineageService:
         code_py = full_path / "code.py"
 
         if transform_sql.exists():
-            code = transform_sql.read_text()
+            code = _read_source(transform_sql)
             code_type = "sql"
         elif code_py.exists():
-            code = code_py.read_text()
+            code = _read_source(code_py)
             code_type = "python"
 
         # Include config row mappings

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/src/keboola_agent_cli/services/sync_service.py

@@ -2350,7 +2350,7 @@ class SyncService(BaseService):
                         {
                             "component_id": keboola_meta.get("component_id", "unknown"),
                             "config_id": keboola_meta.get("config_id", ""),
-                            "path": str(config_dir.relative_to(project_root / branch.path)),
+                            "path": config_dir.relative_to(project_root / branch.path).as_posix(),
                         }
                     )
 

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/src/keboola_agent_cli/sync/manifest.py

@@ -10,7 +10,7 @@ import json
 from pathlib import Path
 from typing import Any
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 from ..constants import KEBOOLA_DIR_NAME, MANIFEST_FILENAME, MANIFEST_VERSION
 
@@ -59,6 +59,26 @@ class ManifestNaming(BaseModel):
     data_app_config: str = Field(default="app/{component_id}/{config_name}", alias="dataAppConfig")
 
 
+def _posix_path(value: str) -> str:
+    r"""Normalise a manifest path to forward slashes.
+
+    The manifest is a **tracked** file -- the point of a sync tree is that a
+    team shares it through git -- so its paths must mean the same thing on
+    every machine. The asymmetry is what makes this bite: ``Path()`` on Windows
+    happily accepts ``a/b``, but on POSIX ``a\b`` is a single filename that
+    merely contains a backslash. So a manifest written on Windows silently
+    stops resolving for everyone else on the team, while the reverse direction
+    works fine and hides the problem.
+
+    Applied on load as well as on write, so a manifest already committed by a
+    Windows kbagent repairs itself on the next read instead of needing a hand
+    edit. Component and config names are slugified by :mod:`.naming` long
+    before they reach a path, so a backslash here is always a separator and
+    never part of a real name.
+    """
+    return value.replace("\\", "/")
+
+
 class ManifestBranch(BaseModel):
     """A branch entry in the manifest."""
 
@@ -68,6 +88,11 @@ class ManifestBranch(BaseModel):
     path: str
     metadata: dict[str, Any] = Field(default_factory=dict)
 
+    @field_validator("path")
+    @classmethod
+    def _normalise_path(cls, value: str) -> str:
+        return _posix_path(value)
+
 
 class ManifestConfigRow(BaseModel):
     """A single configuration row reference.
@@ -85,6 +110,11 @@ class ManifestConfigRow(BaseModel):
     path: str
     metadata: dict[str, Any] = Field(default_factory=dict)
 
+    @field_validator("path")
+    @classmethod
+    def _normalise_path(cls, value: str) -> str:
+        return _posix_path(value)
+
 
 class ManifestConfiguration(BaseModel):
     """A single configuration reference inside the manifest."""
@@ -98,6 +128,11 @@ class ManifestConfiguration(BaseModel):
     metadata: dict[str, Any] = Field(default_factory=dict)
     rows: list[ManifestConfigRow] = Field(default_factory=list)
 
+    @field_validator("path")
+    @classmethod
+    def _normalise_path(cls, value: str) -> str:
+        return _posix_path(value)
+
 
 # ---------------------------------------------------------------------------
 # Root manifest

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/tests/test_deep_lineage_service.py

@@ -4,6 +4,7 @@ import json
 from pathlib import Path
 from unittest.mock import patch
 
+import pytest
 import yaml
 
 from keboola_agent_cli.config_store import ConfigStore
@@ -15,6 +16,7 @@ from keboola_agent_cli.services.deep_lineage_service import (
     Table,
     _collect_create_targets,
     _collect_cte_names,
+    _read_source,
     _strip_comments_and_strings,
     extract_sql_table_refs,
 )
@@ -939,3 +941,48 @@ class TestRenderErDiagramXssRegression:
         )
         assert "<img" not in rendered
         assert "&lt;img" in rendered
+
+
+class TestSourceReadsAreUtf8:
+    """Transformation code must decode the same on every host (issue #570).
+
+    `Path.read_text()` uses the platform default encoding. On a Czech or Polish
+    Windows box that is cp1250, so one accented character in a SQL comment
+    aborted the entire `lineage build` with `UnicodeDecodeError`. These run
+    everywhere by driving the decode explicitly rather than by depending on the
+    host's locale.
+    """
+
+    def test_utf8_content_survives_a_cp1250_default(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """The exact reported crash: UTF-8 bytes read on a cp1250 host."""
+        sql = tmp_path / "transform.sql"
+        # U+0159 encodes to 0xC5 0x99 in UTF-8; 0x99 is undefined in cp1250,
+        # which is what raised in the report.
+        sql.write_bytes("-- příprava dat\nSELECT 1 FROM t;\n".encode())
+
+        real_read_text = Path.read_text
+
+        def cp1250_default(
+            self: Path, encoding: str | None = None, errors: str | None = None
+        ) -> str:
+            # Mimic a Windows host: no explicit encoding means cp1250.
+            return real_read_text(self, encoding=encoding or "cp1250", errors=errors)
+
+        monkeypatch.setattr(Path, "read_text", cp1250_default)
+
+        code = _read_source(sql)
+        assert "SELECT 1 FROM t" in code
+        assert "příprava" in code, "UTF-8 content must round-trip intact"
+
+    def test_genuinely_undecodable_bytes_do_not_abort_the_build(self, tmp_path: Path) -> None:
+        """A locally-edited file saved in the OS codepage costs a comment, not the graph."""
+        sql = tmp_path / "transform.sql"
+        # 0x81 is invalid UTF-8 and is the byte from the report's traceback.
+        sql.write_bytes(b"-- koment\x81\nSELECT 1 FROM in.c_bucket.tbl;\n")
+
+        code = _read_source(sql)
+
+        # The table reference -- the only thing lineage actually needs -- survives.
+        assert "in.c_bucket.tbl" in code

{keboola_cli-0.80.2 → keboola_cli-0.80.4}/tests/test_sync_manifest.py

@@ -1,6 +1,7 @@
 """Tests for sync manifest models and load/save functions."""
 
 import json
+from pathlib import Path
 
 import pytest
 
@@ -275,3 +276,48 @@ class TestManifestExtraFields:
         # Extra field on nested model
         project_dumped = loaded.project.model_dump(mode="json", by_alias=True)
         assert project_dumped["unknownField"] == "kept"
+
+
+class TestManifestPathsArePortable:
+    """The manifest is tracked in git, so its paths must be OS-neutral.
+
+    The asymmetry is what makes this dangerous: `Path()` on Windows accepts
+    `a/b`, but on POSIX `a\\b` is a single filename containing a backslash. A
+    manifest written on Windows therefore stops resolving for every teammate on
+    macOS or Linux, while the reverse direction works and hides the problem.
+    """
+
+    def test_windows_separators_are_normalised_on_load(self) -> None:
+        """A manifest already committed by a Windows kbagent repairs itself."""
+        cfg = ManifestConfiguration(
+            branchId=1,
+            componentId="keboola.ex-http",
+            id="123",
+            path=r"extractor\keboola.ex-http\adopted-extractor",
+        )
+        assert cfg.path == "extractor/keboola.ex-http/adopted-extractor"
+
+    def test_branch_and_row_paths_are_normalised_too(self) -> None:
+        assert ManifestBranch(id=1, path=r"main\nested").path == "main/nested"
+        assert ManifestConfigRow(id="r1", path=r"rows\first").path == "rows/first"
+
+    def test_posix_paths_are_left_alone(self) -> None:
+        """The common case must be untouched, not round-tripped through a rewrite."""
+        cfg = ManifestConfiguration(branchId=1, componentId="c", id="1", path="extractor/c/name")
+        assert cfg.path == "extractor/c/name"
+
+    def test_a_saved_manifest_holds_forward_slashes(self, tmp_path: Path) -> None:
+        """End to end: what lands in git is portable regardless of the writer's OS."""
+        manifest = Manifest(
+            project=ManifestProject(id=1, apiHost="connection.keboola.com"),
+            naming=ManifestNaming(),
+            branches=[ManifestBranch(id=1, path="main")],
+            configurations=[
+                ManifestConfiguration(branchId=1, componentId="c", id="1", path=r"extractor\c\name")
+            ],
+        )
+        save_manifest(tmp_path, manifest)
+
+        raw = (tmp_path / ".keboola" / "manifest.json").read_text(encoding="utf-8")
+        assert "extractor/c/name" in raw
+        assert "\\\\" not in raw, "a backslash in the tracked manifest breaks POSIX checkouts"