anysite-cli 0.1.5__tar.gz → 0.1.7__tar.gz

{anysite_cli-0.1.5 → anysite_cli-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anysite-cli
-Version: 0.1.5
+Version: 0.1.7
 Summary: CLI for Anysite API - web data extraction for humans and AI agents
 Project-URL: Homepage, https://anysite.io
 Project-URL: Documentation, https://docs.anysite.io/cli

{anysite_cli-0.1.5 → anysite_cli-0.1.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "anysite-cli"
-version = "0.1.5"
+version = "0.1.7"
 description = "CLI for Anysite API - web data extraction for humans and AI agents"
 readme = "README.md"
 license = "MIT"

{anysite_cli-0.1.5 → anysite_cli-0.1.7}/src/anysite/dataset/db_loader.py

@@ -315,8 +315,14 @@ class DatasetDbLoader:
                 return _extract_dot_value(record, diff_key)
             return record.get(diff_key)
 
+        # Build field mapping for db_load.fields filtering
+        field_mapping = self._get_db_field_mapping(source)
+
         # Determine the DB column name for the key
-        db_key_col = diff_key.replace(".", "_")
+        if field_mapping and diff_key in field_mapping:
+            db_key_col = field_mapping[diff_key]
+        else:
+            db_key_col = diff_key.replace(".", "_")
 
         # INSERT added records
         if result.added:
@@ -326,13 +332,14 @@ class DatasetDbLoader:
                 total += 1
 
         # DELETE removed records (skipped in append mode)
+        ph = self._placeholder()
         if result.removed and sync_mode == "full":
             safe_col = sanitize_identifier(db_key_col)
             for record in result.removed:
                 key_val = _get_key_val(record)
                 if key_val is not None:
                     self.adapter.execute(
-                        f"DELETE FROM {table_name} WHERE {safe_col} = ?",
+                        f"DELETE FROM {table_name} WHERE {safe_col} = {ph}",
                         (str(key_val),),
                     )
                     total += 1
@@ -348,20 +355,34 @@ class DatasetDbLoader:
                 if not changed_fields:
                     continue
 
-                # Build SET clause from changed fields
+                # Build SET clause — only include fields that exist in the DB
                 set_parts = []
                 params: list[Any] = []
                 for field_name in changed_fields:
-                    new_val = record.get(field_name)
-                    safe_field = sanitize_identifier(field_name)
-                    set_parts.append(f"{safe_field} = ?")
+                    if field_mapping is not None:
+                        if field_name not in field_mapping:
+                            continue
+                        db_col = field_mapping[field_name]
+                    else:
+                        db_col = field_name
+
+                    if "." in field_name:
+                        new_val = _extract_dot_value(record, field_name)
+                    else:
+                        new_val = record.get(field_name)
+
+                    safe_field = sanitize_identifier(db_col)
+                    set_parts.append(f"{safe_field} = {ph}")
                     params.append(new_val)
 
+                if not set_parts:
+                    continue
+
                 params.append(str(key_val))
                 sql = (
                     f"UPDATE {table_name} "
                     f"SET {', '.join(set_parts)} "
-                    f"WHERE {safe_col} = ?"
+                    f"WHERE {safe_col} = {ph}"
                 )
                 self.adapter.execute(sql, tuple(params))
                 total += 1
@@ -375,6 +396,36 @@ class DatasetDbLoader:
                 return other.dependency.field
         return None
 
+    def _get_db_field_mapping(self, source: DatasetSource) -> dict[str, str] | None:
+        """Build mapping of parquet_field -> db_column from db_load.fields.
+
+        Returns None if no explicit fields configured (all fields allowed).
+        """
+        db_load = source.db_load
+        if not db_load or not db_load.fields:
+            return None
+
+        mapping: dict[str, str] = {}
+        for field_spec in db_load.fields:
+            alias = None
+            upper = field_spec.upper()
+            as_idx = upper.find(" AS ")
+            if as_idx != -1:
+                alias = field_spec[as_idx + 4:].strip()
+                source_field = field_spec[:as_idx].strip()
+            else:
+                source_field = field_spec
+
+            col_name = alias or source_field.replace(".", "_")
+            mapping[source_field] = col_name
+        return mapping
+
+    def _placeholder(self) -> str:
+        """Get the parameter placeholder for the dialect."""
+        if self._dialect == "postgres":
+            return "%s"
+        return "?"
+
     def _auto_id_type(self) -> str:
         """Get the auto-increment ID column type for the dialect."""
         if self._dialect == "postgres":

{anysite_cli-0.1.5 → anysite_cli-0.1.7}/tests/test_dataset/test_db_loader.py

@@ -1,7 +1,6 @@
 """Tests for dataset DB loader with SQLite in-memory adapter."""
 
 import json
-
 import pytest
 
 from anysite.dataset.db_loader import DatasetDbLoader, _extract_dot_value, _filter_record
@@ -736,3 +735,226 @@ class TestAppendSyncMode:
             rows = adapter.fetch_all("SELECT * FROM posts")
             assert len(rows) == 1
             assert rows[0]["uid"] == "a"
+
+
+class TestPostgresPlaceholders:
+    """Test that diff-based sync uses %s placeholders for postgres dialect."""
+
+    def _setup_two_snapshots(self, tmp_path, source_id, old_records, new_records):
+        source_dir = get_source_dir(tmp_path / "data", source_id)
+        write_parquet(old_records, source_dir / "2026-01-01.parquet")
+        write_parquet(new_records, source_dir / "2026-01-02.parquet")
+
+    def test_delete_uses_percent_s(self, tmp_path):
+        """DELETE query uses %s placeholder for postgres."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name", sync="full"),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[
+                {"name": "Alice", "score": 90},
+                {"name": "Bob", "score": 80},
+            ],
+            new_records=[{"name": "Alice", "score": 90}],
+        )
+
+        # Use real SQLite adapter for initial load, then mock for diff sync
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            # Patch dialect to postgres and spy on execute
+            loader2 = DatasetDbLoader(config, adapter)
+            loader2._dialect = "postgres"
+            original_execute = adapter.execute
+            calls = []
+
+            def spy_execute(sql, params=None):
+                calls.append((sql, params))
+                # Replace %s with ? for SQLite execution
+                original_execute(sql.replace("%s", "?"), params)
+
+            adapter.execute = spy_execute
+            loader2.load_all()
+
+            # Verify DELETE used %s
+            delete_calls = [c for c in calls if "DELETE" in c[0]]
+            assert len(delete_calls) == 1
+            assert "%s" in delete_calls[0][0]
+            assert "?" not in delete_calls[0][0]
+
+    def test_update_uses_percent_s(self, tmp_path):
+        """UPDATE query uses %s placeholders for postgres."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name"),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[{"name": "Alice", "score": 90}],
+            new_records=[{"name": "Alice", "score": 95}],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            loader2 = DatasetDbLoader(config, adapter)
+            loader2._dialect = "postgres"
+            original_execute = adapter.execute
+            calls = []
+
+            def spy_execute(sql, params=None):
+                calls.append((sql, params))
+                original_execute(sql.replace("%s", "?"), params)
+
+            adapter.execute = spy_execute
+            loader2.load_all()
+
+            update_calls = [c for c in calls if "UPDATE" in c[0]]
+            assert len(update_calls) == 1
+            assert "%s" in update_calls[0][0]
+            assert "?" not in update_calls[0][0]
+
+
+class TestUpdateFieldFiltering:
+    """Test that UPDATE only targets fields present in db_load.fields."""
+
+    def _setup_two_snapshots(self, tmp_path, source_id, old_records, new_records):
+        source_dir = get_source_dir(tmp_path / "data", source_id)
+        write_parquet(old_records, source_dir / "2026-01-01.parquet")
+        write_parquet(new_records, source_dir / "2026-01-02.parquet")
+
+    def test_update_only_db_load_fields(self, tmp_path):
+        """UPDATE should skip fields not in db_load.fields."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name", fields=["name", "score"]),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[{"name": "Alice", "score": 90, "extra": "old"}],
+            new_records=[{"name": "Alice", "score": 95, "extra": "new"}],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            # Full insert only creates columns from db_load.fields
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            # Table should only have id, name, score (no extra)
+            schema = adapter.get_table_schema("items")
+            col_names = [c["name"] for c in schema]
+            assert "extra" not in col_names
+            assert "score" in col_names
+
+            # Diff sync — extra changed but should be skipped
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+            assert results["items"] == 1  # score changed
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert rows[0]["score"] == 95
+
+    def test_update_with_dot_notation_alias(self, tmp_path):
+        """UPDATE uses correct DB column name for aliased dot-notation fields."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(
+                    key="meta.id",
+                    fields=["meta.id AS meta_id", "text", "count"],
+                ),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[
+                {"meta": json.dumps({"id": "x1"}), "text": "hello", "count": 5, "other": "a"},
+            ],
+            new_records=[
+                {"meta": json.dumps({"id": "x1"}), "text": "updated", "count": 10, "other": "b"},
+            ],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert rows[0]["meta_id"] == "x1"
+            assert rows[0]["text"] == "hello"
+
+            # Diff sync — text and count changed, other should be skipped
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert rows[0]["text"] == "updated"
+            assert rows[0]["count"] == 10
+
+    def test_update_skipped_when_no_db_fields_changed(self, tmp_path):
+        """If only non-DB fields changed, no UPDATE should happen."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name", fields=["name", "score"]),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[{"name": "Alice", "score": 90, "extra": "old"}],
+            # score unchanged, only extra changed
+            new_records=[{"name": "Alice", "score": 90, "extra": "new"}],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+            # extra is not in db_load.fields, so no actual update
+            assert results["items"] == 0
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert len(rows) == 1
+            assert rows[0]["score"] == 90