anysite-cli 0.1.6__tar.gz → 0.1.7__tar.gz

{anysite_cli-0.1.6 → anysite_cli-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anysite-cli
-Version: 0.1.6
+Version: 0.1.7
 Summary: CLI for Anysite API - web data extraction for humans and AI agents
 Project-URL: Homepage, https://anysite.io
 Project-URL: Documentation, https://docs.anysite.io/cli

{anysite_cli-0.1.6 → anysite_cli-0.1.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "anysite-cli"
-version = "0.1.6"
+version = "0.1.7"
 description = "CLI for Anysite API - web data extraction for humans and AI agents"
 readme = "README.md"
 license = "MIT"

{anysite_cli-0.1.6 → anysite_cli-0.1.7}/src/anysite/dataset/db_loader.py

@@ -315,8 +315,14 @@ class DatasetDbLoader:
                 return _extract_dot_value(record, diff_key)
             return record.get(diff_key)
 
+        # Build field mapping for db_load.fields filtering
+        field_mapping = self._get_db_field_mapping(source)
+
         # Determine the DB column name for the key
-        db_key_col = diff_key.replace(".", "_")
+        if field_mapping and diff_key in field_mapping:
+            db_key_col = field_mapping[diff_key]
+        else:
+            db_key_col = diff_key.replace(".", "_")
 
         # INSERT added records
         if result.added:
@@ -349,15 +355,29 @@ class DatasetDbLoader:
                 if not changed_fields:
                     continue
 
-                # Build SET clause from changed fields
+                # Build SET clause — only include fields that exist in the DB
                 set_parts = []
                 params: list[Any] = []
                 for field_name in changed_fields:
-                    new_val = record.get(field_name)
-                    safe_field = sanitize_identifier(field_name)
+                    if field_mapping is not None:
+                        if field_name not in field_mapping:
+                            continue
+                        db_col = field_mapping[field_name]
+                    else:
+                        db_col = field_name
+
+                    if "." in field_name:
+                        new_val = _extract_dot_value(record, field_name)
+                    else:
+                        new_val = record.get(field_name)
+
+                    safe_field = sanitize_identifier(db_col)
                     set_parts.append(f"{safe_field} = {ph}")
                     params.append(new_val)
 
+                if not set_parts:
+                    continue
+
                 params.append(str(key_val))
                 sql = (
                     f"UPDATE {table_name} "
@@ -376,6 +396,30 @@ class DatasetDbLoader:
                 return other.dependency.field
         return None
 
+    def _get_db_field_mapping(self, source: DatasetSource) -> dict[str, str] | None:
+        """Build mapping of parquet_field -> db_column from db_load.fields.
+
+        Returns None if no explicit fields configured (all fields allowed).
+        """
+        db_load = source.db_load
+        if not db_load or not db_load.fields:
+            return None
+
+        mapping: dict[str, str] = {}
+        for field_spec in db_load.fields:
+            alias = None
+            upper = field_spec.upper()
+            as_idx = upper.find(" AS ")
+            if as_idx != -1:
+                alias = field_spec[as_idx + 4:].strip()
+                source_field = field_spec[:as_idx].strip()
+            else:
+                source_field = field_spec
+
+            col_name = alias or source_field.replace(".", "_")
+            mapping[source_field] = col_name
+        return mapping
+
     def _placeholder(self) -> str:
         """Get the parameter placeholder for the dialect."""
         if self._dialect == "postgres":

{anysite_cli-0.1.6 → anysite_cli-0.1.7}/tests/test_dataset/test_db_loader.py

@@ -833,3 +833,128 @@ class TestPostgresPlaceholders:
             assert len(update_calls) == 1
             assert "%s" in update_calls[0][0]
             assert "?" not in update_calls[0][0]
+
+
+class TestUpdateFieldFiltering:
+    """Test that UPDATE only targets fields present in db_load.fields."""
+
+    def _setup_two_snapshots(self, tmp_path, source_id, old_records, new_records):
+        source_dir = get_source_dir(tmp_path / "data", source_id)
+        write_parquet(old_records, source_dir / "2026-01-01.parquet")
+        write_parquet(new_records, source_dir / "2026-01-02.parquet")
+
+    def test_update_only_db_load_fields(self, tmp_path):
+        """UPDATE should skip fields not in db_load.fields."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name", fields=["name", "score"]),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[{"name": "Alice", "score": 90, "extra": "old"}],
+            new_records=[{"name": "Alice", "score": 95, "extra": "new"}],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            # Full insert only creates columns from db_load.fields
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            # Table should only have id, name, score (no extra)
+            schema = adapter.get_table_schema("items")
+            col_names = [c["name"] for c in schema]
+            assert "extra" not in col_names
+            assert "score" in col_names
+
+            # Diff sync — extra changed but should be skipped
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+            assert results["items"] == 1  # score changed
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert rows[0]["score"] == 95
+
+    def test_update_with_dot_notation_alias(self, tmp_path):
+        """UPDATE uses correct DB column name for aliased dot-notation fields."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(
+                    key="meta.id",
+                    fields=["meta.id AS meta_id", "text", "count"],
+                ),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[
+                {"meta": json.dumps({"id": "x1"}), "text": "hello", "count": 5, "other": "a"},
+            ],
+            new_records=[
+                {"meta": json.dumps({"id": "x1"}), "text": "updated", "count": 10, "other": "b"},
+            ],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert rows[0]["meta_id"] == "x1"
+            assert rows[0]["text"] == "hello"
+
+            # Diff sync — text and count changed, other should be skipped
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert rows[0]["text"] == "updated"
+            assert rows[0]["count"] == 10
+
+    def test_update_skipped_when_no_db_fields_changed(self, tmp_path):
+        """If only non-DB fields changed, no UPDATE should happen."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name", fields=["name", "score"]),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[{"name": "Alice", "score": 90, "extra": "old"}],
+            # score unchanged, only extra changed
+            new_records=[{"name": "Alice", "score": 90, "extra": "new"}],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+            # extra is not in db_load.fields, so no actual update
+            assert results["items"] == 0
+
+            rows = adapter.fetch_all("SELECT * FROM items")
+            assert len(rows) == 1
+            assert rows[0]["score"] == 90