anysite-cli 0.1.4__tar.gz → 0.1.6__tar.gz

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/CLAUDE.md

@@ -49,12 +49,14 @@ anysite dataset query dataset.yaml --interactive
 anysite dataset stats dataset.yaml --source profiles
 anysite dataset profile dataset.yaml
 anysite dataset load-db dataset.yaml -c pg --drop-existing
+anysite dataset load-db dataset.yaml -c pg --snapshot 2026-01-15
 anysite dataset history my-dataset
 anysite dataset logs my-dataset --run 42
 anysite dataset schedule dataset.yaml --incremental --load-db pg
 anysite dataset schedule dataset.yaml --systemd --load-db pg
 anysite dataset diff dataset.yaml --source profiles --key _input_value
-anysite dataset diff dataset.yaml --source profiles --key urn --from 2026-01-30 --to 2026-02-01
+anysite dataset diff dataset.yaml --source profiles --key urn.value --from 2026-01-30 --to 2026-02-01
+anysite dataset diff dataset.yaml --source profiles --key urn.value --fields "name,headline,follower_count"
 anysite dataset reset-cursor dataset.yaml
 anysite dataset reset-cursor dataset.yaml --source profiles
 
@@ -104,9 +106,9 @@ anysite db upsert mydb --table users --conflict-columns id --stdin
 - `dataset/history.py` - `HistoryStore` (SQLite at `~/.anysite/dataset_history.db`): run start/finish tracking. `LogManager`: file-based per-run logs at `~/.anysite/logs/`
 - `dataset/scheduler.py` - `ScheduleGenerator`: crontab and systemd timer unit generation from cron expressions
 - `dataset/notifications.py` - `WebhookNotifier`: POST to webhook URLs on collection complete/failure
-- `dataset/differ.py` - `DatasetDiffer`: compare two Parquet snapshots using DuckDB (added/removed/changed records). `DiffResult` dataclass, `format_diff_table()` and `format_diff_records()` formatters
+- `dataset/differ.py` - `DatasetDiffer`: compare two Parquet snapshots using DuckDB (added/removed/changed records). Supports dot-notation keys via `json_extract_string()`. `DiffResult` dataclass, `format_diff_table()` and `format_diff_records()` formatters with output field filtering
 - `dataset/cli.py` - Typer subcommands: `init`, `collect` (with `--load-db`), `status`, `query`, `stats`, `profile`, `load-db`, `diff`, `history`, `logs`, `schedule`, `reset-cursor`
-- `dataset/db_loader.py` - `DatasetDbLoader`: loads Parquet data into relational DB with FK linking via provenance, dot-notation field extraction, schema inference
+- `dataset/db_loader.py` - `DatasetDbLoader`: loads Parquet data into relational DB with FK linking via provenance, dot-notation field extraction, schema inference, diff-based incremental sync (`db_load.key` + `db_load.sync: full|append`). Supports diff-based incremental sync via `db_load.key` and `--snapshot` for loading specific dates
 - `dataset/errors.py` - `DatasetError`, `CircularDependencyError`, `SourceNotFoundError`
 - `db/__init__.py` - `check_db_deps()` — verifies optional psycopg is installed for Postgres
 - `db/config.py` - `ConnectionConfig`, `DatabaseType`, `OnConflict` enums and models
@@ -164,8 +166,11 @@ Sources are topologically sorted by dependencies. `input_template` allows transf
 - Schema inference from Parquet records via `infer_table_schema()`
 - Auto-increment `id` primary key per table
 - FK linking via provenance: parent `_input_value` → child `{parent}_id` column
-- Optional `db_load` config per source: field selection, dot-notation extraction, custom table names, field exclusion
+- Optional `db_load` config per source: field selection, dot-notation extraction, custom table names, field exclusion, `key` for diff-based incremental sync
 - Topological loading order (parents before children)
+- Diff-based incremental sync: when `db_load.key` is set and table exists with >=2 snapshots, diffs the two most recent and applies INSERT/DELETE/UPDATE delta
+- `--snapshot YYYY-MM-DD` flag to load a specific snapshot date
+- `--drop-existing` forces full INSERT of latest snapshot
 
 **Dataset Storage Layout**:
 ```

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anysite-cli
-Version: 0.1.4
+Version: 0.1.6
 Summary: CLI for Anysite API - web data extraction for humans and AI agents
 Project-URL: Homepage, https://anysite.io
 Project-URL: Documentation, https://docs.anysite.io/cli
@@ -259,6 +259,8 @@ sources:
         path: ./output/companies-{{date}}.csv
         format: csv
     db_load:
+      key: _input_value                    # Unique key for incremental sync
+      sync: full                           # full (default) or append (no DELETE)
       fields: [name, url, employee_count]
 
   - id: employees
@@ -274,6 +276,8 @@ sources:
       count: 5
     refresh: always                       # Re-collect every run with --incremental
     db_load:
+      key: urn.value                       # Unique key for incremental sync
+      sync: append                         # Keep old records (no DELETE on diff)
       fields: [name, url, headline]
 
 storage:
@@ -318,9 +322,15 @@ anysite dataset query dataset.yaml --interactive
 anysite dataset stats dataset.yaml --source companies
 anysite dataset profile dataset.yaml
 
-# Load into PostgreSQL with automatic FK linking
+# Load into PostgreSQL with automatic FK linking (incremental sync with db_load.key)
+anysite dataset load-db dataset.yaml -c pg
+
+# Drop and reload from latest snapshot
 anysite dataset load-db dataset.yaml -c pg --drop-existing
 
+# Load a specific snapshot date
+anysite dataset load-db dataset.yaml -c pg --snapshot 2026-01-15
+
 # Run history and logs
 anysite dataset history my-dataset
 anysite dataset logs my-dataset --run 42
@@ -328,8 +338,9 @@ anysite dataset logs my-dataset --run 42
 # Generate cron/systemd schedule
 anysite dataset schedule dataset.yaml --incremental --load-db pg
 
-# Compare snapshots (diff two collection dates)
+# Compare snapshots (diff two collection dates, supports dot-notation keys)
 anysite dataset diff dataset.yaml --source employees --key _input_value
+anysite dataset diff dataset.yaml --source profiles --key urn.value --fields "name,headline"
 
 # Reset incremental state
 anysite dataset reset-cursor dataset.yaml

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/README.md

@@ -196,6 +196,8 @@ sources:
         path: ./output/companies-{{date}}.csv
         format: csv
     db_load:
+      key: _input_value                    # Unique key for incremental sync
+      sync: full                           # full (default) or append (no DELETE)
       fields: [name, url, employee_count]
 
   - id: employees
@@ -211,6 +213,8 @@ sources:
       count: 5
     refresh: always                       # Re-collect every run with --incremental
     db_load:
+      key: urn.value                       # Unique key for incremental sync
+      sync: append                         # Keep old records (no DELETE on diff)
       fields: [name, url, headline]
 
 storage:
@@ -255,9 +259,15 @@ anysite dataset query dataset.yaml --interactive
 anysite dataset stats dataset.yaml --source companies
 anysite dataset profile dataset.yaml
 
-# Load into PostgreSQL with automatic FK linking
+# Load into PostgreSQL with automatic FK linking (incremental sync with db_load.key)
+anysite dataset load-db dataset.yaml -c pg
+
+# Drop and reload from latest snapshot
 anysite dataset load-db dataset.yaml -c pg --drop-existing
 
+# Load a specific snapshot date
+anysite dataset load-db dataset.yaml -c pg --snapshot 2026-01-15
+
 # Run history and logs
 anysite dataset history my-dataset
 anysite dataset logs my-dataset --run 42
@@ -265,8 +275,9 @@ anysite dataset logs my-dataset --run 42
 # Generate cron/systemd schedule
 anysite dataset schedule dataset.yaml --incremental --load-db pg
 
-# Compare snapshots (diff two collection dates)
+# Compare snapshots (diff two collection dates, supports dot-notation keys)
 anysite dataset diff dataset.yaml --source employees --key _input_value
+anysite dataset diff dataset.yaml --source profiles --key urn.value --fields "name,headline"
 
 # Reset incremental state
 anysite dataset reset-cursor dataset.yaml

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "anysite-cli"
-version = "0.1.4"
+version = "0.1.6"
 description = "CLI for Anysite API - web data extraction for humans and AI agents"
 readme = "README.md"
 license = "MIT"

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/skills/anysite-cli/SKILL.md

@@ -117,6 +117,8 @@ sources:
         path: ./output/companies-{{date}}.csv
         format: csv
     db_load:
+      key: _input_value                   # Unique key for diff-based incremental sync
+      sync: full                          # full (INSERT/DELETE/UPDATE) or append (no DELETE)
       fields: [name, url, employee_count]
 
   - id: employees
@@ -189,18 +191,32 @@ anysite dataset profile dataset.yaml
 # Load all sources with FK linking
 anysite dataset load-db dataset.yaml -c pg --drop-existing
 
+# Incremental sync (uses diff when db_load.key is set)
+anysite dataset load-db dataset.yaml -c pg
+
+# Load a specific snapshot date
+anysite dataset load-db dataset.yaml -c pg --snapshot 2026-01-15
+
 # Dry run
 anysite dataset load-db dataset.yaml -c pg --dry-run
 ```
 
 `load-db` auto-creates tables with inferred schema, adds `id` primary key, and links child tables to parents via `{parent}_id` FK columns using provenance data.
 
+**Incremental sync**: When `db_load.key` is set and the table already exists with >=2 snapshots, `load-db` diffs the two most recent snapshots and applies only the delta (INSERT added, DELETE removed, UPDATE changed). Without `db_load.key`, it does a full INSERT of the latest snapshot.
+
+**Sync modes** (`db_load.sync`):
+- `full` (default) — applies INSERT, DELETE, and UPDATE from diff
+- `append` — applies INSERT and UPDATE only, skips DELETE (keeps records that disappeared from the API). Use for sources where the API returns only the latest N items (e.g., posts, activity feeds).
+
 Optional `db_load` config per source controls which fields go to DB:
 ```yaml
   - id: profiles
     endpoint: /api/linkedin/user
     db_load:
       table: people              # Custom table name
+      key: urn.value             # Unique key for diff-based incremental sync
+      sync: append               # Keep old records (no DELETE on diff)
       fields:                    # Select specific fields
         - name
         - urn.value AS urn_id    # Dot-notation extraction
@@ -241,11 +257,16 @@ anysite api /api/linkedin/user user=satyanadella -q --format jsonl \
 # Diff two most recent snapshots
 anysite dataset diff dataset.yaml --source employees --key _input_value
 
-# Diff specific dates, compare only certain fields
+# Diff with dot-notation key (for JSON fields like urn)
+anysite dataset diff dataset.yaml --source profiles --key urn.value
+
+# Diff specific dates, compare and output only certain fields
 anysite dataset diff dataset.yaml --source employees --key _input_value \
   --from 2026-01-30 --to 2026-02-01 --fields "name,headline"
 ```
 
+`--key` supports dot-notation for JSON fields (e.g., `urn.value`). `--fields` restricts both comparison and output columns.
+
 ### Step 7: History, Scheduling, and Notifications
 ```bash
 # View run history

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/skills/anysite-cli/references/dataset-guide.md

@@ -39,6 +39,8 @@ sources:
         headers: {X-Token: abc}
     db_load:                   # Database loading config
       table: custom_name       # Override table name
+      key: urn.value           # Unique key for diff-based incremental sync
+      sync: full               # full (INSERT/DELETE/UPDATE) or append (no DELETE)
       fields: [name, url]      # Fields to include
       exclude: [_input_value]  # Fields to exclude
 
@@ -244,6 +246,7 @@ anysite dataset load-db dataset.yaml -c <connection_name> [OPTIONS]
 --connection, -c TEXT    Database connection name (required)
 --source, -s TEXT        Load specific source + dependencies
 --drop-existing          Drop tables before creating
+--snapshot TEXT           Load a specific snapshot date (YYYY-MM-DD)
 --dry-run                Show plan without executing
 --quiet, -q              Suppress output
 ```
@@ -256,6 +259,29 @@ anysite dataset load-db dataset.yaml -c <connection_name> [OPTIONS]
 4. Inserts rows, tracking which `_input_value` maps to which `id`
 5. For child sources: adds `{parent_source}_id` FK column using provenance
 
+### Incremental Sync with `db_load.key`
+
+When `db_load.key` is set and the table already exists with >=2 snapshots, `load-db` uses diff-based incremental sync instead of full re-insertion:
+
+1. Compares the two most recent Parquet snapshots using `DatasetDiffer`
+2. **Added** records → INSERT into DB
+3. **Removed** records → DELETE from DB (by key) — only in `sync: full` mode
+4. **Changed** records → UPDATE modified fields (by key)
+
+This keeps the database in sync without duplicates.
+
+**Sync modes** (`db_load.sync`):
+- `full` (default) — applies INSERT, DELETE, and UPDATE from diff
+- `append` — applies INSERT and UPDATE only, skips DELETE. Use for sources where the API returns only the latest N items (e.g., posts, comments) and you want to accumulate records over time.
+
+| Scenario | Behavior |
+|----------|----------|
+| First load (table doesn't exist) | Full INSERT of latest snapshot |
+| Table exists + `db_load.key` + >=2 snapshots | Diff-based sync (INSERT/DELETE/UPDATE delta) |
+| `--drop-existing` | Drop table, full INSERT of latest snapshot |
+| `--snapshot 2026-01-15` | Full INSERT of that specific snapshot |
+| No `db_load.key` set | Full INSERT of latest snapshot (no diff) |
+
 ### db_load Config
 
 Control which fields go to the database per source:
@@ -263,6 +289,8 @@ Control which fields go to the database per source:
 ```yaml
 db_load:
   table: people                    # Custom table name (default: source ID)
+  key: urn.value                   # Unique key for diff-based incremental sync
+  sync: append                     # full (default) or append (no DELETE on diff)
   fields:                          # Explicit field list
     - name
     - url
@@ -456,8 +484,11 @@ Compare two collection snapshots to find added, removed, and changed records.
 # Compare two most recent snapshots (auto-detect dates)
 anysite dataset diff dataset.yaml --source profiles --key _input_value
 
+# Compare with dot-notation key (JSON fields)
+anysite dataset diff dataset.yaml --source profiles --key urn.value
+
 # Compare specific dates
-anysite dataset diff dataset.yaml --source profiles --key urn --from 2026-01-30 --to 2026-02-01
+anysite dataset diff dataset.yaml --source profiles --key urn.value --from 2026-01-30 --to 2026-02-01
 
 # Only compare specific fields
 anysite dataset diff dataset.yaml --source profiles --key urn --fields "name,headline,follower_count"
@@ -468,9 +499,9 @@ anysite dataset diff dataset.yaml --source profiles --key urn --format json --ou
 
 **Options:**
 - `--source, -s` (required) — source to compare
-- `--key, -k` (required) — field to match records by (e.g., `_input_value`, `urn`)
+- `--key, -k` (required) — field to match records by. Supports dot-notation for JSON fields (e.g., `urn.value`)
 - `--from` / `--to` — snapshot dates (default: two most recent)
-- `--fields, -f` — only compare these fields
+- `--fields, -f` — restrict both comparison and output to these fields
 - `--format` — output format (table, json, jsonl, csv)
 - `--output, -o` — write to file
 

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/src/anysite/dataset/collector.py

@@ -19,6 +19,7 @@ from anysite.dataset.models import DatasetConfig, DatasetSource
 from anysite.dataset.storage import (
     MetadataStore,
     get_parquet_path,
+    read_latest_parquet,
     read_parquet,
     write_parquet,
 )
@@ -412,9 +413,9 @@ async def _collect_dependent(
     if dep is None:
         raise DatasetError(f"Source {source.id} has no dependency defined")
 
-    # Read parent data
+    # Read parent data (latest snapshot only to avoid schema mismatch)
     parent_dir = base_path / "raw" / dep.from_source
-    parent_records = read_parquet(parent_dir)
+    parent_records = read_latest_parquet(parent_dir)
 
     if not parent_records:
         if not quiet:
@@ -627,7 +628,7 @@ def _count_dependent_inputs(
     if dep is None:
         return None
     parent_dir = base_path / "raw" / dep.from_source
-    parent_records = read_parquet(parent_dir)
+    parent_records = read_latest_parquet(parent_dir)
     if not parent_records:
         info = metadata.get_source_info(dep.from_source)
         return info.get("record_count") if info else None

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/src/anysite/dataset/db_loader.py

@@ -301,9 +301,13 @@ class DatasetDbLoader:
         """Diff-based incremental sync: compare two most recent snapshots, apply delta."""
         result = differ.diff(source.id, diff_key)
         total = 0
+        sync_mode = source.db_load.sync if source.db_load else "full"
 
         if dry_run:
-            return len(result.added) + len(result.removed) + len(result.changed)
+            count = len(result.added) + len(result.changed)
+            if sync_mode == "full":
+                count += len(result.removed)
+            return count
 
         # Extract key value from a record (handles dot-notation)
         def _get_key_val(record: dict[str, Any]) -> Any:
@@ -321,14 +325,15 @@ class DatasetDbLoader:
                 self.adapter.insert_batch(table_name, [row])
                 total += 1
 
-        # DELETE removed records
-        if result.removed:
+        # DELETE removed records (skipped in append mode)
+        ph = self._placeholder()
+        if result.removed and sync_mode == "full":
             safe_col = sanitize_identifier(db_key_col)
             for record in result.removed:
                 key_val = _get_key_val(record)
                 if key_val is not None:
                     self.adapter.execute(
-                        f"DELETE FROM {table_name} WHERE {safe_col} = ?",
+                        f"DELETE FROM {table_name} WHERE {safe_col} = {ph}",
                         (str(key_val),),
                     )
                     total += 1
@@ -350,14 +355,14 @@ class DatasetDbLoader:
                 for field_name in changed_fields:
                     new_val = record.get(field_name)
                     safe_field = sanitize_identifier(field_name)
-                    set_parts.append(f"{safe_field} = ?")
+                    set_parts.append(f"{safe_field} = {ph}")
                     params.append(new_val)
 
                 params.append(str(key_val))
                 sql = (
                     f"UPDATE {table_name} "
                     f"SET {', '.join(set_parts)} "
-                    f"WHERE {safe_col} = ?"
+                    f"WHERE {safe_col} = {ph}"
                 )
                 self.adapter.execute(sql, tuple(params))
                 total += 1
@@ -371,6 +376,12 @@ class DatasetDbLoader:
                 return other.dependency.field
         return None
 
+    def _placeholder(self) -> str:
+        """Get the parameter placeholder for the dialect."""
+        if self._dialect == "postgres":
+            return "%s"
+        return "?"
+
     def _auto_id_type(self) -> str:
         """Get the auto-increment ID column type for the dialect."""
         if self._dialect == "postgres":

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/src/anysite/dataset/differ.py

@@ -344,6 +344,9 @@ class DatasetDiffer:
                 old_val = record.get(old_key)
                 if _values_differ(new_val, old_val):
                     changed_fields.append(col)
+            # Fallback: DuckDB detected a change but Python comparison missed it
+            if not changed_fields:
+                changed_fields = list(compare_fields)
             record["_changed_fields"] = changed_fields
 
         return records
@@ -377,6 +380,15 @@ def _values_differ(a: Any, b: Any) -> bool:
             return json.loads(a) != json.loads(b)
         except (json.JSONDecodeError, ValueError):
             pass
+    # Handle complex types (dict, list) — compare via JSON serialization
+    # to catch differences DuckDB sees but Python equality misses
+    if isinstance(a, (dict, list)) or isinstance(b, (dict, list)):
+        try:
+            return json.dumps(a, sort_keys=True, default=str) != json.dumps(
+                b, sort_keys=True, default=str
+            )
+        except (TypeError, ValueError):
+            pass
     return True
 
 
@@ -452,6 +464,8 @@ def format_diff_records(
 
     for record in result.changed:
         row: dict[str, Any] = {"_diff": "changed"}
+        changed_fields = record.get("_changed_fields", [])
+        row["_changed_fields"] = changed_fields
         for k, v in record.items():
             if k == "_changed_fields":
                 continue

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/src/anysite/dataset/models.py

@@ -82,6 +82,10 @@ class DbLoadConfig(BaseModel):
 
     table: str | None = Field(default=None, description="Override table name (default: source id)")
     key: str | None = Field(default=None, description="Unique key field for diff-based DB sync (e.g., urn.value)")
+    sync: Literal["full", "append"] = Field(
+        default="full",
+        description="Sync mode: 'full' applies INSERT/DELETE/UPDATE, 'append' skips DELETE (keeps old records)",
+    )
     fields: list[str] = Field(default_factory=list, description="Fields to include (empty = all)")
     exclude: list[str] = Field(
         default_factory=lambda: ["_input_value", "_parent_source"],

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/src/anysite/dataset/storage.py

@@ -75,7 +75,7 @@ def read_parquet(path: Path) -> list[dict[str, Any]]:
         tables = [pq.read_table(f) for f in files]
         import pyarrow as pa
 
-        table = pa.concat_tables(tables)
+        table = pa.concat_tables(tables, promote_options="permissive")
     else:
         if not path.exists():
             return []
@@ -84,6 +84,26 @@ def read_parquet(path: Path) -> list[dict[str, Any]]:
     return table.to_pylist()
 
 
+def read_latest_parquet(path: Path) -> list[dict[str, Any]]:
+    """Read records from the most recent Parquet snapshot in a directory.
+
+    Unlike ``read_parquet(dir)``, this reads only the latest file, avoiding
+    schema mismatch errors when snapshots have different column types.
+
+    Args:
+        path: Directory containing dated .parquet files.
+
+    Returns:
+        List of dicts from the newest snapshot, or [] if none found.
+    """
+    if not path.is_dir():
+        return read_parquet(path)
+    files = sorted(path.glob("*.parquet"))
+    if not files:
+        return []
+    return read_parquet(files[-1])
+
+
 def get_source_dir(base_path: Path, source_id: str) -> Path:
     """Get the raw data directory for a source."""
     return base_path / "raw" / source_id

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/tests/test_dataset/test_db_loader.py

@@ -1,7 +1,6 @@
 """Tests for dataset DB loader with SQLite in-memory adapter."""
 
 import json
-
 import pytest
 
 from anysite.dataset.db_loader import DatasetDbLoader, _extract_dot_value, _filter_record
@@ -636,3 +635,201 @@ class TestDropExistingWithDiffKey:
             assert len(rows) == 2
             assert rows[0]["name"] == "Alice"
             assert rows[0]["score"] == 95
+
+
+class TestAppendSyncMode:
+    """Test sync: append skips DELETE but still INSERTs and UPDATEs."""
+
+    def _setup_two_snapshots(self, tmp_path, source_id, old_records, new_records):
+        source_dir = get_source_dir(tmp_path / "data", source_id)
+        write_parquet(old_records, source_dir / "2026-01-01.parquet")
+        write_parquet(new_records, source_dir / "2026-01-02.parquet")
+
+    def test_append_keeps_removed_records(self, tmp_path):
+        """With sync: append, records missing from new snapshot are NOT deleted."""
+        sources = [
+            DatasetSource(
+                id="posts", endpoint="/api/posts",
+                db_load=DbLoadConfig(key="uid", sync="append"),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "posts",
+            old_records=[
+                {"uid": "a", "text": "Hello", "likes": 10},
+                {"uid": "b", "text": "World", "likes": 5},
+                {"uid": "c", "text": "Bye", "likes": 3},
+            ],
+            new_records=[
+                {"uid": "a", "text": "Hello", "likes": 15},  # changed
+                {"uid": "d", "text": "New post", "likes": 0},  # added
+                # b and c removed from snapshot
+            ],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            # Set up DB with old data
+            source_dir = get_source_dir(tmp_path / "data", "posts")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "posts", source_dir / "2026-01-01.parquet"
+            )
+            assert len(adapter.fetch_all("SELECT * FROM posts")) == 3
+
+            # Diff sync with append mode
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+            # 1 added + 1 changed = 2 (no deletes)
+            assert results["posts"] == 2
+
+            rows = adapter.fetch_all("SELECT * FROM posts ORDER BY uid")
+            assert len(rows) == 4  # 3 original + 1 added, none deleted
+            uids = [r["uid"] for r in rows]
+            assert "a" in uids  # updated
+            assert "b" in uids  # kept (not deleted)
+            assert "c" in uids  # kept (not deleted)
+            assert "d" in uids  # added
+
+            # Verify update applied
+            a_row = [r for r in rows if r["uid"] == "a"][0]
+            assert a_row["likes"] == 15
+
+    def test_full_sync_deletes_removed_records(self, tmp_path):
+        """With sync: full (default), removed records ARE deleted."""
+        sources = [
+            DatasetSource(
+                id="posts", endpoint="/api/posts",
+                db_load=DbLoadConfig(key="uid", sync="full"),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "posts",
+            old_records=[
+                {"uid": "a", "text": "Hello", "likes": 10},
+                {"uid": "b", "text": "World", "likes": 5},
+            ],
+            new_records=[
+                {"uid": "a", "text": "Hello", "likes": 15},
+                # b removed
+            ],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "posts")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "posts", source_dir / "2026-01-01.parquet"
+            )
+            assert len(adapter.fetch_all("SELECT * FROM posts")) == 2
+
+            loader2 = DatasetDbLoader(config, adapter)
+            results = loader2.load_all()
+            assert results["posts"] == 2  # 1 changed + 1 removed
+
+            rows = adapter.fetch_all("SELECT * FROM posts")
+            assert len(rows) == 1
+            assert rows[0]["uid"] == "a"
+
+
+class TestPostgresPlaceholders:
+    """Test that diff-based sync uses %s placeholders for postgres dialect."""
+
+    def _setup_two_snapshots(self, tmp_path, source_id, old_records, new_records):
+        source_dir = get_source_dir(tmp_path / "data", source_id)
+        write_parquet(old_records, source_dir / "2026-01-01.parquet")
+        write_parquet(new_records, source_dir / "2026-01-02.parquet")
+
+    def test_delete_uses_percent_s(self, tmp_path):
+        """DELETE query uses %s placeholder for postgres."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name", sync="full"),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[
+                {"name": "Alice", "score": 90},
+                {"name": "Bob", "score": 80},
+            ],
+            new_records=[{"name": "Alice", "score": 90}],
+        )
+
+        # Use real SQLite adapter for initial load, then mock for diff sync
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            # Patch dialect to postgres and spy on execute
+            loader2 = DatasetDbLoader(config, adapter)
+            loader2._dialect = "postgres"
+            original_execute = adapter.execute
+            calls = []
+
+            def spy_execute(sql, params=None):
+                calls.append((sql, params))
+                # Replace %s with ? for SQLite execution
+                original_execute(sql.replace("%s", "?"), params)
+
+            adapter.execute = spy_execute
+            loader2.load_all()
+
+            # Verify DELETE used %s
+            delete_calls = [c for c in calls if "DELETE" in c[0]]
+            assert len(delete_calls) == 1
+            assert "%s" in delete_calls[0][0]
+            assert "?" not in delete_calls[0][0]
+
+    def test_update_uses_percent_s(self, tmp_path):
+        """UPDATE query uses %s placeholders for postgres."""
+        sources = [
+            DatasetSource(
+                id="items", endpoint="/api/items",
+                db_load=DbLoadConfig(key="name"),
+            ),
+        ]
+        config = _make_config(tmp_path, sources)
+
+        self._setup_two_snapshots(
+            tmp_path, "items",
+            old_records=[{"name": "Alice", "score": 90}],
+            new_records=[{"name": "Alice", "score": 95}],
+        )
+
+        adapter = _sqlite_adapter()
+        with adapter:
+            source_dir = get_source_dir(tmp_path / "data", "items")
+            loader = DatasetDbLoader(config, adapter)
+            loader._full_insert(
+                sources[0], "items", source_dir / "2026-01-01.parquet"
+            )
+
+            loader2 = DatasetDbLoader(config, adapter)
+            loader2._dialect = "postgres"
+            original_execute = adapter.execute
+            calls = []
+
+            def spy_execute(sql, params=None):
+                calls.append((sql, params))
+                original_execute(sql.replace("%s", "?"), params)
+
+            adapter.execute = spy_execute
+            loader2.load_all()
+
+            update_calls = [c for c in calls if "UPDATE" in c[0]]
+            assert len(update_calls) == 1
+            assert "%s" in update_calls[0][0]
+            assert "?" not in update_calls[0][0]

{anysite_cli-0.1.4 → anysite_cli-0.1.6}/tests/test_dataset/test_differ.py

@@ -8,6 +8,7 @@ import pytest
 from anysite.dataset.differ import (
     DatasetDiffer,
     DiffResult,
+    _values_differ,
     format_diff_records,
     format_diff_table,
 )
@@ -336,7 +337,8 @@ class TestFormatDiffRecords:
         assert len(rows) == 1
         assert rows[0]["name"] == "Alice Updated"
         assert rows[0]["name__old"] == "Alice"
-        assert "_changed_fields" not in rows[0]
+        assert "_changed_fields" in rows[0]
+        assert rows[0]["_changed_fields"] == ["name"]
 
 
 class TestDotNotationKey:
@@ -481,3 +483,92 @@ class TestOutputFieldFiltering:
         assert "name" not in rows[0]
         assert "name__old" not in rows[0]
         assert "urn" in rows[0]  # key always included
+
+
+class TestChangedFieldsDetail:
+    """Verify _changed_fields and __old columns appear in output."""
+
+    def test_changed_fields_in_json_output(self):
+        """format_diff_records includes _changed_fields for changed records."""
+        result = DiffResult(
+            source_id="items",
+            from_date=date(2026, 1, 1),
+            to_date=date(2026, 1, 2),
+            key="id",
+            changed=[{
+                "id": "1",
+                "name": "Alice Updated",
+                "name__old": "Alice",
+                "score": 90,
+                "score__old": 90,
+                "_changed_fields": ["name"],
+            }],
+        )
+
+        rows = format_diff_records(result)
+        assert len(rows) == 1
+        assert "_changed_fields" in rows[0]
+        assert rows[0]["_changed_fields"] == ["name"]
+        assert "name__old" in rows[0]
+        assert rows[0]["name__old"] == "Alice"
+
+    def test_changed_fields_not_in_table_output(self):
+        """format_diff_table uses old→new arrows instead of _changed_fields."""
+        result = DiffResult(
+            source_id="items",
+            from_date=date(2026, 1, 1),
+            to_date=date(2026, 1, 2),
+            key="id",
+            changed=[{
+                "id": "1",
+                "name": "Bob",
+                "name__old": "Alice",
+                "score": 90,
+                "score__old": 90,
+                "_changed_fields": ["name"],
+            }],
+        )
+
+        rows = format_diff_table(result)
+        assert len(rows) == 1
+        assert "_changed_fields" not in rows[0]
+        assert "→" in rows[0]["name"]
+        assert rows[0]["score"] == 90  # unchanged, no arrow
+
+    def test_values_differ_with_dicts(self):
+        """_values_differ handles dict comparison correctly."""
+        assert not _values_differ({"a": 1}, {"a": 1})
+        assert _values_differ({"a": 1}, {"a": 2})
+        assert not _values_differ({"b": 2, "a": 1}, {"a": 1, "b": 2})  # key order
+
+    def test_values_differ_with_lists(self):
+        """_values_differ handles list comparison."""
+        assert not _values_differ([1, 2], [1, 2])
+        assert _values_differ([1, 2], [1, 3])
+
+    def test_values_differ_dict_vs_string(self):
+        """_values_differ handles dict vs JSON string comparison."""
+        assert _values_differ({"a": 1}, '{"a": 1}')
+
+    def test_fallback_all_compare_fields(self, tmp_path):
+        """When Python can't identify changed fields, all compare fields are marked."""
+        source_dir = tmp_path / "raw" / "items"
+        source_dir.mkdir(parents=True)
+
+        # Write old and new snapshots with a value DuckDB sees as different
+        write_parquet(
+            [{"id": "1", "data": json.dumps({"x": 1})}],
+            source_dir / "2026-01-01.parquet",
+        )
+        write_parquet(
+            [{"id": "1", "data": json.dumps({"x": 1, "y": 2})}],
+            source_dir / "2026-01-02.parquet",
+        )
+
+        differ = DatasetDiffer(tmp_path)
+        result = differ.diff("items", "id")
+
+        assert len(result.changed) == 1
+        record = result.changed[0]
+        assert "_changed_fields" in record
+        assert len(record["_changed_fields"]) > 0