anysite-cli 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

anysite/dataset/cli.py

@@ -357,6 +357,10 @@ def load_db(
         bool,
         typer.Option("--quiet", "-q", help="Suppress progress output"),
     ] = False,
+    snapshot: Annotated[
+        str | None,
+        typer.Option("--snapshot", help="Load a specific snapshot date (YYYY-MM-DD)"),
+    ] = None,
 ) -> None:
     """Load collected Parquet data into a relational database with FK linking."""
     config = _load_config(config_path)
@@ -379,6 +383,7 @@ def load_db(
                 source_filter=source,
                 drop_existing=drop_existing,
                 dry_run=dry_run,
+                snapshot=snapshot,
             )
         except Exception as e:
             typer.echo(f"Load error: {e}", err=True)
@@ -413,6 +418,121 @@ def load_db(
         )
 
 
+@app.command("diff")
+def diff_cmd(
+    config_path: Annotated[
+        Path,
+        typer.Argument(help="Path to dataset.yaml"),
+    ],
+    source: Annotated[
+        str,
+        typer.Option("--source", "-s", help="Source to compare"),
+    ],
+    key: Annotated[
+        str,
+        typer.Option("--key", "-k", help="Field to match records by (e.g., _input_value, urn)"),
+    ],
+    from_date: Annotated[
+        str | None,
+        typer.Option("--from", help="Older snapshot date (YYYY-MM-DD)"),
+    ] = None,
+    to_date: Annotated[
+        str | None,
+        typer.Option("--to", help="Newer snapshot date (YYYY-MM-DD)"),
+    ] = None,
+    fields: Annotated[
+        str | None,
+        typer.Option("--fields", "-f", help="Only compare these fields (comma-separated)"),
+    ] = None,
+    format: Annotated[
+        str,
+        typer.Option("--format", help="Output format: table, json, jsonl, csv"),
+    ] = "table",
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="Write output to file"),
+    ] = None,
+    quiet: Annotated[
+        bool,
+        typer.Option("--quiet", "-q", help="Suppress summary, only output data"),
+    ] = False,
+) -> None:
+    """Compare two snapshots of a source to show added, removed, and changed records."""
+    from datetime import date as date_type
+
+    from anysite.dataset.differ import (
+        DatasetDiffer,
+        format_diff_records,
+        format_diff_table,
+    )
+
+    config = _load_config(config_path)
+
+    # Validate source exists
+    src = config.get_source(source)
+    if src is None:
+        typer.echo(f"Error: source '{source}' not found in dataset", err=True)
+        raise typer.Exit(1)
+
+    differ = DatasetDiffer(config.storage_path())
+
+    # Parse dates
+    parsed_from = None
+    parsed_to = None
+    try:
+        if from_date:
+            parsed_from = date_type.fromisoformat(from_date)
+        if to_date:
+            parsed_to = date_type.fromisoformat(to_date)
+    except ValueError as e:
+        typer.echo(f"Error: invalid date format: {e}", err=True)
+        raise typer.Exit(1) from None
+
+    # Parse fields
+    field_list = None
+    if fields:
+        field_list = [f.strip() for f in fields.split(",") if f.strip()]
+
+    try:
+        result = differ.diff(
+            source,
+            key,
+            from_date=parsed_from,
+            to_date=parsed_to,
+            fields=field_list,
+        )
+    except DatasetError as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1) from None
+
+    # Print summary unless quiet
+    if not quiet:
+        console = Console()
+        console.print(
+            f"\n[bold]Diff: {source}[/bold] "
+            f"({result.from_date.isoformat()} → {result.to_date.isoformat()})\n"
+        )
+        console.print(f"  [green]Added:[/green]     {len(result.added)}")
+        console.print(f"  [red]Removed:[/red]   {len(result.removed)}")
+        console.print(f"  [yellow]Changed:[/yellow]   {len(result.changed)}")
+        console.print(f"  Unchanged: {result.unchanged_count}")
+        console.print()
+
+    if not result.has_changes:
+        if not quiet:
+            Console().print("[dim]No changes detected.[/dim]")
+        return
+
+    # Format and output
+    rows = (
+        format_diff_table(result, output_fields=field_list)
+        if format == "table"
+        else format_diff_records(result, output_fields=field_list)
+    )
+
+    _output_results(rows, format, output)
+
+
 @app.command("history")
 def history(
     name: Annotated[

anysite/dataset/collector.py

@@ -43,6 +43,7 @@ class CollectionPlan:
         params: dict[str, Any] | None = None,
         dependency: str | None = None,
         estimated_requests: int | None = None,
+        refresh: str = "auto",
     ) -> None:
         self.steps.append({
             "source": source_id,
@@ -51,6 +52,7 @@ class CollectionPlan:
             "params": params or {},
             "dependency": dependency,
             "estimated_requests": estimated_requests,
+            "refresh": refresh,
         })
 
 
@@ -116,8 +118,8 @@ async def collect_dataset(
 
     try:
         for source in ordered:
-            # Check incremental skip
-            if incremental:
+            # Check incremental skip (refresh: always bypasses this)
+            if incremental and source.refresh != "always":
                 parquet_path = get_parquet_path(base_path, source.id, today)
                 if parquet_path.exists():
                     if not quiet:
@@ -276,8 +278,8 @@ async def _collect_from_file(
             print_warning(f"No values extracted from {file_path}")
         return []
 
-    # Filter already-collected inputs in incremental mode
-    if incremental and metadata:
+    # Filter already-collected inputs in incremental mode (refresh: always bypasses)
+    if incremental and source.refresh != "always" and metadata:
         already = metadata.get_collected_inputs(source.id)
         if already:
             original = len(values)
@@ -432,8 +434,8 @@ async def _collect_dependent(
             f"Source {source.id} has a dependency but no input_key defined"
         )
 
-    # Filter already-collected inputs in incremental mode
-    if incremental and metadata:
+    # Filter already-collected inputs in incremental mode (refresh: always bypasses)
+    if incremental and source.refresh != "always" and metadata:
         already = metadata.get_collected_inputs(source.id)
         if already:
             original = len(values)
@@ -579,7 +581,7 @@ def _build_plan(
     plan = CollectionPlan()
 
     for source in ordered:
-        if incremental:
+        if incremental and source.refresh != "always":
             parquet_path = get_parquet_path(base_path, source.id, today)
             if parquet_path.exists():
                 continue
@@ -592,6 +594,7 @@ def _build_plan(
                 kind="from_file",
                 params={"file": source.from_file, "field": source.file_field},
                 estimated_requests=est,
+                refresh=source.refresh,
             )
         elif source.dependency is None:
             plan.add_step(
@@ -600,6 +603,7 @@ def _build_plan(
                 kind="independent",
                 params=source.params,
                 estimated_requests=1,
+                refresh=source.refresh,
             )
         else:
             est = _count_dependent_inputs(source, base_path, metadata)
@@ -609,6 +613,7 @@ def _build_plan(
                 kind="dependent",
                 dependency=source.dependency.from_source,
                 estimated_requests=est,
+                refresh=source.refresh,
             )
 
     return plan
@@ -665,11 +670,14 @@ def _print_plan(plan: CollectionPlan) -> dict[str, int]:
     table.add_column("Est. Requests")
 
     for i, step in enumerate(plan.steps, 1):
+        kind = step["kind"]
+        if step.get("refresh") == "always":
+            kind += " (refresh)"
         table.add_row(
             str(i),
             step["source"],
             step["endpoint"],
-            step["kind"],
+            kind,
             step.get("dependency") or "-",
             str(step.get("estimated_requests") or "?"),
         )

anysite/dataset/db_loader.py

@@ -3,12 +3,18 @@
 from __future__ import annotations
 
 import json
+import logging
+from datetime import date
+from pathlib import Path
 from typing import Any
 
 from anysite.dataset.models import DatasetConfig, DatasetSource
 from anysite.dataset.storage import get_source_dir, read_parquet
 from anysite.db.adapters.base import DatabaseAdapter
 from anysite.db.schema.inference import infer_table_schema
+from anysite.db.utils.sanitize import sanitize_identifier
+
+logger = logging.getLogger(__name__)
 
 
 def _get_dialect(adapter: DatabaseAdapter) -> str:
@@ -86,15 +92,31 @@ def _filter_record(
         return {k: v for k, v in record.items() if k not in exclude}
 
 
+def _get_latest_parquet(base_path: Path, source_id: str) -> Path | None:
+    """Return the path to the most recent snapshot for a source."""
+    source_dir = get_source_dir(base_path, source_id)
+    if not source_dir.exists():
+        return None
+    files = sorted(source_dir.glob("*.parquet"))
+    return files[-1] if files else None
+
+
+def _get_snapshot_for_date(base_path: Path, source_id: str, d: date) -> Path | None:
+    """Return the parquet path for a specific snapshot date."""
+    source_dir = get_source_dir(base_path, source_id)
+    path = source_dir / f"{d.isoformat()}.parquet"
+    return path if path.exists() else None
+
+
 class DatasetDbLoader:
     """Load dataset Parquet data into a relational database.
 
-    Handles:
-    - Schema inference from Parquet records
-    - Auto-increment primary keys (``id`` column)
-    - Foreign key linking via provenance ``_input_value`` column
-    - Dot-notation field extraction for JSON columns
-    - Topological loading order (parents before children)
+    Supports diff-based incremental sync when ``db_load.key`` is configured:
+    compares the two most recent snapshots and applies INSERT/DELETE/UPDATE
+    to keep the database in sync.
+
+    Falls back to full INSERT of the latest snapshot when no key is set
+    or when the table doesn't exist yet.
     """
 
     def __init__(
@@ -115,16 +137,18 @@ class DatasetDbLoader:
         source_filter: str | None = None,
         drop_existing: bool = False,
         dry_run: bool = False,
+        snapshot: str | None = None,
     ) -> dict[str, int]:
         """Load all sources into the database in dependency order.
 
         Args:
             source_filter: Only load this source (and dependencies).
-            drop_existing: Drop tables before creating.
+            drop_existing: Drop tables before creating, then full INSERT latest.
             dry_run: Show plan without executing.
+            snapshot: Load a specific snapshot date (YYYY-MM-DD).
 
         Returns:
-            Mapping of source_id to number of rows loaded.
+            Mapping of source_id to number of rows loaded/affected.
         """
         sources = self.config.topological_sort()
 
@@ -139,6 +163,7 @@ class DatasetDbLoader:
                 source,
                 drop_existing=drop_existing,
                 dry_run=dry_run,
+                snapshot=snapshot,
             )
             results[source.id] = count
 
@@ -150,18 +175,64 @@ class DatasetDbLoader:
         *,
         drop_existing: bool = False,
         dry_run: bool = False,
+        snapshot: str | None = None,
     ) -> int:
-        """Load a single source into the database."""
-        source_dir = get_source_dir(self.base_path, source.id)
-        if not source_dir.exists() or not any(source_dir.glob("*.parquet")):
+        """Load a single source into the database.
+
+        Strategy:
+        1. ``drop_existing``: drop table → full INSERT of latest snapshot
+        2. ``snapshot``: full INSERT of that specific snapshot
+        3. Table doesn't exist: full INSERT of latest snapshot
+        4. Table exists + ``db_load.key`` set + ≥2 snapshots: diff-based sync
+        5. Fallback: full INSERT of latest snapshot
+        """
+        table_name = _table_name_for(source)
+
+        # Handle drop_existing
+        if drop_existing and self.adapter.table_exists(table_name):
+            self.adapter.execute(f"DROP TABLE {table_name}")
+
+        # Determine which parquet to load
+        if snapshot:
+            snapshot_date = date.fromisoformat(snapshot)
+            parquet_path = _get_snapshot_for_date(self.base_path, source.id, snapshot_date)
+            if parquet_path is None:
+                return 0
+            return self._full_insert(source, table_name, parquet_path, dry_run=dry_run)
+
+        # Check if we can do diff-based sync
+        diff_key = source.db_load.key if source.db_load else None
+        table_exists = self.adapter.table_exists(table_name)
+
+        if diff_key and table_exists and not drop_existing:
+            from anysite.dataset.differ import DatasetDiffer
+            differ = DatasetDiffer(self.base_path)
+            dates = differ.available_dates(source.id)
+
+            if len(dates) >= 2:
+                return self._diff_sync(
+                    source, table_name, diff_key, differ, dates, dry_run=dry_run
+                )
+
+        # Fallback: full INSERT of latest snapshot
+        latest = _get_latest_parquet(self.base_path, source.id)
+        if latest is None:
             return 0
+        return self._full_insert(source, table_name, latest, dry_run=dry_run)
 
-        raw_records = read_parquet(source_dir)
+    def _full_insert(
+        self,
+        source: DatasetSource,
+        table_name: str,
+        parquet_path: Path,
+        *,
+        dry_run: bool = False,
+    ) -> int:
+        """Full INSERT: read parquet, transform, create table if needed, insert all rows."""
+        raw_records = read_parquet(parquet_path)
         if not raw_records:
             return 0
 
-        table_name = _table_name_for(source)
-
         # Determine parent info for FK linking
         parent_source_id = None
         parent_fk_col = None
@@ -174,7 +245,6 @@ class DatasetDbLoader:
         for record in raw_records:
             row = _filter_record(record, source)
 
-            # Add FK column if this is a dependent source
             if parent_source_id and parent_fk_col:
                 input_val = record.get("_input_value")
                 parent_map = self._value_to_id.get(parent_source_id, {})
@@ -189,17 +259,12 @@ class DatasetDbLoader:
             return len(rows)
 
         # Determine the lookup field for children to reference this source
-        # This is the field that child dependencies extract from this source
         lookup_field = self._get_child_lookup_field(source)
 
-        # Create table
-        if drop_existing and self.adapter.table_exists(table_name):
-            self.adapter.execute(f"DROP TABLE {table_name}")
-
+        # Create table if needed
         if not self.adapter.table_exists(table_name):
             schema = infer_table_schema(table_name, rows)
             sql_types = schema.to_sql_types(self._dialect)
-            # Add auto-increment id column
             col_defs = {"id": self._auto_id_type()}
             col_defs.update(sql_types)
             self.adapter.create_table(table_name, col_defs, primary_key="id")
@@ -208,10 +273,8 @@ class DatasetDbLoader:
         value_map: dict[str, int] = {}
         for i, row in enumerate(rows):
             self.adapter.insert_batch(table_name, [row])
-            # Get the last inserted id
             last_id = self._get_last_id(table_name)
 
-            # Build value→id map for child sources
             if lookup_field and last_id is not None:
                 raw_record = raw_records[i]
                 lookup_val = _extract_dot_value(raw_record, lookup_field)
@@ -225,6 +288,82 @@ class DatasetDbLoader:
 
         return len(rows)
 
+    def _diff_sync(
+        self,
+        source: DatasetSource,
+        table_name: str,
+        diff_key: str,
+        differ: Any,
+        dates: list[date],
+        *,
+        dry_run: bool = False,
+    ) -> int:
+        """Diff-based incremental sync: compare two most recent snapshots, apply delta."""
+        result = differ.diff(source.id, diff_key)
+        total = 0
+
+        if dry_run:
+            return len(result.added) + len(result.removed) + len(result.changed)
+
+        # Extract key value from a record (handles dot-notation)
+        def _get_key_val(record: dict[str, Any]) -> Any:
+            if "." in diff_key:
+                return _extract_dot_value(record, diff_key)
+            return record.get(diff_key)
+
+        # Determine the DB column name for the key
+        db_key_col = diff_key.replace(".", "_")
+
+        # INSERT added records
+        if result.added:
+            for record in result.added:
+                row = _filter_record(record, source)
+                self.adapter.insert_batch(table_name, [row])
+                total += 1
+
+        # DELETE removed records
+        if result.removed:
+            safe_col = sanitize_identifier(db_key_col)
+            for record in result.removed:
+                key_val = _get_key_val(record)
+                if key_val is not None:
+                    self.adapter.execute(
+                        f"DELETE FROM {table_name} WHERE {safe_col} = ?",
+                        (str(key_val),),
+                    )
+                    total += 1
+
+        # UPDATE changed records
+        if result.changed:
+            safe_col = sanitize_identifier(db_key_col)
+            for record in result.changed:
+                key_val = _get_key_val(record)
+                if key_val is None:
+                    continue
+                changed_fields = record.get("_changed_fields", [])
+                if not changed_fields:
+                    continue
+
+                # Build SET clause from changed fields
+                set_parts = []
+                params: list[Any] = []
+                for field_name in changed_fields:
+                    new_val = record.get(field_name)
+                    safe_field = sanitize_identifier(field_name)
+                    set_parts.append(f"{safe_field} = ?")
+                    params.append(new_val)
+
+                params.append(str(key_val))
+                sql = (
+                    f"UPDATE {table_name} "
+                    f"SET {', '.join(set_parts)} "
+                    f"WHERE {safe_col} = ?"
+                )
+                self.adapter.execute(sql, tuple(params))
+                total += 1
+
+        return total
+
     def _get_child_lookup_field(self, source: DatasetSource) -> str | None:
         """Find which field children use to reference this source."""
         for other in self.config.sources:

anysite/dataset/differ.py

@@ -0,0 +1,496 @@
+"""Compare two dataset snapshots to find added, removed, and changed records."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+from anysite.dataset.errors import DatasetError
+from anysite.dataset.storage import get_source_dir
+
+
+def _build_key_expr(key: str, all_columns: list[str]) -> tuple[str, str]:
+    """Build a DuckDB key expression, supporting dot-notation for JSON fields.
+
+    Returns:
+        (key_expr, key_alias) — the SQL expression and a display alias.
+        For simple keys: ('"field"', 'field')
+        For dot-notation: ("json_extract_string(\"urn\", '$.value')", 'urn.value')
+    """
+    if "." not in key:
+        if key not in all_columns:
+            raise DatasetError(
+                f"Key field '{key}' not found. "
+                f"Available: {', '.join(all_columns)}"
+            )
+        return f'"{key}"', key
+
+    root, rest = key.split(".", 1)
+    if root not in all_columns:
+        raise DatasetError(
+            f"Root field '{root}' (from key '{key}') not found. "
+            f"Available: {', '.join(all_columns)}"
+        )
+    return f"json_extract_string(\"{root}\", '$.{rest}')", key
+
+
+@dataclass
+class DiffResult:
+    """Result of comparing two dataset snapshots."""
+
+    source_id: str
+    from_date: date
+    to_date: date
+    key: str
+    added: list[dict[str, Any]] = field(default_factory=list)
+    removed: list[dict[str, Any]] = field(default_factory=list)
+    changed: list[dict[str, Any]] = field(default_factory=list)
+    unchanged_count: int = 0
+    fields: list[str] | None = field(default=None)
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.added or self.removed or self.changed)
+
+
+class DatasetDiffer:
+    """Compare two Parquet snapshots for a dataset source."""
+
+    def __init__(self, base_path: Path) -> None:
+        self.base_path = base_path
+
+    def available_dates(self, source_id: str) -> list[date]:
+        """List available snapshot dates for a source, sorted ascending."""
+        source_dir = get_source_dir(self.base_path, source_id)
+        if not source_dir.exists():
+            return []
+
+        dates: list[date] = []
+        for f in sorted(source_dir.glob("*.parquet")):
+            try:
+                dates.append(date.fromisoformat(f.stem))
+            except ValueError:
+                continue
+        return dates
+
+    def diff(
+        self,
+        source_id: str,
+        key: str,
+        *,
+        from_date: date | None = None,
+        to_date: date | None = None,
+        fields: list[str] | None = None,
+    ) -> DiffResult:
+        """Compare two snapshots using DuckDB.
+
+        Args:
+            source_id: Source to compare.
+            key: Field to match records by.  Supports dot-notation for
+                JSON fields (e.g., ``urn.value``).
+            from_date: Older snapshot date (default: second-to-last).
+            to_date: Newer snapshot date (default: latest).
+            fields: Only compare (and output) these fields (default: all).
+
+        Returns:
+            DiffResult with added, removed, changed lists.
+        """
+        from_date, to_date = self._resolve_dates(source_id, from_date, to_date)
+        source_dir = get_source_dir(self.base_path, source_id)
+        old_path = source_dir / f"{from_date.isoformat()}.parquet"
+        new_path = source_dir / f"{to_date.isoformat()}.parquet"
+
+        if not old_path.exists():
+            raise DatasetError(
+                f"No snapshot for {source_id} on {from_date.isoformat()}"
+            )
+        if not new_path.exists():
+            raise DatasetError(
+                f"No snapshot for {source_id} on {to_date.isoformat()}"
+            )
+
+        return self._diff_with_duckdb(
+            source_id, key, old_path, new_path, from_date, to_date, fields
+        )
+
+    def _resolve_dates(
+        self,
+        source_id: str,
+        from_date: date | None,
+        to_date: date | None,
+    ) -> tuple[date, date]:
+        """Resolve from/to dates, defaulting to two most recent snapshots."""
+        if from_date and to_date:
+            return from_date, to_date
+
+        dates = self.available_dates(source_id)
+        if len(dates) < 2:
+            raise DatasetError(
+                f"Need at least 2 snapshots to diff, "
+                f"found {len(dates)} for {source_id}"
+            )
+
+        if to_date and not from_date:
+            # Find the date just before to_date
+            earlier = [d for d in dates if d < to_date]
+            if not earlier:
+                raise DatasetError(
+                    f"No snapshot before {to_date.isoformat()} for {source_id}"
+                )
+            return earlier[-1], to_date
+
+        if from_date and not to_date:
+            # Find the date just after from_date
+            later = [d for d in dates if d > from_date]
+            if not later:
+                raise DatasetError(
+                    f"No snapshot after {from_date.isoformat()} for {source_id}"
+                )
+            return from_date, later[0]
+
+        # Both None — use two most recent
+        return dates[-2], dates[-1]
+
+    def _diff_with_duckdb(
+        self,
+        source_id: str,
+        key: str,
+        old_path: Path,
+        new_path: Path,
+        from_date: date,
+        to_date: date,
+        fields: list[str] | None,
+    ) -> DiffResult:
+        """Run diff queries using DuckDB."""
+        import duckdb
+
+        conn = duckdb.connect(":memory:")
+        try:
+            conn.execute(
+                f"CREATE VIEW _old AS SELECT * FROM read_parquet('{old_path}')"
+            )
+            conn.execute(
+                f"CREATE VIEW _new AS SELECT * FROM read_parquet('{new_path}')"
+            )
+
+            # Get all column names from the new snapshot
+            info = conn.execute("DESCRIBE _new").fetchall()
+            all_columns = [col[0] for col in info]
+
+            # Build key expression (supports dot-notation)
+            key_expr, key_alias = _build_key_expr(key, all_columns)
+
+            # Determine which fields to compare
+            compare_fields = fields if fields else [
+                c for c in all_columns if c != key and c != key.split(".")[0]
+            ]
+            # Filter to fields that actually exist
+            compare_fields = [c for c in compare_fields if c in all_columns]
+
+            # Determine output columns: if fields specified, restrict to key + fields
+            if fields:
+                output_columns = [key_alias] + [
+                    f for f in fields if f in all_columns
+                ]
+            else:
+                output_columns = None  # all columns
+
+            # Added: in new but not in old
+            added_records = self._query_added_removed(
+                conn, "_new", "_old", key_expr, key_alias, all_columns, output_columns
+            )
+
+            # Removed: in old but not in new
+            removed_records = self._query_added_removed(
+                conn, "_old", "_new", key_expr, key_alias, all_columns, output_columns
+            )
+
+            # Changed: matching key, different values
+            changed_records = self._find_changed(
+                conn, key_expr, key_alias, compare_fields, all_columns, output_columns
+            )
+
+            # Count unchanged
+            total_matched = conn.execute(
+                f"SELECT COUNT(*) FROM _new n "
+                f"JOIN _old o ON ({_requalify(key_expr, 'n')}) = ({_requalify(key_expr, 'o')})"
+            ).fetchone()
+            matched_count = total_matched[0] if total_matched else 0
+            unchanged_count = matched_count - len(changed_records)
+
+            return DiffResult(
+                source_id=source_id,
+                from_date=from_date,
+                to_date=to_date,
+                key=key,
+                added=added_records,
+                removed=removed_records,
+                changed=changed_records,
+                unchanged_count=unchanged_count,
+                fields=fields,
+            )
+        finally:
+            conn.close()
+
+    @staticmethod
+    def _query_added_removed(
+        conn: Any,
+        present_view: str,
+        absent_view: str,
+        key_expr: str,
+        key_alias: str,
+        all_columns: list[str],
+        output_columns: list[str] | None,
+    ) -> list[dict[str, Any]]:
+        """Query records present in one view but not the other."""
+        # Build SELECT list
+        if output_columns:
+            select_parts = []
+            for col in output_columns:
+                if col == key_alias and "." in col:
+                    select_parts.append(f"{key_expr} AS \"{key_alias}\"")
+                else:
+                    select_parts.append(f'"{col}"')
+            select_clause = ", ".join(select_parts)
+        else:
+            if "." in key_alias:
+                select_clause = f"*, {key_expr} AS \"{key_alias}\""
+            else:
+                select_clause = "*"
+
+        sql = (
+            f"SELECT {select_clause} FROM {present_view} "
+            f"WHERE ({key_expr}) NOT IN (SELECT ({key_expr}) FROM {absent_view})"
+        )
+        result = conn.execute(sql)
+        columns = [desc[0] for desc in result.description]
+        rows = result.fetchall()
+        return [dict(zip(columns, row, strict=False)) for row in rows]
+
+    @staticmethod
+    def _find_changed(
+        conn: Any,
+        key_expr: str,
+        key_alias: str,
+        compare_fields: list[str],
+        all_columns: list[str],
+        output_columns: list[str] | None,
+    ) -> list[dict[str, Any]]:
+        """Find records that exist in both snapshots but have different values."""
+        if not compare_fields:
+            return []
+
+        # Build WHERE clause: any compared field differs
+        where_parts = []
+        for col in compare_fields:
+            qc = f'"{col}"'
+            where_parts.append(f"n.{qc} IS DISTINCT FROM o.{qc}")
+        where_clause = " OR ".join(where_parts)
+
+        # Build JOIN condition
+        join_key_n = _requalify(key_expr, "n")
+        join_key_o = _requalify(key_expr, "o")
+        join_cond = f"({join_key_n}) = ({join_key_o})"
+
+        # Build SELECT: key + output fields + __old for compare fields
+        if output_columns:
+            # Restricted output
+            select_parts = []
+            for col in output_columns:
+                if col == key_alias and "." in col:
+                    select_parts.append(f"{_requalify(key_expr, 'n')} AS \"{key_alias}\"")
+                else:
+                    select_parts.append(f"n.\"{col}\"")
+            for col in compare_fields:
+                # Include __old for compare fields that are in output
+                if col in [c for c in output_columns if c != key_alias]:
+                    select_parts.append(f"o.\"{col}\" AS \"{col}__old\"")
+        else:
+            # Full output
+            select_parts = []
+            if "." in key_alias:
+                select_parts.append(f"{_requalify(key_expr, 'n')} AS \"{key_alias}\"")
+            else:
+                select_parts.append(f"n.\"{key_alias}\"")
+            for col in all_columns:
+                if col == key_alias:
+                    continue
+                select_parts.append(f"n.\"{col}\"")
+            for col in compare_fields:
+                select_parts.append(f"o.\"{col}\" AS \"{col}__old\"")
+
+        select_clause = ", ".join(select_parts)
+
+        sql = (
+            f"SELECT {select_clause} FROM _new n "
+            f"JOIN _old o ON {join_cond} "
+            f"WHERE {where_clause}"
+        )
+
+        result = conn.execute(sql)
+        columns = [desc[0] for desc in result.description]
+        rows = result.fetchall()
+        records = [dict(zip(columns, row, strict=False)) for row in rows]
+
+        # Add _changed_fields metadata to each record
+        for record in records:
+            changed_fields = []
+            for col in compare_fields:
+                old_key = f"{col}__old"
+                new_val = record.get(col)
+                old_val = record.get(old_key)
+                if _values_differ(new_val, old_val):
+                    changed_fields.append(col)
+            record["_changed_fields"] = changed_fields
+
+        return records
+
+
+def _requalify(key_expr: str, prefix: str) -> str:
+    """Requalify a key expression with a table alias prefix.
+
+    For simple keys like '"field"', returns 'prefix."field"'.
+    For json_extract_string("col", '$.path'), returns
+    json_extract_string(prefix."col", '$.path').
+    """
+    if key_expr.startswith("json_extract_string("):
+        # Replace the column reference inside json_extract_string
+        inner = key_expr[len("json_extract_string("):]
+        # inner looks like: "col", '$.path')
+        col_end = inner.index(",")
+        col = inner[:col_end].strip()
+        rest = inner[col_end:]
+        return f"json_extract_string({prefix}.{col}{rest}"
+    return f"{prefix}.{key_expr}"
+
+
+def _values_differ(a: Any, b: Any) -> bool:
+    """Compare two values, treating JSON strings as equivalent to their parsed form."""
+    if a == b:
+        return False
+    # Handle JSON string comparison
+    if isinstance(a, str) and isinstance(b, str):
+        try:
+            return json.loads(a) != json.loads(b)
+        except (json.JSONDecodeError, ValueError):
+            pass
+    return True
+
+
+def format_diff_table(
+    result: DiffResult,
+    *,
+    output_fields: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """Format a DiffResult into a flat list of dicts for table/json output.
+
+    Each record gets a ``_diff`` column with value ``added``, ``removed``,
+    or ``changed``.  For changed records in table mode, modified field
+    values are formatted as ``old → new``.
+
+    Args:
+        result: The diff result.
+        output_fields: If set, only include these fields (plus ``_diff`` and key).
+    """
+    allowed = _build_allowed_set(result.key, output_fields)
+    rows: list[dict[str, Any]] = []
+
+    for record in result.added:
+        row = {"_diff": "added", **_filter_row(record, allowed)}
+        rows.append(row)
+
+    for record in result.removed:
+        row = {"_diff": "removed", **_filter_row(record, allowed)}
+        rows.append(row)
+
+    for record in result.changed:
+        row: dict[str, Any] = {"_diff": "changed"}
+        changed_fields = record.get("_changed_fields", [])
+        for k, v in record.items():
+            if k == "_changed_fields":
+                continue
+            if k.endswith("__old"):
+                continue
+            if allowed and k not in allowed:
+                continue
+            # For changed fields, format as "old → new"
+            if k in changed_fields:
+                old_val = record.get(f"{k}__old")
+                row[k] = f"{_format_val(old_val)} → {_format_val(v)}"
+            else:
+                row[k] = v
+        rows.append(row)
+
+    return rows
+
+
+def format_diff_records(
+    result: DiffResult,
+    *,
+    output_fields: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """Format a DiffResult for JSON/CSV output.
+
+    Each record gets ``_diff`` column.  Changed records include both
+    current values and ``field__old`` columns.
+
+    Args:
+        result: The diff result.
+        output_fields: If set, only include these fields (plus ``_diff``, key, and ``__old``).
+    """
+    allowed = _build_allowed_set(result.key, output_fields)
+    rows: list[dict[str, Any]] = []
+
+    for record in result.added:
+        rows.append({"_diff": "added", **_filter_row(record, allowed)})
+
+    for record in result.removed:
+        rows.append({"_diff": "removed", **_filter_row(record, allowed)})
+
+    for record in result.changed:
+        row: dict[str, Any] = {"_diff": "changed"}
+        for k, v in record.items():
+            if k == "_changed_fields":
+                continue
+            if allowed and k not in allowed and not k.endswith("__old"):
+                continue
+            if k.endswith("__old") and allowed:
+                base = k[: -len("__old")]
+                if base not in allowed:
+                    continue
+            row[k] = v
+        rows.append(row)
+
+    return rows
+
+
+def _build_allowed_set(key: str, output_fields: list[str] | None) -> set[str] | None:
+    """Build the set of allowed field names for output filtering."""
+    if not output_fields:
+        return None
+    allowed = set(output_fields)
+    allowed.add(key)
+    # Also add the root column for dot-notation keys
+    if "." in key:
+        allowed.add(key.split(".")[0])
+    return allowed
+
+
+def _filter_row(record: dict[str, Any], allowed: set[str] | None) -> dict[str, Any]:
+    """Filter a record to only allowed fields."""
+    if not allowed:
+        return record
+    return {k: v for k, v in record.items() if k in allowed}
+
+
+def _format_val(v: Any) -> str:
+    """Format a value for display, truncating long strings."""
+    if v is None:
+        return "null"
+    s = str(v)
+    if len(s) > 40:
+        return s[:37] + "..."
+    return s

anysite/dataset/models.py

@@ -81,6 +81,7 @@ class DbLoadConfig(BaseModel):
     """Configuration for loading a source into a relational database."""
 
     table: str | None = Field(default=None, description="Override table name (default: source id)")
+    key: str | None = Field(default=None, description="Unique key field for diff-based DB sync (e.g., urn.value)")
     fields: list[str] = Field(default_factory=list, description="Fields to include (empty = all)")
     exclude: list[str] = Field(
         default_factory=lambda: ["_input_value", "_parent_source"],
@@ -129,6 +130,10 @@ class DatasetSource(BaseModel):
         default_factory=list,
         description="Export destinations (file/webhook) applied after Parquet write",
     )
+    refresh: Literal["auto", "always"] = Field(
+        default="auto",
+        description="Refresh mode: 'auto' uses incremental caching, 'always' re-collects every run",
+    )
 
     @field_validator("endpoint")
     @classmethod

{anysite_cli-0.1.2.dist-info → anysite_cli-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anysite-cli
-Version: 0.1.2
+Version: 0.1.4
 Summary: CLI for Anysite API - web data extraction for humans and AI agents
 Project-URL: Homepage, https://anysite.io
 Project-URL: Documentation, https://docs.anysite.io/cli
@@ -272,6 +272,7 @@ sources:
         - type: company
           value: "{value}"
       count: 5
+    refresh: always                       # Re-collect every run with --incremental
     db_load:
       fields: [name, url, headline]
 
@@ -327,6 +328,9 @@ anysite dataset logs my-dataset --run 42
 # Generate cron/systemd schedule
 anysite dataset schedule dataset.yaml --incremental --load-db pg
 
+# Compare snapshots (diff two collection dates)
+anysite dataset diff dataset.yaml --source employees --key _input_value
+
 # Reset incremental state
 anysite dataset reset-cursor dataset.yaml
 ```

{anysite_cli-0.1.2.dist-info → anysite_cli-0.1.4.dist-info}/RECORD

@@ -19,13 +19,14 @@ anysite/config/paths.py,sha256=EmHJD8wlf4Q9IUn8Gp1JQ8Z3ffrIYAt5iHRyImQOf5I,1087
 anysite/config/settings.py,sha256=Hc0j_aCCtkJeL4nHw-EFyfJ8WEDk57G08iNUFquUhpM,5235
 anysite/dataset/__init__.py,sha256=J0sKQkGwVOPtvp6pka7LcdeUEADvjWRs71yRuROzJxI,847
 anysite/dataset/analyzer.py,sha256=8dsPW32SbSaUTy1F0NIed1U45wjiMgQeJ2iWX7hBxRQ,9245
-anysite/dataset/cli.py,sha256=elBpp7XEmMcWBdyjkeDOi1730bCxVM77D0aVcmM9A0s,19423
-anysite/dataset/collector.py,sha256=GGzkkvNETcdgeyIV_6YNthDY2SHtDhl-U8YsXeeOuFo,23350
-anysite/dataset/db_loader.py,sha256=nlMJrDJiGBX5H1StcjsontSxLXbsFe4rwOEnDehzpk8,8443
+anysite/dataset/cli.py,sha256=rEWK1ka-YQ_Vbbj2nMaMYTD9g3wa3ethUWSoaWRSGTY,23066
+anysite/dataset/collector.py,sha256=6CfJt8fKZZ2xvZWJ7jwnx0V9BnjoJxmBZkm8xWQiU54,23840
+anysite/dataset/db_loader.py,sha256=TMcvI-pX-XctbkTdo5eTyW8Co4_3uK-dEdXn_r9g8Oc,13547
+anysite/dataset/differ.py,sha256=b-qU5Laf8RkteZAlblKq4atTvnJ21W4QbxfpHBFYMJ8,17053
 anysite/dataset/errors.py,sha256=r8cZXoIzSeTGCWpeYjntnN0AduCu74YZyWs3sFu17J4,914
 anysite/dataset/exporters.py,sha256=mA2FYbYJbHfrwkXbHDu4g5qPG_JJKnkVciXFKPkF1Vw,3708
 anysite/dataset/history.py,sha256=avFs0ADlM7Hr-ttqC1FfjJiQxvQP20sScM7ZoY4lvU0,5471
-anysite/dataset/models.py,sha256=YzpXmkmcyy_-BUxnYWrZMidpii75DbbZHBMTkw8Y1qo,9516
+anysite/dataset/models.py,sha256=-Qnh6QvbN3nzlfsYqgCiYKBqOeLcJCYK_hYrmxVCRTA,9810
 anysite/dataset/notifications.py,sha256=ORzo9XOgOxzLb7rk4pevlKPB_Taf-jejlrtmO4Zgl2c,2367
 anysite/dataset/scheduler.py,sha256=zpbA5tRUQZXr-9lZnG58dvE3E7ZBlAd-U-PTXExe9f0,3339
 anysite/dataset/storage.py,sha256=d03goKLI5NWKJowHwCgGqQkcVTO1NctPxMu-Xu-tru4,5326
@@ -57,8 +58,8 @@ anysite/streaming/writer.py,sha256=HfMsC4umUdJuNIAPK57YAxEGyTwUmy-zNrqFkwY6aew,4
 anysite/utils/__init__.py,sha256=7SnbxpxKENK-2ecUL5NfnZ9okGI7COKYw4WF46172HM,23
 anysite/utils/fields.py,sha256=bSrHadzNmabL4qubqhXXZoWb_P8KA-3S7_FLVT8nGBc,7410
 anysite/utils/retry.py,sha256=89TbXvavi5t22P2mTYCLAS6SSZoW65gQ0nnYNbYAF0M,2684
-anysite_cli-0.1.2.dist-info/METADATA,sha256=ucMaKonPqQeg6HsoPfVCbmjiVyIZxW5Pb2HElHiAtHo,11576
-anysite_cli-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-anysite_cli-0.1.2.dist-info/entry_points.txt,sha256=FDPxNasy0fRRcOgJdZRVP7Qw01C3TwRa1OwPJiskNyg,45
-anysite_cli-0.1.2.dist-info/licenses/LICENSE,sha256=gVAxkI23CFm4x4HV_fkQYw_bGq93mQmVZEwxNs-YTa4,1069
-anysite_cli-0.1.2.dist-info/RECORD,,
+anysite_cli-0.1.4.dist-info/METADATA,sha256=w5DUgDWzJgXynKRogJVm9baLqTJVSrg0ciHuWfWa9l0,11781
+anysite_cli-0.1.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+anysite_cli-0.1.4.dist-info/entry_points.txt,sha256=FDPxNasy0fRRcOgJdZRVP7Qw01C3TwRa1OwPJiskNyg,45
+anysite_cli-0.1.4.dist-info/licenses/LICENSE,sha256=gVAxkI23CFm4x4HV_fkQYw_bGq93mQmVZEwxNs-YTa4,1069
+anysite_cli-0.1.4.dist-info/RECORD,,