anysite-cli 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

anysite/dataset/cli.py

@@ -413,6 +413,117 @@ def load_db(
         )
 
 
+@app.command("diff")
+def diff_cmd(
+    config_path: Annotated[
+        Path,
+        typer.Argument(help="Path to dataset.yaml"),
+    ],
+    source: Annotated[
+        str,
+        typer.Option("--source", "-s", help="Source to compare"),
+    ],
+    key: Annotated[
+        str,
+        typer.Option("--key", "-k", help="Field to match records by (e.g., _input_value, urn)"),
+    ],
+    from_date: Annotated[
+        str | None,
+        typer.Option("--from", help="Older snapshot date (YYYY-MM-DD)"),
+    ] = None,
+    to_date: Annotated[
+        str | None,
+        typer.Option("--to", help="Newer snapshot date (YYYY-MM-DD)"),
+    ] = None,
+    fields: Annotated[
+        str | None,
+        typer.Option("--fields", "-f", help="Only compare these fields (comma-separated)"),
+    ] = None,
+    format: Annotated[
+        str,
+        typer.Option("--format", help="Output format: table, json, jsonl, csv"),
+    ] = "table",
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="Write output to file"),
+    ] = None,
+    quiet: Annotated[
+        bool,
+        typer.Option("--quiet", "-q", help="Suppress summary, only output data"),
+    ] = False,
+) -> None:
+    """Compare two snapshots of a source to show added, removed, and changed records."""
+    from datetime import date as date_type
+
+    from anysite.dataset.differ import (
+        DatasetDiffer,
+        format_diff_records,
+        format_diff_table,
+    )
+
+    config = _load_config(config_path)
+
+    # Validate source exists
+    src = config.get_source(source)
+    if src is None:
+        typer.echo(f"Error: source '{source}' not found in dataset", err=True)
+        raise typer.Exit(1)
+
+    differ = DatasetDiffer(config.storage_path())
+
+    # Parse dates
+    parsed_from = None
+    parsed_to = None
+    try:
+        if from_date:
+            parsed_from = date_type.fromisoformat(from_date)
+        if to_date:
+            parsed_to = date_type.fromisoformat(to_date)
+    except ValueError as e:
+        typer.echo(f"Error: invalid date format: {e}", err=True)
+        raise typer.Exit(1) from None
+
+    # Parse fields
+    field_list = None
+    if fields:
+        field_list = [f.strip() for f in fields.split(",") if f.strip()]
+
+    try:
+        result = differ.diff(
+            source,
+            key,
+            from_date=parsed_from,
+            to_date=parsed_to,
+            fields=field_list,
+        )
+    except DatasetError as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1) from None
+
+    # Print summary unless quiet
+    if not quiet:
+        console = Console()
+        console.print(
+            f"\n[bold]Diff: {source}[/bold] "
+            f"({result.from_date.isoformat()} → {result.to_date.isoformat()})\n"
+        )
+        console.print(f"  [green]Added:[/green]     {len(result.added)}")
+        console.print(f"  [red]Removed:[/red]   {len(result.removed)}")
+        console.print(f"  [yellow]Changed:[/yellow]   {len(result.changed)}")
+        console.print(f"  Unchanged: {result.unchanged_count}")
+        console.print()
+
+    if not result.has_changes:
+        if not quiet:
+            Console().print("[dim]No changes detected.[/dim]")
+        return
+
+    # Format and output
+    rows = format_diff_table(result) if format == "table" else format_diff_records(result)
+
+    _output_results(rows, format, output)
+
+
 @app.command("history")
 def history(
     name: Annotated[

anysite/dataset/collector.py

@@ -43,6 +43,7 @@ class CollectionPlan:
         params: dict[str, Any] | None = None,
         dependency: str | None = None,
         estimated_requests: int | None = None,
+        refresh: str = "auto",
     ) -> None:
         self.steps.append({
             "source": source_id,
@@ -51,6 +52,7 @@ class CollectionPlan:
             "params": params or {},
             "dependency": dependency,
             "estimated_requests": estimated_requests,
+            "refresh": refresh,
         })
 
 
@@ -116,8 +118,8 @@ async def collect_dataset(
 
     try:
         for source in ordered:
-            # Check incremental skip
-            if incremental:
+            # Check incremental skip (refresh: always bypasses this)
+            if incremental and source.refresh != "always":
                 parquet_path = get_parquet_path(base_path, source.id, today)
                 if parquet_path.exists():
                     if not quiet:
@@ -276,8 +278,8 @@ async def _collect_from_file(
             print_warning(f"No values extracted from {file_path}")
         return []
 
-    # Filter already-collected inputs in incremental mode
-    if incremental and metadata:
+    # Filter already-collected inputs in incremental mode (refresh: always bypasses)
+    if incremental and source.refresh != "always" and metadata:
         already = metadata.get_collected_inputs(source.id)
         if already:
             original = len(values)
@@ -432,8 +434,8 @@ async def _collect_dependent(
             f"Source {source.id} has a dependency but no input_key defined"
         )
 
-    # Filter already-collected inputs in incremental mode
-    if incremental and metadata:
+    # Filter already-collected inputs in incremental mode (refresh: always bypasses)
+    if incremental and source.refresh != "always" and metadata:
         already = metadata.get_collected_inputs(source.id)
         if already:
             original = len(values)
@@ -579,7 +581,7 @@ def _build_plan(
     plan = CollectionPlan()
 
     for source in ordered:
-        if incremental:
+        if incremental and source.refresh != "always":
             parquet_path = get_parquet_path(base_path, source.id, today)
             if parquet_path.exists():
                 continue
@@ -592,6 +594,7 @@ def _build_plan(
                 kind="from_file",
                 params={"file": source.from_file, "field": source.file_field},
                 estimated_requests=est,
+                refresh=source.refresh,
             )
         elif source.dependency is None:
             plan.add_step(
@@ -600,6 +603,7 @@ def _build_plan(
                 kind="independent",
                 params=source.params,
                 estimated_requests=1,
+                refresh=source.refresh,
             )
         else:
             est = _count_dependent_inputs(source, base_path, metadata)
@@ -609,6 +613,7 @@ def _build_plan(
                 kind="dependent",
                 dependency=source.dependency.from_source,
                 estimated_requests=est,
+                refresh=source.refresh,
             )
 
     return plan
@@ -665,11 +670,14 @@ def _print_plan(plan: CollectionPlan) -> dict[str, int]:
     table.add_column("Est. Requests")
 
     for i, step in enumerate(plan.steps, 1):
+        kind = step["kind"]
+        if step.get("refresh") == "always":
+            kind += " (refresh)"
         table.add_row(
             str(i),
             step["source"],
             step["endpoint"],
-            step["kind"],
+            kind,
             step.get("dependency") or "-",
             str(step.get("estimated_requests") or "?"),
         )

anysite/dataset/differ.py

@@ -0,0 +1,355 @@
+"""Compare two dataset snapshots to find added, removed, and changed records."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+from anysite.dataset.errors import DatasetError
+from anysite.dataset.storage import get_source_dir
+
+
+@dataclass
+class DiffResult:
+    """Result of comparing two dataset snapshots."""
+
+    source_id: str
+    from_date: date
+    to_date: date
+    key: str
+    added: list[dict[str, Any]] = field(default_factory=list)
+    removed: list[dict[str, Any]] = field(default_factory=list)
+    changed: list[dict[str, Any]] = field(default_factory=list)
+    unchanged_count: int = 0
+
+    @property
+    def has_changes(self) -> bool:
+        return bool(self.added or self.removed or self.changed)
+
+
+class DatasetDiffer:
+    """Compare two Parquet snapshots for a dataset source."""
+
+    def __init__(self, base_path: Path) -> None:
+        self.base_path = base_path
+
+    def available_dates(self, source_id: str) -> list[date]:
+        """List available snapshot dates for a source, sorted ascending."""
+        source_dir = get_source_dir(self.base_path, source_id)
+        if not source_dir.exists():
+            return []
+
+        dates: list[date] = []
+        for f in sorted(source_dir.glob("*.parquet")):
+            try:
+                dates.append(date.fromisoformat(f.stem))
+            except ValueError:
+                continue
+        return dates
+
+    def diff(
+        self,
+        source_id: str,
+        key: str,
+        *,
+        from_date: date | None = None,
+        to_date: date | None = None,
+        fields: list[str] | None = None,
+    ) -> DiffResult:
+        """Compare two snapshots using DuckDB.
+
+        Args:
+            source_id: Source to compare.
+            key: Field to match records by (e.g., ``_input_value``, ``urn``).
+            from_date: Older snapshot date (default: second-to-last).
+            to_date: Newer snapshot date (default: latest).
+            fields: Only compare these fields (default: all).
+
+        Returns:
+            DiffResult with added, removed, changed lists.
+        """
+        from_date, to_date = self._resolve_dates(source_id, from_date, to_date)
+        source_dir = get_source_dir(self.base_path, source_id)
+        old_path = source_dir / f"{from_date.isoformat()}.parquet"
+        new_path = source_dir / f"{to_date.isoformat()}.parquet"
+
+        if not old_path.exists():
+            raise DatasetError(
+                f"No snapshot for {source_id} on {from_date.isoformat()}"
+            )
+        if not new_path.exists():
+            raise DatasetError(
+                f"No snapshot for {source_id} on {to_date.isoformat()}"
+            )
+
+        return self._diff_with_duckdb(
+            source_id, key, old_path, new_path, from_date, to_date, fields
+        )
+
+    def _resolve_dates(
+        self,
+        source_id: str,
+        from_date: date | None,
+        to_date: date | None,
+    ) -> tuple[date, date]:
+        """Resolve from/to dates, defaulting to two most recent snapshots."""
+        if from_date and to_date:
+            return from_date, to_date
+
+        dates = self.available_dates(source_id)
+        if len(dates) < 2:
+            raise DatasetError(
+                f"Need at least 2 snapshots to diff, "
+                f"found {len(dates)} for {source_id}"
+            )
+
+        if to_date and not from_date:
+            # Find the date just before to_date
+            earlier = [d for d in dates if d < to_date]
+            if not earlier:
+                raise DatasetError(
+                    f"No snapshot before {to_date.isoformat()} for {source_id}"
+                )
+            return earlier[-1], to_date
+
+        if from_date and not to_date:
+            # Find the date just after from_date
+            later = [d for d in dates if d > from_date]
+            if not later:
+                raise DatasetError(
+                    f"No snapshot after {from_date.isoformat()} for {source_id}"
+                )
+            return from_date, later[0]
+
+        # Both None — use two most recent
+        return dates[-2], dates[-1]
+
+    def _diff_with_duckdb(
+        self,
+        source_id: str,
+        key: str,
+        old_path: Path,
+        new_path: Path,
+        from_date: date,
+        to_date: date,
+        fields: list[str] | None,
+    ) -> DiffResult:
+        """Run diff queries using DuckDB."""
+        import duckdb
+
+        conn = duckdb.connect(":memory:")
+        try:
+            conn.execute(
+                f"CREATE VIEW _old AS SELECT * FROM read_parquet('{old_path}')"
+            )
+            conn.execute(
+                f"CREATE VIEW _new AS SELECT * FROM read_parquet('{new_path}')"
+            )
+
+            # Get all column names from the new snapshot
+            info = conn.execute("DESCRIBE _new").fetchall()
+            all_columns = [col[0] for col in info]
+
+            if key not in all_columns:
+                raise DatasetError(
+                    f"Key field '{key}' not found in {source_id}. "
+                    f"Available: {', '.join(all_columns)}"
+                )
+
+            # Determine which fields to compare
+            compare_fields = fields if fields else [
+                c for c in all_columns if c != key
+            ]
+            # Filter to fields that actually exist
+            compare_fields = [c for c in compare_fields if c in all_columns]
+
+            quoted_key = f'"{key}"'
+
+            # Added: in new but not in old
+            added = conn.execute(
+                f"SELECT * FROM _new "
+                f"WHERE {quoted_key} NOT IN (SELECT {quoted_key} FROM _old)"
+            ).fetchall()
+            added_cols = [d[0] for d in conn.execute(
+                "DESCRIBE _new"
+            ).fetchall()]
+            added_records = [dict(zip(added_cols, row, strict=False)) for row in added]
+
+            # Removed: in old but not in new
+            removed = conn.execute(
+                f"SELECT * FROM _old "
+                f"WHERE {quoted_key} NOT IN (SELECT {quoted_key} FROM _new)"
+            ).fetchall()
+            removed_cols = [d[0] for d in conn.execute(
+                "DESCRIBE _old"
+            ).fetchall()]
+            removed_records = [dict(zip(removed_cols, row, strict=False)) for row in removed]
+
+            # Changed: matching key, different values
+            changed_records = self._find_changed(
+                conn, key, compare_fields, all_columns
+            )
+
+            # Count unchanged
+            total_matched = conn.execute(
+                f"SELECT COUNT(*) FROM _new n "
+                f"JOIN _old o ON n.{quoted_key} = o.{quoted_key}"
+            ).fetchone()
+            matched_count = total_matched[0] if total_matched else 0
+            unchanged_count = matched_count - len(changed_records)
+
+            return DiffResult(
+                source_id=source_id,
+                from_date=from_date,
+                to_date=to_date,
+                key=key,
+                added=added_records,
+                removed=removed_records,
+                changed=changed_records,
+                unchanged_count=unchanged_count,
+            )
+        finally:
+            conn.close()
+
+    def _find_changed(
+        self,
+        conn: Any,
+        key: str,
+        compare_fields: list[str],
+        all_columns: list[str],
+    ) -> list[dict[str, Any]]:
+        """Find records that exist in both snapshots but have different values."""
+        if not compare_fields:
+            return []
+
+        quoted_key = f'"{key}"'
+
+        # Build WHERE clause: any compared field differs
+        where_parts = []
+        for col in compare_fields:
+            qc = f'"{col}"'
+            where_parts.append(f"n.{qc} IS DISTINCT FROM o.{qc}")
+        where_clause = " OR ".join(where_parts)
+
+        # Select new values + old values for compared fields
+        select_parts = [f"n.{quoted_key}"]
+        for col in all_columns:
+            if col != key:
+                qc = f'"{col}"'
+                select_parts.append(f"n.{qc}")
+        for col in compare_fields:
+            qc = f'"{col}"'
+            select_parts.append(f"o.{qc} AS \"{col}__old\"")
+
+        select_clause = ", ".join(select_parts)
+
+        sql = (
+            f"SELECT {select_clause} FROM _new n "
+            f"JOIN _old o ON n.{quoted_key} = o.{quoted_key} "
+            f"WHERE {where_clause}"
+        )
+
+        result = conn.execute(sql)
+        columns = [desc[0] for desc in result.description]
+        rows = result.fetchall()
+        records = [dict(zip(columns, row, strict=False)) for row in rows]
+
+        # Add _changed_fields metadata to each record
+        for record in records:
+            changed_fields = []
+            for col in compare_fields:
+                old_key = f"{col}__old"
+                new_val = record.get(col)
+                old_val = record.get(old_key)
+                if _values_differ(new_val, old_val):
+                    changed_fields.append(col)
+            record["_changed_fields"] = changed_fields
+
+        return records
+
+
+def _values_differ(a: Any, b: Any) -> bool:
+    """Compare two values, treating JSON strings as equivalent to their parsed form."""
+    if a == b:
+        return False
+    # Handle JSON string comparison
+    if isinstance(a, str) and isinstance(b, str):
+        try:
+            return json.loads(a) != json.loads(b)
+        except (json.JSONDecodeError, ValueError):
+            pass
+    return True
+
+
+def format_diff_table(result: DiffResult) -> list[dict[str, Any]]:
+    """Format a DiffResult into a flat list of dicts for table/json output.
+
+    Each record gets a ``_diff`` column with value ``added``, ``removed``,
+    or ``changed``.  For changed records in table mode, modified field
+    values are formatted as ``old → new``.
+    """
+    rows: list[dict[str, Any]] = []
+
+    for record in result.added:
+        row = {"_diff": "added", **record}
+        rows.append(row)
+
+    for record in result.removed:
+        row = {"_diff": "removed", **record}
+        rows.append(row)
+
+    for record in result.changed:
+        row: dict[str, Any] = {"_diff": "changed"}
+        changed_fields = record.get("_changed_fields", [])
+        for k, v in record.items():
+            if k == "_changed_fields":
+                continue
+            if k.endswith("__old"):
+                continue
+            # For changed fields, format as "old → new"
+            if k in changed_fields:
+                old_val = record.get(f"{k}__old")
+                row[k] = f"{_format_val(old_val)} → {_format_val(v)}"
+            else:
+                row[k] = v
+        rows.append(row)
+
+    return rows
+
+
+def format_diff_records(result: DiffResult) -> list[dict[str, Any]]:
+    """Format a DiffResult for JSON/CSV output.
+
+    Each record gets ``_diff`` column.  Changed records include both
+    current values and ``field__old`` columns.
+    """
+    rows: list[dict[str, Any]] = []
+
+    for record in result.added:
+        rows.append({"_diff": "added", **record})
+
+    for record in result.removed:
+        rows.append({"_diff": "removed", **record})
+
+    for record in result.changed:
+        row = {"_diff": "changed"}
+        for k, v in record.items():
+            if k == "_changed_fields":
+                continue
+            row[k] = v
+        rows.append(row)
+
+    return rows
+
+
+def _format_val(v: Any) -> str:
+    """Format a value for display, truncating long strings."""
+    if v is None:
+        return "null"
+    s = str(v)
+    if len(s) > 40:
+        return s[:37] + "..."
+    return s

anysite/dataset/models.py

@@ -129,6 +129,10 @@ class DatasetSource(BaseModel):
         default_factory=list,
         description="Export destinations (file/webhook) applied after Parquet write",
     )
+    refresh: Literal["auto", "always"] = Field(
+        default="auto",
+        description="Refresh mode: 'auto' uses incremental caching, 'always' re-collects every run",
+    )
 
     @field_validator("endpoint")
     @classmethod

{anysite_cli-0.1.2.dist-info → anysite_cli-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anysite-cli
-Version: 0.1.2
+Version: 0.1.3
 Summary: CLI for Anysite API - web data extraction for humans and AI agents
 Project-URL: Homepage, https://anysite.io
 Project-URL: Documentation, https://docs.anysite.io/cli
@@ -272,6 +272,7 @@ sources:
         - type: company
           value: "{value}"
       count: 5
+    refresh: always                       # Re-collect every run with --incremental
     db_load:
       fields: [name, url, headline]
 
@@ -327,6 +328,9 @@ anysite dataset logs my-dataset --run 42
 # Generate cron/systemd schedule
 anysite dataset schedule dataset.yaml --incremental --load-db pg
 
+# Compare snapshots (diff two collection dates)
+anysite dataset diff dataset.yaml --source employees --key _input_value
+
 # Reset incremental state
 anysite dataset reset-cursor dataset.yaml
 ```

{anysite_cli-0.1.2.dist-info → anysite_cli-0.1.3.dist-info}/RECORD

@@ -19,13 +19,14 @@ anysite/config/paths.py,sha256=EmHJD8wlf4Q9IUn8Gp1JQ8Z3ffrIYAt5iHRyImQOf5I,1087
 anysite/config/settings.py,sha256=Hc0j_aCCtkJeL4nHw-EFyfJ8WEDk57G08iNUFquUhpM,5235
 anysite/dataset/__init__.py,sha256=J0sKQkGwVOPtvp6pka7LcdeUEADvjWRs71yRuROzJxI,847
 anysite/dataset/analyzer.py,sha256=8dsPW32SbSaUTy1F0NIed1U45wjiMgQeJ2iWX7hBxRQ,9245
-anysite/dataset/cli.py,sha256=elBpp7XEmMcWBdyjkeDOi1730bCxVM77D0aVcmM9A0s,19423
-anysite/dataset/collector.py,sha256=GGzkkvNETcdgeyIV_6YNthDY2SHtDhl-U8YsXeeOuFo,23350
+anysite/dataset/cli.py,sha256=zaCo0kKeA1KNU7EZgW4WwxrP07xuKayPlolfUnCSoYI,22801
+anysite/dataset/collector.py,sha256=6CfJt8fKZZ2xvZWJ7jwnx0V9BnjoJxmBZkm8xWQiU54,23840
 anysite/dataset/db_loader.py,sha256=nlMJrDJiGBX5H1StcjsontSxLXbsFe4rwOEnDehzpk8,8443
+anysite/dataset/differ.py,sha256=hbUwoS73syTkrj0VC0gaJzuB0pVCoQXQMbsNXtpsig8,11634
 anysite/dataset/errors.py,sha256=r8cZXoIzSeTGCWpeYjntnN0AduCu74YZyWs3sFu17J4,914
 anysite/dataset/exporters.py,sha256=mA2FYbYJbHfrwkXbHDu4g5qPG_JJKnkVciXFKPkF1Vw,3708
 anysite/dataset/history.py,sha256=avFs0ADlM7Hr-ttqC1FfjJiQxvQP20sScM7ZoY4lvU0,5471
-anysite/dataset/models.py,sha256=YzpXmkmcyy_-BUxnYWrZMidpii75DbbZHBMTkw8Y1qo,9516
+anysite/dataset/models.py,sha256=_f1cg9A4FlQwWGpg-s0b9q5WMlaIRN-ENlpU9CE6mrQ,9695
 anysite/dataset/notifications.py,sha256=ORzo9XOgOxzLb7rk4pevlKPB_Taf-jejlrtmO4Zgl2c,2367
 anysite/dataset/scheduler.py,sha256=zpbA5tRUQZXr-9lZnG58dvE3E7ZBlAd-U-PTXExe9f0,3339
 anysite/dataset/storage.py,sha256=d03goKLI5NWKJowHwCgGqQkcVTO1NctPxMu-Xu-tru4,5326
@@ -57,8 +58,8 @@ anysite/streaming/writer.py,sha256=HfMsC4umUdJuNIAPK57YAxEGyTwUmy-zNrqFkwY6aew,4
 anysite/utils/__init__.py,sha256=7SnbxpxKENK-2ecUL5NfnZ9okGI7COKYw4WF46172HM,23
 anysite/utils/fields.py,sha256=bSrHadzNmabL4qubqhXXZoWb_P8KA-3S7_FLVT8nGBc,7410
 anysite/utils/retry.py,sha256=89TbXvavi5t22P2mTYCLAS6SSZoW65gQ0nnYNbYAF0M,2684
-anysite_cli-0.1.2.dist-info/METADATA,sha256=ucMaKonPqQeg6HsoPfVCbmjiVyIZxW5Pb2HElHiAtHo,11576
-anysite_cli-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-anysite_cli-0.1.2.dist-info/entry_points.txt,sha256=FDPxNasy0fRRcOgJdZRVP7Qw01C3TwRa1OwPJiskNyg,45
-anysite_cli-0.1.2.dist-info/licenses/LICENSE,sha256=gVAxkI23CFm4x4HV_fkQYw_bGq93mQmVZEwxNs-YTa4,1069
-anysite_cli-0.1.2.dist-info/RECORD,,
+anysite_cli-0.1.3.dist-info/METADATA,sha256=lD_AF5pq5ayHerMVMMWTTkgccwWEsKLBGCwvPfZ5y_Y,11781
+anysite_cli-0.1.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+anysite_cli-0.1.3.dist-info/entry_points.txt,sha256=FDPxNasy0fRRcOgJdZRVP7Qw01C3TwRa1OwPJiskNyg,45
+anysite_cli-0.1.3.dist-info/licenses/LICENSE,sha256=gVAxkI23CFm4x4HV_fkQYw_bGq93mQmVZEwxNs-YTa4,1069
+anysite_cli-0.1.3.dist-info/RECORD,,