anysite-cli 0.1.0__py3-none-any.whl

anysite/dataset/models.py

@@ -0,0 +1,245 @@
+"""Pydantic models for dataset YAML configuration."""
+
+from __future__ import annotations
+
+from collections import deque
+from pathlib import Path
+from typing import Any, Literal
+
+import yaml
+from pydantic import BaseModel, Field, PrivateAttr, field_validator, model_validator
+
+from anysite.dataset.errors import CircularDependencyError, SourceNotFoundError
+
+# ---------------------------------------------------------------------------
+# New models for transform / export / schedule / notifications
+# ---------------------------------------------------------------------------
+
+
+class TransformConfig(BaseModel):
+    """Per-source transform: filter → select fields → add columns."""
+
+    filter: str | None = Field(default=None, description="Filter expression (e.g., '.employee_count > 10')")
+    fields: list[str] = Field(default_factory=list, description="Fields to keep (empty = all)")
+    add_columns: dict[str, Any] = Field(default_factory=dict, description="Static columns to add")
+
+
+class ExportDestination(BaseModel):
+    """Per-source export destination (file or webhook)."""
+
+    type: Literal["file", "webhook"] = Field(description="Export type")
+    path: str | None = Field(default=None, description="Output file path (file type)")
+    format: str = Field(default="jsonl", description="File format: json, jsonl, csv")
+    url: str | None = Field(default=None, description="Webhook URL (webhook type)")
+    headers: dict[str, str] = Field(default_factory=dict, description="HTTP headers for webhook")
+
+    @model_validator(mode="after")
+    def validate_type_fields(self) -> ExportDestination:
+        if self.type == "file" and not self.path:
+            raise ValueError("File export requires 'path'")
+        if self.type == "webhook" and not self.url:
+            raise ValueError("Webhook export requires 'url'")
+        return self
+
+
+class ScheduleConfig(BaseModel):
+    """Cron-based schedule for dataset collection."""
+
+    cron: str = Field(description="Cron expression (e.g., '0 9 * * MON')")
+
+
+class WebhookNotification(BaseModel):
+    """A single webhook notification endpoint."""
+
+    url: str = Field(description="Webhook URL")
+    headers: dict[str, str] = Field(default_factory=dict, description="HTTP headers")
+
+
+class NotificationsConfig(BaseModel):
+    """Notification webhooks for collection events."""
+
+    on_complete: list[WebhookNotification] = Field(default_factory=list)
+    on_failure: list[WebhookNotification] = Field(default_factory=list)
+
+
+class SourceDependency(BaseModel):
+    """Dependency on another source's output."""
+
+    from_source: str = Field(description="Source ID to depend on")
+    field: str | None = Field(
+        default=None,
+        description="Field to extract from parent records (dot notation)",
+    )
+    match_by: str | None = Field(
+        default=None,
+        description="Field for fuzzy matching by name",
+    )
+    dedupe: bool = Field(default=False, description="Deduplicate extracted values")
+
+
+class DbLoadConfig(BaseModel):
+    """Configuration for loading a source into a relational database."""
+
+    table: str | None = Field(default=None, description="Override table name (default: source id)")
+    fields: list[str] = Field(default_factory=list, description="Fields to include (empty = all)")
+    exclude: list[str] = Field(
+        default_factory=lambda: ["_input_value", "_parent_source"],
+        description="Fields to exclude (default: provenance metadata)",
+    )
+
+
+class DatasetSource(BaseModel):
+    """A single data source within a dataset."""
+
+    id: str = Field(description="Unique source identifier")
+    endpoint: str = Field(description="API endpoint path (e.g., /api/linkedin/search/users)")
+    params: dict[str, Any] = Field(default_factory=dict, description="Static API parameters")
+    dependency: SourceDependency | None = Field(
+        default=None,
+        description="Dependency on another source",
+    )
+    input_key: str | None = Field(
+        default=None,
+        description="Parameter name for dependent input values",
+    )
+    input_template: dict[str, Any] | None = Field(
+        default=None,
+        description="Template for input value — use {value} placeholder (e.g., {type: company, value: '{value}'})",
+    )
+    from_file: str | None = Field(
+        default=None,
+        description="Path to input file (CSV/JSONL/text) with values to iterate over",
+    )
+    file_field: str | None = Field(
+        default=None,
+        description="Column name to extract from CSV input file",
+    )
+    parallel: int = Field(default=1, ge=1, description="Parallel requests for dependent collection")
+    rate_limit: str | None = Field(default=None, description="Rate limit (e.g., '10/s')")
+    on_error: str = Field(default="skip", description="Error handling: stop or skip")
+    db_load: DbLoadConfig | None = Field(
+        default=None,
+        description="Database loading configuration (optional)",
+    )
+    transform: TransformConfig | None = Field(
+        default=None,
+        description="Post-collection transform (filter/fields/add_columns)",
+    )
+    export: list[ExportDestination] = Field(
+        default_factory=list,
+        description="Export destinations (file/webhook) applied after Parquet write",
+    )
+
+    @field_validator("endpoint")
+    @classmethod
+    def validate_endpoint(cls, v: str) -> str:
+        if not v.startswith("/"):
+            raise ValueError(f"Endpoint must start with '/', got: {v}")
+        return v
+
+
+class StorageConfig(BaseModel):
+    """Storage configuration for dataset output."""
+
+    format: str = Field(default="parquet", description="Storage format")
+    path: str = Field(default="./data/", description="Base directory for data files")
+    partition_by: list[str] = Field(
+        default_factory=lambda: ["source_id", "collected_date"],
+        description="Partition dimensions",
+    )
+
+
+class DatasetConfig(BaseModel):
+    """Top-level dataset configuration parsed from YAML."""
+
+    name: str = Field(description="Dataset name")
+    description: str = Field(default="", description="Dataset description")
+    sources: list[DatasetSource] = Field(description="Data sources to collect")
+    storage: StorageConfig = Field(default_factory=StorageConfig)
+    schedule: ScheduleConfig | None = Field(default=None, description="Collection schedule")
+    notifications: NotificationsConfig | None = Field(default=None, description="Webhook notifications")
+
+    _config_dir: Path | None = PrivateAttr(default=None)
+
+    @field_validator("sources")
+    @classmethod
+    def validate_unique_ids(cls, v: list[DatasetSource]) -> list[DatasetSource]:
+        ids = [s.id for s in v]
+        dupes = [sid for sid in ids if ids.count(sid) > 1]
+        if dupes:
+            raise ValueError(f"Duplicate source IDs: {set(dupes)}")
+        return v
+
+    @classmethod
+    def from_yaml(cls, path: Path) -> DatasetConfig:
+        """Load dataset configuration from a YAML file."""
+        with open(path) as f:
+            data = yaml.safe_load(f)
+        config = cls.model_validate(data)
+        config._config_dir = path.resolve().parent
+        return config
+
+    def get_source(self, source_id: str) -> DatasetSource | None:
+        """Get a source by ID."""
+        for s in self.sources:
+            if s.id == source_id:
+                return s
+        return None
+
+    def topological_sort(self) -> list[DatasetSource]:
+        """Sort sources by dependency order using Kahn's algorithm.
+
+        Returns:
+            List of sources in execution order (independent first).
+
+        Raises:
+            CircularDependencyError: If dependencies form a cycle.
+            SourceNotFoundError: If a dependency references a non-existent source.
+        """
+        source_map = {s.id: s for s in self.sources}
+
+        # Build adjacency: in_degree counts and adjacency list
+        in_degree: dict[str, int] = {s.id: 0 for s in self.sources}
+        dependents: dict[str, list[str]] = {s.id: [] for s in self.sources}
+
+        for source in self.sources:
+            if source.dependency:
+                parent_id = source.dependency.from_source
+                if parent_id not in source_map:
+                    raise SourceNotFoundError(parent_id, source.id)
+                in_degree[source.id] += 1
+                dependents[parent_id].append(source.id)
+
+        # Kahn's algorithm
+        queue: deque[str] = deque()
+        for sid, degree in in_degree.items():
+            if degree == 0:
+                queue.append(sid)
+
+        result: list[DatasetSource] = []
+        while queue:
+            sid = queue.popleft()
+            result.append(source_map[sid])
+            for dep_id in dependents[sid]:
+                in_degree[dep_id] -= 1
+                if in_degree[dep_id] == 0:
+                    queue.append(dep_id)
+
+        if len(result) != len(self.sources):
+            # Find the cycle
+            remaining = [s.id for s in self.sources if s.id not in {r.id for r in result}]
+            raise CircularDependencyError(remaining)
+
+        return result
+
+    def storage_path(self) -> Path:
+        """Resolve the storage base path.
+
+        Relative paths are resolved against the directory containing the
+        YAML config file (set by ``from_yaml``).  Absolute paths and
+        programmatic configs (no config dir) use the path as-is.
+        """
+        p = Path(self.storage.path)
+        if not p.is_absolute() and self._config_dir is not None:
+            return self._config_dir / p
+        return p

anysite/dataset/notifications.py

@@ -0,0 +1,87 @@
+"""Webhook notifications for dataset collection events."""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC, datetime
+from typing import Any
+
+from anysite.dataset.models import NotificationsConfig
+
+logger = logging.getLogger(__name__)
+
+
+class WebhookNotifier:
+    """Send webhook notifications on collection complete/failure."""
+
+    def __init__(self, config: NotificationsConfig) -> None:
+        self.config = config
+
+    async def notify_complete(
+        self,
+        dataset_name: str,
+        record_count: int,
+        source_count: int,
+        duration: float,
+    ) -> None:
+        """Send on_complete webhooks."""
+        if not self.config.on_complete:
+            return
+
+        payload = _build_payload(
+            event="complete",
+            dataset_name=dataset_name,
+            record_count=record_count,
+            source_count=source_count,
+            duration=duration,
+        )
+
+        for hook in self.config.on_complete:
+            await self._send(hook.url, hook.headers, payload)
+
+    async def notify_failure(
+        self,
+        dataset_name: str,
+        error: str,
+        duration: float,
+    ) -> None:
+        """Send on_failure webhooks."""
+        if not self.config.on_failure:
+            return
+
+        payload = _build_payload(
+            event="failure",
+            dataset_name=dataset_name,
+            error=error,
+            duration=duration,
+        )
+
+        for hook in self.config.on_failure:
+            await self._send(hook.url, hook.headers, payload)
+
+    async def _send(self, url: str, headers: dict[str, str], payload: dict[str, Any]) -> None:
+        """Send a single webhook POST."""
+        try:
+            import httpx
+
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.post(url, json=payload, headers=headers)
+                resp.raise_for_status()
+            logger.info("Notification sent to %s", url)
+        except Exception as e:
+            logger.error("Notification to %s failed: %s", url, e)
+
+
+def _build_payload(
+    *,
+    event: str,
+    dataset_name: str,
+    **kwargs: Any,
+) -> dict[str, Any]:
+    """Build a webhook notification payload."""
+    return {
+        "event": event,
+        "dataset": dataset_name,
+        "timestamp": datetime.now(UTC).isoformat(),
+        **kwargs,
+    }

anysite/dataset/scheduler.py

@@ -0,0 +1,107 @@
+"""Schedule generation for dataset collection — crontab and systemd."""
+
+from __future__ import annotations
+
+import shutil
+
+
+class ScheduleGenerator:
+    """Generate crontab or systemd timer entries for dataset collection."""
+
+    def __init__(self, dataset_name: str, cron_expr: str, yaml_path: str) -> None:
+        self.dataset_name = dataset_name
+        self.cron = cron_expr
+        self.yaml_path = yaml_path
+
+    def generate_crontab(self, *, incremental: bool = False, load_db: str | None = None) -> str:
+        """Generate a crontab entry."""
+        anysite = shutil.which("anysite") or "anysite"
+        cmd = f"{anysite} dataset collect {self.yaml_path}"
+        if incremental:
+            cmd += " --incremental"
+        if load_db:
+            cmd += f" --load-db {load_db}"
+        return f"{self.cron} {cmd} >> ~/.anysite/logs/{self.dataset_name}_cron.log 2>&1"
+
+    def generate_systemd(self, *, incremental: bool = False, load_db: str | None = None) -> dict[str, str]:
+        """Generate systemd service and timer unit files."""
+        anysite = shutil.which("anysite") or "anysite"
+        cmd = f"{anysite} dataset collect {self.yaml_path}"
+        if incremental:
+            cmd += " --incremental"
+        if load_db:
+            cmd += f" --load-db {load_db}"
+
+        service_name = f"anysite-dataset-{self.dataset_name}"
+        on_calendar = _cron_to_oncalendar(self.cron)
+
+        service = f"""[Unit]
+Description=Anysite dataset collection: {self.dataset_name}
+
+[Service]
+Type=oneshot
+ExecStart={cmd}
+StandardOutput=journal
+StandardError=journal
+"""
+
+        timer = f"""[Unit]
+Description=Timer for anysite dataset: {self.dataset_name}
+
+[Timer]
+OnCalendar={on_calendar}
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+"""
+
+        return {
+            f"{service_name}.service": service,
+            f"{service_name}.timer": timer,
+        }
+
+
+def _cron_to_oncalendar(cron: str) -> str:
+    """Convert a cron expression to systemd OnCalendar format (best effort).
+
+    Handles common patterns:
+        0 9 * * MON   -> Mon *-*-* 09:00:00
+        */5 * * * *   -> *-*-* *:0/5:00
+        0 0 1 * *     -> *-*-01 00:00:00
+    """
+    parts = cron.split()
+    if len(parts) != 5:
+        return cron  # pass through as-is
+
+    minute, hour, day, month, dow = parts
+
+    # Day-of-week mapping
+    dow_map = {
+        "0": "Sun", "1": "Mon", "2": "Tue", "3": "Wed",
+        "4": "Thu", "5": "Fri", "6": "Sat", "7": "Sun",
+        "MON": "Mon", "TUE": "Tue", "WED": "Wed",
+        "THU": "Thu", "FRI": "Fri", "SAT": "Sat", "SUN": "Sun",
+    }
+
+    # Format components
+    dow_str = ""
+    if dow != "*":
+        dow_parts = dow.replace(",", " ").split()
+        dow_str = ",".join(dow_map.get(d.upper(), d) for d in dow_parts) + " "
+
+    month_str = month if month != "*" else "*"
+    day_str = day.zfill(2) if day != "*" and "/" not in day else day
+
+    hour_str = hour.zfill(2) if hour != "*" and "/" not in hour else hour
+    min_str = minute.zfill(2) if minute != "*" and "/" not in minute else minute
+
+    # Handle step values
+    if "/" in minute:
+        base, step = minute.split("/")
+        min_str = f"{base or '0'}/{step}"
+    if "/" in hour:
+        base, step = hour.split("/")
+        hour_str = f"{base or '0'}/{step}"
+
+    return f"{dow_str}*-{month_str}-{day_str} {hour_str}:{min_str}:00"

anysite/dataset/storage.py

@@ -0,0 +1,171 @@
+"""Parquet storage layer for dataset records."""
+
+from __future__ import annotations
+
+import json
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+
+def write_parquet(
+    records: list[dict[str, Any]],
+    path: Path,
+) -> int:
+    """Write records to a Parquet file.
+
+    Args:
+        records: List of dicts to write.
+        path: Output file path.
+
+    Returns:
+        Number of records written.
+    """
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+
+    if not records:
+        return 0
+
+    # Normalize: flatten nested structures to JSON strings for non-scalar types
+    normalized = _normalize_records(records)
+
+    table = pa.Table.from_pylist(normalized)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    pq.write_table(table, path)
+    return len(records)
+
+
+def _normalize_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Normalize records for Parquet compatibility.
+
+    Converts nested dicts/lists to JSON strings so that pyarrow
+    can infer a consistent schema across heterogeneous records.
+    """
+    if not records:
+        return records
+
+    result = []
+    for record in records:
+        normalized: dict[str, Any] = {}
+        for key, value in record.items():
+            if isinstance(value, (dict, list)):
+                normalized[key] = json.dumps(value, default=str)
+            else:
+                normalized[key] = value
+        result.append(normalized)
+    return result
+
+
+def read_parquet(path: Path) -> list[dict[str, Any]]:
+    """Read records from a Parquet file or directory of Parquet files.
+
+    Args:
+        path: Parquet file or directory containing .parquet files.
+
+    Returns:
+        List of dicts.
+    """
+    import pyarrow.parquet as pq
+
+    if path.is_dir():
+        files = sorted(path.glob("*.parquet"))
+        if not files:
+            return []
+        tables = [pq.read_table(f) for f in files]
+        import pyarrow as pa
+
+        table = pa.concat_tables(tables)
+    else:
+        if not path.exists():
+            return []
+        table = pq.read_table(path)
+
+    return table.to_pylist()
+
+
+def get_source_dir(base_path: Path, source_id: str) -> Path:
+    """Get the raw data directory for a source."""
+    return base_path / "raw" / source_id
+
+
+def get_parquet_path(base_path: Path, source_id: str, collected_date: date | None = None) -> Path:
+    """Get the Parquet file path for a source on a given date."""
+    if collected_date is None:
+        collected_date = date.today()
+    source_dir = get_source_dir(base_path, source_id)
+    return source_dir / f"{collected_date.isoformat()}.parquet"
+
+
+class MetadataStore:
+    """Read/write metadata.json for dataset state tracking."""
+
+    def __init__(self, base_path: Path) -> None:
+        self.path = base_path / "metadata.json"
+
+    def load(self) -> dict[str, Any]:
+        if self.path.exists():
+            with open(self.path) as f:
+                return json.load(f)  # type: ignore[no-any-return]
+        return {"sources": {}}
+
+    def save(self, data: dict[str, Any]) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self.path, "w") as f:
+            json.dump(data, f, indent=2, default=str)
+
+    def update_source(
+        self,
+        source_id: str,
+        record_count: int,
+        collected_date: date | None = None,
+    ) -> None:
+        """Update metadata for a collected source."""
+        if collected_date is None:
+            collected_date = date.today()
+
+        data = self.load()
+        sources = data.setdefault("sources", {})
+        sources[source_id] = {
+            "last_collected": collected_date.isoformat(),
+            "record_count": record_count,
+        }
+        data["last_run"] = collected_date.isoformat()
+        self.save(data)
+
+    def get_source_info(self, source_id: str) -> dict[str, Any] | None:
+        """Get metadata for a specific source."""
+        data = self.load()
+        return data.get("sources", {}).get(source_id)
+
+    def get_all_sources(self) -> dict[str, Any]:
+        """Get metadata for all sources."""
+        data = self.load()
+        return data.get("sources", {})
+
+    def update_collected_inputs(
+        self, source_id: str, inputs: list[str]
+    ) -> None:
+        """Append collected input values to metadata for dedup tracking."""
+        data = self.load()
+        sources = data.setdefault("sources", {})
+        source_info = sources.setdefault(source_id, {})
+        existing = set(source_info.get("collected_inputs", []))
+        existing.update(str(v) for v in inputs)
+        source_info["collected_inputs"] = sorted(existing)
+        self.save(data)
+
+    def get_collected_inputs(self, source_id: str) -> set[str]:
+        """Get the set of already-collected input values for a source."""
+        info = self.get_source_info(source_id)
+        if info and "collected_inputs" in info:
+            return set(info["collected_inputs"])
+        return set()
+
+    def reset_collected_inputs(self, source_id: str) -> None:
+        """Clear collected input tracking for a source (forces re-collection)."""
+        data = self.load()
+        source_info = data.get("sources", {}).get(source_id, {})
+        if "collected_inputs" in source_info:
+            del source_info["collected_inputs"]
+            self.save(data)