data-collection-framework 0.1.0__py3-none-any.whl

dcf/config/__init__.py

@@ -0,0 +1,4 @@
+from .loader import load_collector, load_all_collectors
+from .models import Collector
+
+__all__ = ["load_collector", "load_all_collectors", "Collector"]

dcf/config/loader.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import yaml
+
+from .models import Collector
+
+
+def _project_config() -> dict:
+    """Load project.yml from the project root, returning an empty dict if absent."""
+    from ..project import find_project_root
+    try:
+        cfg_path = find_project_root() / "project.yml"
+    except RuntimeError:
+        return {}
+    if cfg_path.exists():
+        return yaml.safe_load(cfg_path.read_text()) or {}
+    return {}
+
+
+def _resolve_env(value: str, project_cfg: dict) -> str:
+    """Replace {{ env.VAR }} placeholders.
+
+    Resolution order:
+      1. OS environment variable
+      2. project.yml key (VAR lowercased, e.g. PORTLANDMAPS_API_KEY → portlandmaps_api_key)
+    """
+    import re
+    def replacer(match):
+        var = match.group(1).strip()
+        resolved = os.environ.get(var)
+        if resolved is None:
+            resolved = project_cfg.get(var.lower())
+        if not resolved:
+            raise EnvironmentError(
+                f"'{var}' is not set — add it as an environment variable "
+                f"or set '{var.lower()}' in project.yml"
+            )
+        return resolved
+    return re.sub(r"\{\{\s*env\.(\w+)\s*\}\}", replacer, value)
+
+
+def _resolve_env_in(obj, project_cfg: dict):
+    if isinstance(obj, dict):
+        return {k: _resolve_env_in(v, project_cfg) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_resolve_env_in(v, project_cfg) for v in obj]
+    if isinstance(obj, str):
+        return _resolve_env(obj, project_cfg)
+    return obj
+
+
+def load_collector(path: Path, resolve_env: bool = True) -> Collector:
+    raw = yaml.safe_load(path.read_text())
+    if resolve_env:
+        raw = _resolve_env_in(raw, _project_config())
+    else:
+        raw = _strip_env_placeholders(raw)
+    return Collector.from_dict(raw)
+
+
+def _strip_env_placeholders(obj):
+    """Replace {{ env.VAR }} with a placeholder string for structural validation."""
+    import re
+    if isinstance(obj, dict):
+        return {k: _strip_env_placeholders(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_strip_env_placeholders(v) for v in obj]
+    if isinstance(obj, str):
+        return re.sub(r"\{\{\s*env\.\w+\s*\}\}", "<env>", obj)
+    return obj
+
+
+def load_all_collectors(collectors_dir: Path, resolve_env: bool = True) -> list[Collector]:
+    return [load_collector(p, resolve_env=resolve_env) for p in sorted(collectors_dir.glob("*.yml"))]

dcf/config/models.py

@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+import re as _re
+from typing import Any, Literal, Annotated, Union
+from pydantic import BaseModel, Field, model_validator
+
+_CRON_FIELD_RE = _re.compile(r'^(\*|[0-9,\-\*/]+)$')
+
+
+# ------------------------------------------------------------------ #
+# Source — params                                                      #
+# ------------------------------------------------------------------ #
+
+class Param(BaseModel):
+    name: str
+    type: Literal["string", "integer", "float", "date", "boolean"]
+    format: str | None = None   # e.g. "%m/%d/%Y" for date URL serialization
+    value: Any | None = None    # present → static; absent → must be covered by iterate
+
+
+class Auth(BaseModel):
+    type: Literal["query_param", "header", "bearer"]
+    key: str | None = None   # param name or header name; unused (and optional) for bearer
+    value: str               # supports "{{ env.VAR }}"
+
+    @model_validator(mode="after")
+    def key_required_for_non_bearer(self) -> "Auth":
+        if self.type in ("query_param", "header") and not self.key:
+            raise ValueError(f"auth.key is required when type is '{self.type}'")
+        return self
+
+
+class RateLimit(BaseModel):
+    requests: int
+    per_minutes: float
+
+
+class Response(BaseModel):
+    format: Literal["json", "csv"]
+    records_path: str | None = None   # key (or dot-path) in JSON holding the records array
+
+
+# ------------------------------------------------------------------ #
+# Source — iteration                                                   #
+# ------------------------------------------------------------------ #
+
+class DateRangeIterate(BaseModel):
+    type: Literal["date_range"]
+    params: list[str]       # one or two param names that receive the window start/end
+    start: str              # ISO date or "today"
+    end: str                # ISO date or "today"
+    step: str               # e.g. "1 day", "7 days"
+    window: str | None = None  # defaults to step when absent
+
+
+class CategoricalIterate(BaseModel):
+    type: Literal["categorical"]
+    param: str
+    values: list[Any]
+
+
+IterateSpec = DateRangeIterate | CategoricalIterate
+
+
+def _validate_dynamic_params(params: list[Param], iterate: list[IterateSpec]) -> None:
+    dynamic = {p.name for p in params if p.value is None}
+    covered: set[str] = set()
+    for it in iterate:
+        if isinstance(it, DateRangeIterate):
+            covered.update(it.params)
+        elif isinstance(it, CategoricalIterate):
+            covered.add(it.param)
+    missing = dynamic - covered
+    if missing:
+        raise ValueError(f"Params declared without a value or iterator: {missing}")
+
+
+# ------------------------------------------------------------------ #
+# Schema                                                               #
+# ------------------------------------------------------------------ #
+
+class CrsReprojectTransform(BaseModel):
+    type: Literal["crs_reproject"]
+    from_columns: list[str]
+    from_crs: str
+    to_crs: str
+    component: Literal["x", "y"]
+
+
+class ArrayJoinTransform(BaseModel):
+    type: Literal["array_join"]
+    path: str              # dot-notation path to the array field in the raw record
+    separator: str = ","   # delimiter used to join elements
+
+
+Transform = Annotated[
+    Union[CrsReprojectTransform, ArrayJoinTransform],
+    Field(discriminator="type"),
+]
+
+
+class Column(BaseModel):
+    name: str
+    path: str | None = None         # key in the raw record (dot-notation for nested)
+    type: Literal["string", "integer", "float", "date", "timestamp", "boolean"] | None = None
+    transform: Transform | None = None
+
+    @model_validator(mode="after")
+    def has_source(self) -> Column:
+        if self.path is None and self.transform is None:
+            raise ValueError(f"Column '{self.name}' must have either 'path' or 'transform'")
+        return self
+
+
+class Schema(BaseModel):
+    columns: list[Column]
+
+
+# ------------------------------------------------------------------ #
+# Source types                                                         #
+# ------------------------------------------------------------------ #
+
+class HttpSource(BaseModel):
+    model_config = {"populate_by_name": True}
+    type: Literal["http"]
+    url: str
+    method: Literal["GET", "POST"] = "GET"
+    auth: Auth | None = None
+    params: list[Param] = []
+    response: Response = Response(format="json")
+    rate_limit: RateLimit | None = None
+    schema_: Schema | None = Field(default=None, alias="schema")
+
+
+class PythonSource(BaseModel):
+    """Calls a Python function that returns list[dict]; handles its own pagination."""
+    model_config = {"populate_by_name": True}
+    type: Literal["python"]
+    module: str       # importable module path, e.g. "connectors.craigslist_apts"
+    function: str     # function name; called as fn(dynamic_params) -> list[dict]
+    params: list[Param] = []
+    schema_: Schema | None = Field(default=None, alias="schema")
+
+
+class PubSubSource(BaseModel):
+    """Continuously reads JSON messages from a GCP Pub/Sub subscription."""
+    model_config = {"populate_by_name": True}
+    type: Literal["pubsub"]
+    subscription: str   # full resource path: projects/<project>/subscriptions/<name>
+    schema_: Schema | None = Field(default=None, alias="schema")
+
+
+Source = Annotated[Union[HttpSource, PythonSource, PubSubSource], Field(discriminator="type")]
+
+
+# ------------------------------------------------------------------ #
+# Cadence                                                              #
+# ------------------------------------------------------------------ #
+
+class StagingConfig(BaseModel):
+    partition_param: str        # which iterate param splits into separate staging tables
+    table_pattern: str          # e.g. "permits_{date_type}_loader_staging"
+
+
+class MergeDedup(BaseModel):
+    type: Literal["latest_non_null"]
+    columns: list[str]
+
+
+class MergeConfig(BaseModel):
+    table: str
+    key: str
+    dedup: MergeDedup | None = None
+
+
+class Cadence(BaseModel):
+    iterate: list[IterateSpec] = []
+    strategy: Literal["incremental", "append", "full_refresh"]
+    primary_key: str | None = None
+    staging: StagingConfig | None = None
+    merge: MergeConfig | None = None
+
+
+# ------------------------------------------------------------------ #
+# Deployment                                                           #
+# ------------------------------------------------------------------ #
+
+class Deployment(BaseModel):
+    type: Literal["batch", "streaming"] = "batch"
+    # batch fields
+    schedule: str | None = None
+    paused: bool = False
+    # streaming fields
+    window_seconds: int = 60
+
+    @model_validator(mode="after")
+    def validate_deployment(self) -> "Deployment":
+        if self.type == "batch":
+            if not self.schedule:
+                raise ValueError(
+                    "deployment.schedule is required for batch deployments "
+                    "(e.g. schedule: \"0 8 * * *\")"
+                )
+            parts = self.schedule.strip().split()
+            if len(parts) != 5 or not all(_CRON_FIELD_RE.match(p) for p in parts):
+                raise ValueError(
+                    f"deployment.schedule '{self.schedule}' is not a valid cron expression. "
+                    "Expected 5 space-separated fields: minute hour day-of-month month day-of-week "
+                    "(e.g. '0 8 * * *' for daily at 8 AM UTC)"
+                )
+        return self
+
+
+# ------------------------------------------------------------------ #
+# Collector (top-level)                                                #
+# ------------------------------------------------------------------ #
+
+class Collector(BaseModel):
+    name: str
+    namespace: str | None = None   # warehouse namespace; defaults to collector name when absent
+    description: str | None = None
+    source: Source
+    cadence: Cadence
+    deployment: Deployment | None = None
+
+    model_config = {"populate_by_name": True}
+
+    @classmethod
+    def model_fields_set(cls):
+        return super().model_fields_set()
+
+    @model_validator(mode="after")
+    def all_dynamic_params_have_iterators(self) -> "Collector":
+        if isinstance(self.source, (HttpSource, PythonSource)):
+            _validate_dynamic_params(self.source.params, self.cadence.iterate)
+        return self
+
+    @classmethod
+    def from_dict(cls, data: dict) -> Collector:
+        return cls.model_validate(data)

dcf/engine/__init__.py

@@ -0,0 +1,6 @@
+from .runner import run_collector
+from .iterator import build_request_sequence
+from .fetcher import fetch_records
+from .projector import project
+
+__all__ = ["run_collector", "build_request_sequence", "fetch_records", "project"]

dcf/engine/fetcher.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import io
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import requests
+
+from ..config.models import HttpSource, PythonSource, Param
+
+
+def _resolve_static_params(params: list[Param]) -> dict[str, Any]:
+    return {p.name: p.value for p in params if p.value is not None}
+
+
+def _rate_limit_sleep(rate_limit) -> None:
+    if rate_limit is None:
+        return
+    sleep_secs = (rate_limit.per_minutes * 60) / rate_limit.requests
+    time.sleep(sleep_secs)
+
+
+def _get_nested(record: dict, path: str) -> Any:
+    """Resolve a dot-notation path into a nested dict."""
+    parts = path.split(".")
+    val = record
+    for part in parts:
+        if not isinstance(val, dict):
+            return None
+        val = val.get(part)
+    return val
+
+
+def _parse_response(response: requests.Response, source: HttpSource) -> list[dict]:
+    fmt = source.response.format
+
+    if fmt == "csv":
+        try:
+            df = pd.read_csv(io.StringIO(response.text))
+        except pd.errors.EmptyDataError:
+            return []
+        return df.to_dict(orient="records")
+
+    if fmt == "json":
+        data = response.json()
+        if source.response.records_path:
+            for key in source.response.records_path.split("."):
+                if not isinstance(data, dict):
+                    raise ValueError(
+                        f"records_path '{source.response.records_path}' could not be followed: "
+                        f"expected a JSON object at key '{key}' but found {type(data).__name__}. "
+                        f"If the response is a top-level array, omit records_path entirely."
+                    )
+                data = data.get(key, [])
+        if isinstance(data, list):
+            return data
+        return [data]
+
+    raise ValueError(f"Unsupported response format: '{fmt}'")
+
+
+def _fetch_http(source: HttpSource, dynamic_params: dict[str, Any]) -> list[dict]:
+    params = _resolve_static_params(source.params)
+    params.update(dynamic_params)
+
+    if source.auth and source.auth.type == "query_param":
+        params[source.auth.key] = source.auth.value
+
+    headers = {}
+    if source.auth and source.auth.type == "header":
+        headers[source.auth.key] = source.auth.value
+    if source.auth and source.auth.type == "bearer":
+        headers["Authorization"] = f"Bearer {source.auth.value}"
+
+    _rate_limit_sleep(source.rate_limit)
+
+    response = requests.request(
+        method=source.method,
+        url=source.url,
+        params=params,
+        headers=headers,
+        timeout=60,
+    )
+    try:
+        response.raise_for_status()
+    except requests.HTTPError:
+        status = response.status_code
+        hint = {
+            401: "Check that your API token or key is correct and has not expired.",
+            403: "Your credentials may lack the required permissions for this endpoint.",
+            404: "The URL may be wrong, or this resource does not exist.",
+            429: "Rate limit exceeded. Add a rate_limit block to your collector YAML to slow down requests.",
+        }.get(status, "")
+        msg = f"HTTP {status} from {source.url}"
+        if hint:
+            msg += f" — {hint}"
+        raise requests.HTTPError(msg, response=response)
+    return _parse_response(response, source)
+
+
+def _fetch_python(source: PythonSource, dynamic_params: dict[str, Any]) -> list[dict]:
+    from ..project import find_project_root
+    project_root = str(find_project_root())
+    if project_root not in sys.path:
+        sys.path.insert(0, project_root)
+    import importlib
+    mod = importlib.import_module(source.module)
+    fn = getattr(mod, source.function)
+    return fn(dynamic_params)
+
+
+def fetch_records(source, dynamic_params: dict[str, Any]) -> list[dict]:
+    if isinstance(source, PythonSource):
+        return _fetch_python(source, dynamic_params)
+    return _fetch_http(source, dynamic_params)

dcf/engine/iterator.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import itertools
+from datetime import date, timedelta
+from typing import Any
+
+from ..config.models import DateRangeIterate, CategoricalIterate, IterateSpec, Param
+
+
+def _parse_duration(s: str) -> timedelta:
+    """Parse simple duration strings like '1 day', '7 days', '2 weeks'."""
+    parts = s.strip().split()
+    if len(parts) != 2:
+        raise ValueError(f"Cannot parse duration: '{s}'. Expected '<n> <unit>'")
+    n = int(parts[0])
+    unit = parts[1].rstrip("s")  # normalize "days" → "day"
+    if unit == "day":
+        return timedelta(days=n)
+    if unit == "week":
+        return timedelta(weeks=n)
+    if unit == "month":
+        return timedelta(days=30 * n)
+    raise ValueError(f"Unknown duration unit: '{unit}'")
+
+
+def _resolve_date(value: str) -> date:
+    if value == "today":
+        return date.today()
+    return date.fromisoformat(value)
+
+
+def _format_date(d: date, fmt: str | None, param_defs: dict[str, Param]) -> str:
+    """Apply the param's declared format, falling back to ISO."""
+    if fmt:
+        return d.strftime(fmt)
+    return d.isoformat()
+
+
+def _date_range_steps(spec: DateRangeIterate, param_defs: dict[str, Param]) -> list[dict[str, Any]]:
+    """
+    Yield one dict per step. Each dict maps param names to their formatted values.
+    When spec.params has two entries, the first receives the window start
+    and the second receives the window end.
+    """
+    step = _parse_duration(spec.step)
+    window = _parse_duration(spec.window) if spec.window else step
+
+    start = _resolve_date(spec.start)
+    end = _resolve_date(spec.end)
+
+    steps = []
+    window_start = start
+    while window_start <= end:
+        window_end = min(window_start + window - timedelta(days=1), end)
+
+        def fmt(d: date, param_name: str) -> str:
+            p = param_defs.get(param_name)
+            return _format_date(d, p.format if p else None, param_defs)
+
+        if len(spec.params) == 1:
+            steps.append({spec.params[0]: fmt(window_start, spec.params[0])})
+        else:
+            steps.append({
+                spec.params[0]: fmt(window_start, spec.params[0]),
+                spec.params[1]: fmt(window_end, spec.params[1]),
+            })
+        window_start += step
+    return steps
+
+
+def _categorical_steps(spec: CategoricalIterate) -> list[dict[str, Any]]:
+    return [{spec.param: v} for v in spec.values]
+
+
+def build_request_sequence(
+    iterate: list[IterateSpec],
+    param_defs: dict[str, Param],
+) -> list[dict[str, Any]]:
+    """
+    Return the cartesian product of all iteration axes.
+    Each element is a dict of {param_name: value} for one request.
+    """
+    if not iterate:
+        return [{}]
+
+    axes: list[list[dict[str, Any]]] = []
+    for spec in iterate:
+        if isinstance(spec, DateRangeIterate):
+            axes.append(_date_range_steps(spec, param_defs))
+        elif isinstance(spec, CategoricalIterate):
+            axes.append(_categorical_steps(spec))
+
+    return [
+        {k: v for d in combo for k, v in d.items()}
+        for combo in itertools.product(*axes)
+    ]

dcf/engine/projector.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+from ..config.models import Schema, Column
+from .transforms import apply_transform
+from ..engine.fetcher import _get_nested
+
+
+_CAST = {
+    "string":    str,
+    "integer":   lambda v: int(float(v)) if v is not None else None,
+    "float":     lambda v: float(v) if v is not None else None,
+    "boolean":   lambda v: bool(v) if v is not None else None,
+    "date":      lambda v: pd.to_datetime(v, errors="coerce"),
+    "timestamp": lambda v: pd.to_datetime(v, errors="coerce"),
+}
+
+
+def _cast(value: Any, col_type: str | None) -> Any:
+    if col_type is None or value is None:
+        return value
+    try:
+        return _CAST[col_type](value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _extract(record: dict, col: Column) -> Any:
+    if col.transform is not None:
+        return apply_transform(col.transform, record)
+    return _get_nested(record, col.path)
+
+
+def project(records: list[dict], schema: Schema | None) -> pd.DataFrame:
+    """
+    Apply transforms, extract declared columns only, cast types.
+    Columns not listed in the schema are dropped.
+    When schema is None, all fields are selected as-is.
+    """
+    if schema is None:
+        return pd.DataFrame(records)
+
+    if not records:
+        return pd.DataFrame(columns=[c.name for c in schema.columns])
+
+    rows = []
+    for record in records:
+        row = {}
+        for col in schema.columns:
+            row[col.name] = _cast(_extract(record, col), col.type)
+        rows.append(row)
+
+    return pd.DataFrame(rows)

dcf/engine/runner.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import textwrap
+import traceback
+
+from ..config.models import Collector, PythonSource
+from .iterator import build_request_sequence
+from .fetcher import fetch_records
+from .projector import project
+from .. import writer as iceberg_writer
+
+
+def run_collector(
+    collector: Collector,
+    catalog: str = "local",
+    limit: int | None = None,
+    param_overrides: dict | None = None,
+) -> None:
+    # GCS write path bypasses Spark entirely — skip JVM startup
+    if catalog == "gcp":
+        spark = None
+    else:
+        from dcf.spark_session import get_spark
+        spark = get_spark("dcf")
+
+    param_defs = {p.name: p for p in collector.source.params}
+    request_sequence = build_request_sequence(collector.cadence.iterate, param_defs)
+
+    if limit is not None:
+        request_sequence = request_sequence[:limit]
+
+    # Static params declared in the YAML (value is set) flow through to Python sources
+    static_params = {p.name: p.value for p in collector.source.params if p.value is not None}
+
+    print(f"\n[dcf] Running '{collector.name}' — {len(request_sequence)} requests\n")
+
+    failed = 0
+
+    for i, dynamic_params in enumerate(request_sequence, 1):
+        label = " ".join(f"{k}={v}" for k, v in dynamic_params.items())
+        print(f"  [{i}/{len(request_sequence)}] {label}")
+
+        # Build full params: static defaults → iterate values → CLI overrides
+        full_params = {**static_params, **dynamic_params, **(param_overrides or {})}
+
+        # For http sources, iterate-driven params are already handled in the fetcher;
+        # pass full_params only to python sources which need everything in one dict
+        source_params = full_params if isinstance(collector.source, PythonSource) else dynamic_params
+
+        try:
+            records = fetch_records(collector.source, source_params)
+        except Exception as e:
+            failed += 1
+            print(f"    fetch error ({type(e).__name__}): {e}")
+            print(textwrap.indent(traceback.format_exc(), "      "))
+            continue
+
+        if not records:
+            print(f"    0 records — skipping")
+            continue
+
+        df = project(records, collector.source.schema_)
+        print(f"    {len(df)} rows → writing")
+
+        iceberg_writer.write(spark, collector, df, catalog=catalog, dynamic_params=dynamic_params)
+
+    if catalog == "gcp":
+        from .. import writer as _w
+        bucket = _w.iceberg._gcs_warehouse_bucket()
+        if collector.namespace:
+            dest = f"gs://{bucket}/{collector.namespace}/{collector.name}/data"
+        else:
+            dest = f"gs://{bucket}/{collector.name}/data"
+    else:
+        from ..project import find_project_root
+        if collector.namespace:
+            dest = str(find_project_root() / "warehouse" / collector.namespace / collector.name / "data")
+        else:
+            dest = str(find_project_root() / "warehouse" / collector.name / "data")
+
+    total = len(request_sequence)
+    if failed == total:
+        print(f"\n[dcf] '{collector.name}' FAILED — all {total} iteration(s) errored → {dest}\n")
+    elif failed:
+        print(f"\n[dcf] '{collector.name}' complete with errors — {failed}/{total} iteration(s) failed → {dest}\n")
+    else:
+        print(f"\n[dcf] '{collector.name}' complete → {dest}\n")
+
+    if spark is not None:
+        spark.stop()