afly 0.1.0__py3-none-any.whl

afly/__init__.py

@@ -0,0 +1,5 @@
+"""afly — idempotent AppsFlyer → ClickHouse extractor."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

afly/alerting/__init__.py

@@ -0,0 +1,3 @@
+"""afly.alerting — run-failure notifications (project-level, not data-quality)."""
+
+from __future__ import annotations

afly/alerting/webhook.py

@@ -0,0 +1,92 @@
+"""Failure-alert delivery over a webhook (Mattermost/Slack "attachments", or plain JSON).
+
+This only ever fires for a *run-level* failure (the pipeline itself broke —
+auth, ClickHouse, an aborted run) — see
+``afly.config.project_config.ErrorAlertingConfig`` and
+``afly.run.runner._send_alerts``. It is not a data-quality alert.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable, Sequence
+from typing import Any
+
+import requests
+
+from afly import __version__
+from afly.config.profile import AlertChannelConfig
+
+logger = logging.getLogger(__name__)
+
+_ATTACHMENT_COLOR = "#D63232"
+
+
+def send_failure_alert(
+    channel: AlertChannelConfig,
+    *,
+    project: str,
+    profile: str,
+    run_id: str,
+    title: str,
+    lines: Sequence[str],
+    mentions: Sequence[str] = (),
+    post: Callable[..., Any] = requests.post,
+) -> bool:
+    """POST a failure alert to *channel*. Never raises — returns ``False`` and
+    logs a warning on any failure (network error, non-2xx, bad channel config).
+
+    The webhook URL (which may itself embed a token, per Mattermost/Slack
+    convention) is only ever used as the POST target — nothing here echoes
+    any credential back into the message body.
+    """
+    try:
+        payload = _build_payload(
+            channel, project=project, run_id=run_id, title=title, lines=lines, mentions=mentions
+        )
+        response = post(channel.webhook_url, json=payload, timeout=channel.timeout)
+        response.raise_for_status()
+        return True
+    except Exception:
+        logger.warning(
+            "failed to send failure alert via %s channel for run %s",
+            channel.type,
+            run_id,
+            exc_info=True,
+        )
+        return False
+
+
+def _build_payload(
+    channel: AlertChannelConfig,
+    *,
+    project: str,
+    run_id: str,
+    title: str,
+    lines: Sequence[str],
+    mentions: Sequence[str],
+) -> dict[str, Any]:
+    if channel.type == "webhook":
+        return {"title": title, "text": "\n".join(lines), "project": project, "run_id": run_id}
+
+    # mattermost / slack: both speak the same incoming-webhook "attachments" shape.
+    payload: dict[str, Any] = {
+        "username": channel.username,
+        "text": " ".join(mentions),
+        "attachments": [
+            {
+                "color": _ATTACHMENT_COLOR,
+                "title": title,
+                "text": "\n".join(lines),
+                "footer": f"afly {__version__}",
+            }
+        ],
+    }
+    if channel.icon_emoji:
+        payload["icon_emoji"] = channel.icon_emoji
+    if channel.channel:
+        payload["channel"] = channel.channel
+    return payload
+
+
+__all__ = ["send_failure_alert"]

afly/appsflyer/__init__.py

@@ -0,0 +1 @@
+"""AppsFlyer HTTP client: errors, retry policy, and the Pull/management APIs."""

afly/appsflyer/client.py

@@ -0,0 +1,83 @@
+"""Thin HTTP wrapper around AppsFlyer's REST APIs.
+
+Deliberately does *not* interpret response status codes — that's
+``afly.appsflyer.errors.classify_response``'s job, applied by the caller
+(``pull_api``/``mng_api``) once it knows which endpoint's error semantics
+apply. This class only owns: auth headers, timeouts, redirect-following, and
+turning a network-level failure (not an HTTP error response) into a
+:class:`TransientError` the retry policy already knows how to handle.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+import requests
+
+from afly import __version__
+from afly.appsflyer.errors import TransientError
+
+_DEFAULT_BASE_URL = "https://hq1.appsflyer.com"
+
+
+class AppsFlyerClient:
+    """Authenticated HTTP client for one AppsFlyer account (one token)."""
+
+    def __init__(
+        self,
+        token: str,
+        base_url: str = _DEFAULT_BASE_URL,
+        timeout_seconds: int = 120,
+        user_agent: str | None = None,
+        session: requests.Session | None = None,
+    ) -> None:
+        self._token = token
+        self.base_url = base_url.rstrip("/")
+        self.timeout_seconds = timeout_seconds
+        self.user_agent = user_agent or f"afly/{__version__}"
+        self._session = session or requests.Session()
+        self.api_calls = 0
+
+    def reset_counter(self) -> None:
+        """Zero the ``api_calls`` counter (e.g. between extracts in a run)."""
+        self.api_calls = 0
+
+    def get(
+        self,
+        path: str,
+        params: Mapping[str, Any] | None = None,
+        accept: str = "application/json",
+    ) -> requests.Response:
+        """GET ``base_url + path``, following redirects, with afly's standard headers.
+
+        Increments ``api_calls`` exactly once per call — including a call
+        that raises, since the network round-trip still happened and counts
+        against AppsFlyer's rate limit either way.
+        """
+        url = f"{self.base_url}{path}"
+        headers = {
+            "authorization": f"Bearer {self._token}",
+            "accept": accept,
+            "user-agent": self.user_agent,
+        }
+        try:
+            response = self._session.get(
+                url,
+                params=params,
+                headers=headers,
+                timeout=self.timeout_seconds,
+                allow_redirects=True,
+            )
+        except requests.RequestException as exc:
+            self.api_calls += 1
+            raise TransientError(
+                f"network error calling AppsFlyer: {exc}", status=None, body=str(exc), url=url
+            ) from exc
+
+        self.api_calls += 1
+        return response
+
+    def __repr__(self) -> str:
+        masked = f"{self._token[:4]}…" if self._token else "(empty)"
+        return f"AppsFlyerClient(base_url={self.base_url!r}, token={masked!r})"

afly/appsflyer/errors.py

@@ -0,0 +1,204 @@
+"""Exception hierarchy for the AppsFlyer HTTP client, and response classification.
+
+Everything afly's retry/backoff logic (``afly.appsflyer.retry.RetryPolicy``)
+and callers need to decide is captured in *which exception type* a call
+raised — never a bare status-code check scattered across the codebase. This
+module is the single place an HTTP response becomes one of these types, so
+that decision is made once and made consistently.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from email.utils import parsedate_to_datetime
+
+_LIMIT_MARKER = "limit reached for"
+
+
+class AppsFlyerError(Exception):
+    """Base for every error raised talking to the AppsFlyer API.
+
+    ``body_excerpt`` is capped at 300 chars — enough to see the error
+    message AppsFlyer sent without risking a multi-KB HTML error page ending
+    up in a log line or a CLI's stderr.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status: int | None = None,
+        body: str = "",
+        url: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.status = status
+        self.body_excerpt = (body or "")[:300]
+        self.url = url
+
+
+class AuthError(AppsFlyerError):
+    """401, or a 403 that never turned into a real quota response after retries."""
+
+
+class RateLimitError(AppsFlyerError):
+    """403 with the ``"Limit reached for"`` marker, or a plain 429."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status: int | None = None,
+        body: str = "",
+        url: str | None = None,
+        retry_after: float | None = None,
+        scope_hint: str | None = None,
+    ) -> None:
+        super().__init__(message, status=status, body=body, url=url)
+        self.retry_after = retry_after
+        self.scope_hint = scope_hint
+
+
+class TransientError(AppsFlyerError):
+    """5xx, a network/timeout failure, or a 403 without the quota marker.
+
+    AppsFlyer sometimes fronts a real quota/abuse block with a bare 403 that
+    carries no ``"Limit reached for"`` text — indistinguishable, at the HTTP
+    layer, from a transient abuse-protection hiccup. Treating it as
+    transient (retry with backoff) is the safe default; see
+    ``RetryPolicy.execute`` for what happens once retries are exhausted.
+    """
+
+
+class PermanentError(AppsFlyerError):
+    """Any other 4xx: bad app id (404), bad params/date-range (400), etc."""
+
+
+class EmptyBodyError(AppsFlyerError):
+    """A 200 response whose body has no CSV header line at all."""
+
+
+def classify_response(status: int, body: str, headers: object) -> type[AppsFlyerError] | None:
+    """Map an HTTP status + body to an error class, or ``None`` for success.
+
+    ``headers`` isn't used to *classify* (only the status and, for 403, the
+    body marker matter) — it's accepted here so callers can pass the same
+    three values they'd pass to :func:`build_error` without re-shaping them.
+    """
+    if 200 <= status < 300:
+        return None
+    if status == 401:
+        return AuthError
+    if status == 403:
+        return RateLimitError if _LIMIT_MARKER in body.lower() else TransientError
+    if status == 429:
+        return RateLimitError
+    if 500 <= status < 600:
+        return TransientError
+    return PermanentError
+
+
+def parse_retry_after(value: str | None, now: datetime | None = None) -> float | None:
+    """Parse a ``Retry-After`` header value into seconds from *now*.
+
+    Accepts both forms RFC 7231 allows: an integer/float number of seconds,
+    or an HTTP-date. Returns ``None`` for a missing/unparseable value, and
+    never a negative number (a date already in the past clamps to 0).
+    """
+    if value is None:
+        return None
+    stripped = value.strip()
+    if not stripped:
+        return None
+
+    try:
+        return max(float(stripped), 0.0)
+    except ValueError:
+        pass
+
+    try:
+        target = parsedate_to_datetime(stripped)
+    except (TypeError, ValueError, IndexError):
+        return None
+    if target.tzinfo is None:
+        target = target.replace(tzinfo=timezone.utc)
+
+    reference = now if now is not None else datetime.now(timezone.utc)
+    if reference.tzinfo is None:
+        reference = reference.replace(tzinfo=timezone.utc)
+
+    return max((target - reference).total_seconds(), 0.0)
+
+
+def _extract_scope_hint(body: str) -> str | None:
+    """Best-effort snippet of *what* quota was hit, for logging/diagnostics.
+
+    AppsFlyer's message shape after ``"Limit reached for "`` isn't
+    documented/stable, so this is deliberately a short, best-effort excerpt
+    rather than a parsed enum — good enough for a human reading a retry log,
+    not meant to be branched on.
+    """
+    lowered = body.lower()
+    idx = lowered.find(_LIMIT_MARKER)
+    if idx == -1:
+        return None
+    tail = body[idx + len(_LIMIT_MARKER) : idx + len(_LIMIT_MARKER) + 120]
+    for sep in (".", "\n", '"'):
+        pos = tail.find(sep)
+        if pos != -1:
+            tail = tail[:pos]
+    tail = tail.strip()
+    return tail or None
+
+
+def build_error(
+    status: int,
+    body: str,
+    *,
+    url: str | None = None,
+    headers: object = None,
+) -> AppsFlyerError:
+    """Build the right :class:`AppsFlyerError` subclass instance for *status*/*body*.
+
+    Raises :class:`ValueError` if *status* isn't actually an error status —
+    callers should check :func:`classify_response` (or just call this only
+    after confirming the response failed) rather than relying on this as the
+    classification step itself.
+    """
+    headers = headers or {}
+    error_cls = classify_response(status, body, headers)
+    if error_cls is None:
+        raise ValueError(f"status {status} is not an AppsFlyer error status")
+
+    if error_cls is RateLimitError:
+        retry_after_raw = _get_header(headers, "Retry-After")
+        return RateLimitError(
+            f"AppsFlyer rate limit hit (status {status})",
+            status=status,
+            body=body,
+            url=url,
+            retry_after=parse_retry_after(retry_after_raw),
+            scope_hint=_extract_scope_hint(body),
+        )
+
+    messages = {
+        AuthError: f"AppsFlyer authentication failed (status {status})",
+        TransientError: f"AppsFlyer transient error (status {status})",
+        PermanentError: f"AppsFlyer request rejected (status {status})",
+    }
+    return error_cls(messages[error_cls], status=status, body=body, url=url)
+
+
+def _get_header(headers: object, name: str) -> str | None:
+    """Case-insensitively fetch *name* from a mapping-like ``headers`` object."""
+    if headers is None:
+        return None
+    getter = getattr(headers, "get", None)
+    if getter is None:
+        return None
+    value = getter(name)
+    if value is None:
+        value = getter(name.lower())
+    if value is None:
+        value = getter(name.upper())
+    return value

afly/appsflyer/mng_api.py

@@ -0,0 +1,98 @@
+"""AppsFlyer management API — the account's app list.
+
+Used by ``afly apps`` (a standalone lookup) and by ``afly debug``/``afly run``
+to validate an app id before spending a Pull API call on it.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from afly.appsflyer.client import AppsFlyerClient
+from afly.appsflyer.errors import build_error, classify_response
+from afly.appsflyer.retry import RetryPolicy
+
+_APPS_PATH = "/api/mng/apps"
+
+
+@dataclass(frozen=True)
+class AppInfo:
+    """One app from the AppsFlyer account, as much as afly needs of it."""
+
+    id: str
+    name: str
+    platform: str
+    currency: str | None
+    time_zone: str | None
+
+
+def list_apps(client: AppsFlyerClient, policy: RetryPolicy, limit: int = 1000) -> list[AppInfo]:
+    """Fetch every app visible to *client*'s token, paginating by offset.
+
+    Stops once a page returns fewer than *limit* rows, or once ``offset``
+    reaches ``meta.total_items`` — whichever comes first, so a server that
+    omits ``total_items`` (or gets it wrong) still terminates correctly off
+    the short-page signal alone.
+    """
+    apps: list[AppInfo] = []
+    offset = 0
+    total_items: int | None = None
+
+    while True:
+        page = _fetch_page(client, policy, limit, offset)
+        data = page.get("data") or []
+        apps.extend(_to_app_info(item) for item in data)
+
+        meta = page.get("meta") or {}
+        if total_items is None:
+            total_items = meta.get("total_items")
+
+        offset += limit
+        if len(data) < limit:
+            break
+        if total_items is not None and offset >= total_items:
+            break
+
+    apps.sort(key=lambda a: a.id)
+    return apps
+
+
+def _fetch_page(
+    client: AppsFlyerClient, policy: RetryPolicy, limit: int, offset: int
+) -> dict[str, Any]:
+    def _attempt() -> dict[str, Any]:
+        response = client.get(
+            _APPS_PATH, params={"limit": limit, "offset": offset}, accept="application/json"
+        )
+        error_cls = classify_response(response.status_code, response.text, response.headers)
+        if error_cls is not None:
+            raise build_error(
+                response.status_code, response.text, url=response.url, headers=response.headers
+            )
+        result: dict[str, Any] = response.json()
+        return result
+
+    return policy.execute(_attempt)
+
+
+def _to_app_info(item: dict[str, Any]) -> AppInfo:
+    attrs = item.get("attributes") or {}
+    return AppInfo(
+        id=item.get("id", ""),
+        name=attrs.get("name", ""),
+        platform=attrs.get("platform", ""),
+        currency=attrs.get("currency"),
+        time_zone=attrs.get("time_zone"),
+    )
+
+
+def filter_apps(apps: list[AppInfo], platforms: list[str] | None) -> list[AppInfo]:
+    """Keep only apps whose platform matches one of *platforms* (case-insensitive).
+
+    ``platforms=None`` (or empty) means "no filter" — returns a copy of *apps*.
+    """
+    if not platforms:
+        return list(apps)
+    wanted = {p.lower() for p in platforms}
+    return [a for a in apps if (a.platform or "").lower() in wanted]

afly/appsflyer/pull_api.py

@@ -0,0 +1,158 @@
+"""AppsFlyer aggregate Pull API v5 — request building and report fetching.
+
+Split into a pure request-builder (:func:`build_pull_request`, easy to unit
+test param-by-param) and the actual network call (:func:`fetch_report`, which
+drives it through a :class:`~afly.appsflyer.retry.RetryPolicy`) — so every
+``category``/``reattr``/``extra_params`` edge case can be asserted on the
+URL/params alone, without mocking HTTP for each one.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import date
+from typing import Any
+
+from afly.appsflyer.client import AppsFlyerClient
+from afly.appsflyer.errors import EmptyBodyError, build_error, classify_response
+from afly.appsflyer.retry import RetryPolicy
+
+_RESERVED_PARAMS = ("from", "to")
+
+
+@dataclass(frozen=True)
+class PullRequestSpec:
+    """What to pull — the AppsFlyer-facing knobs of one extract, decoupled
+    from afly's own config model (see :meth:`from_extract`)."""
+
+    report_type: str
+    category: str = "standard"
+    media_source: str | None = None
+    reattr: bool = False
+    attribution_touch_type: str | None = None
+    timezone: str | None = None
+    currency: str | None = None
+    extra_params: Mapping[str, str] = field(default_factory=dict)
+
+    @classmethod
+    def from_extract(cls, extract: Any) -> PullRequestSpec:
+        """Build a spec from an ``ExtractConfig``-shaped object, via ``getattr``.
+
+        Reads by attribute name rather than importing ``afly.config``
+        (owned by a different milestone) — keeps this module usable and
+        testable independently of that package's exact type, both before it
+        exists and after.
+        """
+        return cls(
+            report_type=extract.report_type,
+            category=getattr(extract, "category", "standard") or "standard",
+            media_source=getattr(extract, "media_source", None),
+            reattr=bool(getattr(extract, "reattr", False)),
+            attribution_touch_type=getattr(extract, "attribution_touch_type", None),
+            timezone=getattr(extract, "timezone", None),
+            currency=getattr(extract, "currency", None),
+            extra_params=dict(getattr(extract, "extra_params", None) or {}),
+        )
+
+
+def build_pull_request(
+    spec: PullRequestSpec,
+    app_id: str,
+    from_date: date,
+    to_date: date,
+    base_url: str,
+) -> tuple[str, dict[str, str]]:
+    """Build the ``(path, params)`` for one Pull API v5 call.
+
+    ``base_url`` is accepted (not used in the returned path, which is always
+    the same ``/api/agg-data/export/...`` shape regardless of region) purely
+    so callers can pass everything they know about the target in one place
+    without the function silently depending on a client's private state.
+    """
+    path = f"/api/agg-data/export/app/{app_id}/{spec.report_type}/v5"
+    params: dict[str, str] = {"from": from_date.isoformat(), "to": to_date.isoformat()}
+
+    if spec.media_source:
+        params["media_source"] = spec.media_source
+    if spec.category and spec.category != "standard":
+        params["category"] = spec.category
+    if spec.reattr:
+        params["reattr"] = "true"
+    if spec.attribution_touch_type == "impression":
+        params["attribution_touch_type"] = "impression"
+    if spec.timezone:
+        params["timezone"] = spec.timezone
+    if spec.currency and spec.currency != "preferred":
+        params["currency"] = spec.currency
+
+    for key, value in spec.extra_params.items():
+        if key in _RESERVED_PARAMS:
+            raise ValueError(
+                f"extra_params may not override {key!r} — it's controlled by the pull window"
+            )
+        params[key] = value
+
+    return path, params
+
+
+@dataclass
+class RawReport:
+    """The raw fetched CSV plus enough metadata to log/debug the call."""
+
+    text: str
+    status: int
+    url: str
+    params: dict[str, str]
+    api_calls: int
+    elapsed_ms: int
+
+
+def fetch_report(
+    client: AppsFlyerClient,
+    spec: PullRequestSpec,
+    app_id: str,
+    from_date: date,
+    to_date: date,
+    policy: RetryPolicy,
+) -> RawReport:
+    """Fetch one report, retrying per *policy*. Returns the decoded CSV body.
+
+    ``api_calls``/``elapsed_ms`` on the result cover the *whole* fetch
+    (every retried attempt), not just the final successful one — that's what
+    a quota scheduler (M4) or a debug summary actually wants to know: what
+    did this fetch cost, not what did its last attempt cost.
+    """
+    path, params = build_pull_request(spec, app_id, from_date, to_date, client.base_url)
+    start_calls = client.api_calls
+    start_time = time.monotonic()
+
+    def _attempt() -> tuple[str, int, str]:
+        response = client.get(path, params=params, accept="text/csv")
+        error_cls = classify_response(response.status_code, response.text, response.headers)
+        if error_cls is not None:
+            raise build_error(
+                response.status_code, response.text, url=response.url, headers=response.headers
+            )
+
+        text = response.content.decode("utf-8-sig")
+        if not text.strip():
+            raise EmptyBodyError(
+                "AppsFlyer returned an empty report body",
+                status=response.status_code,
+                body="",
+                url=response.url,
+            )
+        return text, response.status_code, response.url
+
+    text, status, url = policy.execute(_attempt)
+
+    return RawReport(
+        text=text,
+        status=status,
+        url=url,
+        params=dict(params),
+        api_calls=client.api_calls - start_calls,
+        elapsed_ms=int((time.monotonic() - start_time) * 1000),
+    )