insto 0.1.0__py3-none-any.whl

insto/__init__.py

@@ -0,0 +1,3 @@
+from insto._version import __version__
+
+__all__ = ["__version__"]

insto/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from insto.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

insto/_redact.py

@@ -0,0 +1,88 @@
+"""Single source of truth for secret redaction.
+
+Used by `insto.cli._format_error` (user-facing error strings) and by the
+logging formatter (file-side log output). Anything that may end up in
+front of human eyes — stderr, log files, copy-pasted bug reports — passes
+through `redact_secrets()` first.
+
+Patterns covered:
+
+- the literal value of `$HIKERAPI_TOKEN` if it is set in the environment;
+- the literal values of any tokens / proxy credentials registered at
+  runtime via `register_secret()` (used by the config loader so a token
+  loaded from `~/.insto/config.toml` or supplied by `--proxy user:pass@`
+  is redacted the same way `$HIKERAPI_TOKEN` is);
+- query-string `signature=` and `token=` parameters in URLs (HikerAPI
+  signs every CDN URL — those signatures are short-lived but still
+  sensitive);
+- `Authorization: Bearer <token>` style headers if they ever surface
+  in an exception message;
+- `proxy://user:pass@host` userinfo segments.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import threading
+
+_QS_SECRET_RE = re.compile(
+    r"((?:^|[?&])(?:signature|token)=)[^&\s'\"]+",
+    re.IGNORECASE,
+)
+_BEARER_RE = re.compile(
+    r"(Bearer\s+)[A-Za-z0-9._~+/=-]+",
+    re.IGNORECASE,
+)
+_PROXY_USERINFO_RE = re.compile(
+    r"(\b[a-zA-Z][a-zA-Z0-9+.-]*://)([^:/@\s]+):([^@/\s]+)@",
+)
+
+_secrets_lock = threading.Lock()
+_registered_secrets: set[str] = set()
+
+
+def register_secret(value: str | None) -> None:
+    """Add `value` to the runtime redaction set.
+
+    Safe to call multiple times with the same value. Values shorter than
+    4 characters are ignored to avoid pathological matches against common
+    substrings. Threadsafe under a small mutex so logging handlers that
+    call `redact_secrets` concurrently never see a torn set.
+    """
+    if not value or len(value) < 4:
+        return
+    with _secrets_lock:
+        _registered_secrets.add(value)
+
+
+def clear_registered_secrets() -> None:
+    """Drop every value registered via `register_secret`. Useful in tests."""
+    with _secrets_lock:
+        _registered_secrets.clear()
+
+
+def redact_secrets(text: str) -> str:
+    """Return `text` with known secret-shaped substrings replaced with `***`.
+
+    Stable: never raises. Threadsafe; the registered-secrets set is read
+    under a short mutex.
+    """
+    if not text:
+        return text
+    redacted = text
+    env_token = os.environ.get("HIKERAPI_TOKEN")
+    if env_token and len(env_token) >= 4:
+        redacted = redacted.replace(env_token, "***")
+    with _secrets_lock:
+        registered = tuple(_registered_secrets)
+    for secret in registered:
+        if secret and secret in redacted:
+            redacted = redacted.replace(secret, "***")
+    redacted = _PROXY_USERINFO_RE.sub(r"\1***:***@", redacted)
+    redacted = _QS_SECRET_RE.sub(r"\1***", redacted)
+    redacted = _BEARER_RE.sub(r"\1***", redacted)
+    return redacted
+
+
+__all__ = ["clear_registered_secrets", "redact_secrets", "register_secret"]

insto/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

insto/backends/__init__.py

@@ -0,0 +1,52 @@
+"""Backend factory.
+
+`make_backend(name, **opts)` is the single entry point used by the service
+facade to construct a backend. Concrete backend modules are imported lazily
+so that pulling in `insto.backends` does not pay the cost (and surface the
+runtime dependency footprint) of every backend at once.
+
+Practically: `import insto` does not import `hikerapi`. Only the
+`make_backend("hiker", ...)` call does — and that import lives inside the
+function body.
+
+Setting `INSTO_BACKEND=fake` in the environment overrides the requested
+name with `"fake"` — a self-contained, network-free backend used by E2E
+tests. The override is intentionally global so the same CLI / REPL entry
+points the user runs are exercised end-to-end without test-only patches.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from insto.backends._base import OSINTBackend
+
+BACKEND_OVERRIDE_ENV = "INSTO_BACKEND"
+
+__all__ = ["BACKEND_OVERRIDE_ENV", "OSINTBackend", "make_backend"]
+
+
+def make_backend(name: str, **opts: Any) -> OSINTBackend:
+    """Construct a backend by short name.
+
+    Known names:
+        "hiker" — `HikerBackend` (HikerAPI SDK). Imports `hikerapi` lazily.
+        "fake"  — `FakeBackendProd`, hardcoded in-process data for E2E
+                  tests. Selected when `INSTO_BACKEND=fake` is set even if
+                  the caller asked for another backend.
+
+    Raises `ValueError` for unknown backend names.
+    """
+    override = os.environ.get(BACKEND_OVERRIDE_ENV)
+    if override:
+        name = override
+    if name == "hiker":
+        from insto.backends.hiker import HikerBackend
+
+        return HikerBackend(**opts)
+    if name == "fake":
+        from insto.backends._fake import FakeBackendProd
+
+        return FakeBackendProd(**opts)
+    raise ValueError(f"unknown backend: {name!r}")

insto/backends/_base.py

@@ -0,0 +1,135 @@
+"""Abstract OSINT backend interface.
+
+`OSINTBackend` is the contract every backend (HikerAPI v0.1, aiograpi v0.2,
+future TikTok / Bluesky / Threads providers) must implement. The command and
+service layers depend on this ABC, never on a concrete backend — that is what
+keeps v0.2 a pure addition.
+
+All collection-returning methods are async generators (`AsyncIterator[T]`)
+with an optional `limit: int | None` parameter. Cursors / page tokens are an
+internal implementation detail of each backend and never leak above this
+layer.
+
+The methods raise exceptions from `insto.exceptions` exclusively; raw HTTP /
+SDK errors must be mapped to the taxonomy by the backend itself.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator
+from typing import Any
+
+from insto.models import (
+    Comment,
+    Highlight,
+    HighlightItem,
+    Post,
+    Profile,
+    Quota,
+    Story,
+    User,
+)
+
+
+class OSINTBackend(ABC):
+    """Async OSINT data source for one social platform.
+
+    Implementations are expected to be safe for concurrent use within a single
+    asyncio event loop (the REPL drives one loop and may dispatch watch tasks
+    in parallel). They are NOT required to be process-safe.
+    """
+
+    # Capability tokens this backend exposes. Commands declare what they need
+    # via `@command(..., requires=("followed",))`; the dispatcher rejects the
+    # call when the active backend does not advertise the required tokens.
+    # HikerAPI exposes only public OSINT, so the default is empty; an
+    # `aiograpi` backend would extend this with `{"followed", ...}`.
+    capabilities: frozenset[str] = frozenset()
+
+    @abstractmethod
+    async def resolve_target(self, username: str) -> str:
+        """Return the stable `pk` for `username`, or raise `ProfileNotFound`."""
+
+    @abstractmethod
+    async def get_profile(self, pk: str) -> Profile:
+        """Fetch the full profile DTO for `pk`."""
+
+    @abstractmethod
+    async def get_user_about(self, pk: str) -> dict[str, Any]:
+        """Fetch the `user_about` payload (verification, dates, links)."""
+
+    @abstractmethod
+    def iter_user_posts(self, pk: str, *, limit: int | None = None) -> AsyncIterator[Post]:
+        """Iterate the user's feed posts in reverse chronological order."""
+
+    @abstractmethod
+    def iter_user_followers(self, pk: str, *, limit: int | None = None) -> AsyncIterator[User]:
+        """Iterate the user's followers."""
+
+    @abstractmethod
+    def iter_user_following(self, pk: str, *, limit: int | None = None) -> AsyncIterator[User]:
+        """Iterate accounts the user is following."""
+
+    @abstractmethod
+    def iter_user_tagged(self, pk: str, *, limit: int | None = None) -> AsyncIterator[Post]:
+        """Iterate posts the user is tagged in."""
+
+    @abstractmethod
+    def iter_user_highlights(
+        self, pk: str, *, limit: int | None = None
+    ) -> AsyncIterator[Highlight]:
+        """Iterate highlight reels owned by the user."""
+
+    @abstractmethod
+    def iter_highlight_items(
+        self, highlight_id: str, *, limit: int | None = None
+    ) -> AsyncIterator[HighlightItem]:
+        """Iterate items inside a highlight reel."""
+
+    @abstractmethod
+    def iter_post_comments(
+        self, media_pk: str, *, limit: int | None = None
+    ) -> AsyncIterator[Comment]:
+        """Iterate comments on a post."""
+
+    @abstractmethod
+    def iter_post_likers(self, media_pk: str, *, limit: int | None = None) -> AsyncIterator[User]:
+        """Iterate users who liked a post."""
+
+    @abstractmethod
+    def iter_user_stories(self, pk: str, *, limit: int | None = None) -> AsyncIterator[Story]:
+        """Iterate currently-active stories of a user."""
+
+    @abstractmethod
+    async def get_suggested(self, pk: str) -> list[User]:
+        """Fetch accounts suggested as similar to `pk`."""
+
+    @abstractmethod
+    def iter_hashtag_posts(self, tag: str, *, limit: int | None = None) -> AsyncIterator[Post]:
+        """Iterate top / recent posts under a hashtag."""
+
+    @abstractmethod
+    def get_quota(self) -> Quota:
+        """Return the last-known quota state for the backend."""
+
+    @abstractmethod
+    def get_last_error(self) -> BaseException | None:
+        """Return the last exception raised by this backend, if any."""
+
+    def get_schema_drift_count(self) -> int:
+        """Return the number of `SchemaDrift` errors observed this session.
+
+        Default 0 so simple backends (in-process fakes) need not track. Real
+        backends override to expose a running counter — surfaced by `/health`
+        so an operator can spot provider degradation.
+        """
+        return 0
+
+    async def aclose(self) -> None:  # noqa: B027 — intentional empty default
+        """Release backend-owned resources (HTTP clients, sockets, …).
+
+        Default implementation is a no-op so simple in-memory backends (the
+        test fakes, future mock backends) need not override. Real backends
+        with network clients (HikerBackend) override to close them.
+        """

insto/backends/_cdn.py

@@ -0,0 +1,343 @@
+"""CDN streamer with defense in depth.
+
+Single helper used by the service facade to download CDN-hosted media
+(profile pictures, posts, stories, highlights) into a target directory.
+All defenses live here so that command and service code never touch raw
+HTTP for media.
+
+Defenses (each one is exercised by tests/test_cdn.py):
+
+- HTTPS-only (http:// rejected, http:// redirects rejected)
+- Host allowlist (only `*.cdninstagram.com` and `*.fbcdn.net` accepted, both
+  for the initial URL and any redirect target)
+- Filename built from the caller-supplied ``dest`` (intended to be the post
+  pk or similar stable id) plus an extension chosen from response
+  Content-Type cross-checked against magic-byte sniffing — the CDN-supplied
+  filename in the URL is ignored entirely
+- Whitelist of extensions (.jpg/.jpeg/.png/.webp/.mp4/.mov)
+- Per-resource byte budget (default 500 MB)
+- Pre-flight free-disk check (default 1 GB minimum)
+- Atomic write: stream to ``<final>.part``, fsync, rename → final
+- Collision suffix: never overwrites an existing file; appends ``_<n>``
+- ``mtime`` is set from ``taken_at`` if supplied (so file dates match post
+  capture time, not download time)
+- macOS xattr tagging via ctypes ``setxattr(2)`` — adds
+  ``com.apple.metadata:kMDItemUserTags = insto`` so downloaded media is
+  visible in Finder Smart Folders. No-op on non-darwin. On darwin, an
+  ``OSError`` (e.g. NFS, exFAT) is reported once-per-process to stderr
+  and otherwise silently swallowed.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import ctypes
+import ctypes.util
+import os
+import shutil
+import sys
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+import httpx
+
+from insto.exceptions import BackendError
+
+ALLOWED_HOST_SUFFIXES: tuple[str, ...] = ("cdninstagram.com", "fbcdn.net")
+ALLOWED_EXTENSIONS: frozenset[str] = frozenset({".jpg", ".jpeg", ".png", ".webp", ".mp4", ".mov"})
+
+CT_TO_EXT: dict[str, str] = {
+    "image/jpeg": ".jpg",
+    "image/jpg": ".jpg",
+    "image/png": ".png",
+    "image/webp": ".webp",
+    "video/mp4": ".mp4",
+    "video/quicktime": ".mov",
+}
+
+DEFAULT_BYTE_BUDGET: int = 500 * 1024 * 1024
+DEFAULT_MIN_FREE_DISK: int = 1024 * 1024 * 1024
+SNIFF_SIZE: int = 512
+CHUNK_SIZE: int = 64 * 1024
+MAX_REDIRECTS: int = 5
+DEFAULT_TIMEOUT: float = 30.0
+
+_XATTR_NAME: bytes = b"com.apple.metadata:kMDItemUserTags"
+_XATTR_VALUE: bytes = b"insto"
+_XATTR_WARN_LINE: str = "note: filesystem does not support xattr; tagging skipped"
+
+_xattr_warned: bool = False
+
+
+def _is_host_allowed(host: str) -> bool:
+    host = host.lower()
+    return any(host == s or host.endswith("." + s) for s in ALLOWED_HOST_SUFFIXES)
+
+
+def _normalize_ct(ct: str | None) -> str | None:
+    if not ct:
+        return None
+    return ct.split(";", 1)[0].strip().lower() or None
+
+
+def _sniff(prefix: bytes) -> tuple[str, str] | None:
+    """Sniff magic bytes; return (extension, mime) or None."""
+    if prefix.startswith(b"\xff\xd8\xff"):
+        return ".jpg", "image/jpeg"
+    if prefix.startswith(b"\x89PNG\r\n\x1a\n"):
+        return ".png", "image/png"
+    if prefix.startswith(b"RIFF") and len(prefix) >= 12 and prefix[8:12] == b"WEBP":
+        return ".webp", "image/webp"
+    if len(prefix) >= 12 and prefix[4:8] == b"ftyp":
+        brand = prefix[8:12]
+        if brand == b"qt  ":
+            return ".mov", "video/quicktime"
+        return ".mp4", "video/mp4"
+    return None
+
+
+def _ct_compatible(declared: str, sniffed: str) -> bool:
+    if declared == sniffed:
+        return True
+    return {declared, sniffed} <= {"image/jpeg", "image/jpg"}
+
+
+def _resolve_collision(base: Path) -> Path:
+    """Return a non-existing path; if ``base`` exists, append ``_1``, ``_2`` …"""
+    if not base.exists():
+        return base
+    stem, suffix, parent = base.stem, base.suffix, base.parent
+    n = 1
+    while True:
+        candidate = parent / f"{stem}_{n}{suffix}"
+        if not candidate.exists():
+            return candidate
+        n += 1
+
+
+def _set_macos_tag(path: Path) -> None:
+    """Tag ``path`` with ``com.apple.metadata:kMDItemUserTags=insto`` on darwin.
+
+    No-op on other platforms. On darwin, errors from the underlying
+    ``setxattr(2)`` (e.g. filesystem without xattr support such as NFS or
+    exFAT) are swallowed and reported once per process to stderr.
+    """
+    global _xattr_warned
+    if sys.platform != "darwin":
+        return
+    libc_path = ctypes.util.find_library("c")
+    if libc_path is None:  # pragma: no cover - libc is always present on darwin
+        return
+    libc = ctypes.CDLL(libc_path, use_errno=True)
+    libc.setxattr.argtypes = [
+        ctypes.c_char_p,
+        ctypes.c_char_p,
+        ctypes.c_void_p,
+        ctypes.c_size_t,
+        ctypes.c_uint32,
+        ctypes.c_int,
+    ]
+    libc.setxattr.restype = ctypes.c_int
+    rc = libc.setxattr(
+        str(path).encode("utf-8"),
+        _XATTR_NAME,
+        _XATTR_VALUE,
+        len(_XATTR_VALUE),
+        0,
+        0,
+    )
+    if rc != 0 and not _xattr_warned:
+        _xattr_warned = True
+        print(_XATTR_WARN_LINE, file=sys.stderr)
+
+
+def _validate_url(url: str) -> None:
+    parsed = urlparse(url)
+    if parsed.scheme != "https":
+        raise BackendError(f"non-https CDN url rejected: {url}")
+    if not parsed.hostname or not _is_host_allowed(parsed.hostname):
+        raise BackendError(f"CDN host not in allowlist: {parsed.hostname}")
+
+
+def _validate_redirect(current: str, location: str) -> str:
+    new_url = urljoin(current, location)
+    parsed = urlparse(new_url)
+    if parsed.scheme != "https":
+        raise BackendError(f"CDN redirect to non-https rejected: {new_url}")
+    if not parsed.hostname or not _is_host_allowed(parsed.hostname):
+        raise BackendError(f"CDN cross-host redirect rejected: {parsed.hostname}")
+    return new_url
+
+
+def _decide_extension(declared_ct: str | None, sniffed: tuple[str, str]) -> str:
+    sniff_ext, sniff_mime = sniffed
+    if declared_ct is not None:
+        if not _ct_compatible(declared_ct, sniff_mime):
+            raise BackendError(
+                f"CDN content-type mismatch: header={declared_ct} sniff={sniff_mime}"
+            )
+        ext = CT_TO_EXT.get(declared_ct, sniff_ext)
+    else:
+        ext = sniff_ext
+    if ext not in ALLOWED_EXTENSIONS:
+        raise BackendError(f"CDN extension not in allowlist: {ext}")
+    return ext
+
+
+def _coerce_taken_at(taken_at: datetime | float | int | None) -> float | None:
+    if taken_at is None:
+        return None
+    if isinstance(taken_at, datetime):
+        return taken_at.timestamp()
+    return float(taken_at)
+
+
+async def stream_to_file(
+    url: str,
+    dest: Path,
+    *,
+    content_type_hint: str | None = None,
+    byte_budget: int = DEFAULT_BYTE_BUDGET,
+    taken_at: datetime | float | int | None = None,
+    client: httpx.AsyncClient | None = None,
+    min_free_disk: int = DEFAULT_MIN_FREE_DISK,
+    timeout: float = DEFAULT_TIMEOUT,
+) -> Path:
+    """Stream a CDN URL to ``dest`` (path *without* extension).
+
+    The chosen extension comes from the response Content-Type, cross-checked
+    against magic-byte sniffing of the leading bytes; if both the header and
+    sniff agree, that extension is appended to ``dest``. If the resulting
+    path already exists, ``_1`` / ``_2`` … is appended before the extension.
+
+    Returns the path actually written.
+    """
+
+    _validate_url(url)
+
+    parent = dest.parent
+    parent.mkdir(parents=True, exist_ok=True)
+
+    free = shutil.disk_usage(parent).free
+    if free < min_free_disk:
+        raise BackendError(f"insufficient disk space: {free} bytes free < {min_free_disk}")
+
+    owns_client = client is None
+    if client is None:
+        client = httpx.AsyncClient(follow_redirects=False, timeout=timeout)
+
+    try:
+        return await _download_with_redirects(
+            url=url,
+            dest=dest,
+            content_type_hint=content_type_hint,
+            byte_budget=byte_budget,
+            taken_at=_coerce_taken_at(taken_at),
+            client=client,
+        )
+    finally:
+        if owns_client:
+            await client.aclose()
+
+
+async def _download_with_redirects(
+    *,
+    url: str,
+    dest: Path,
+    content_type_hint: str | None,
+    byte_budget: int,
+    taken_at: float | None,
+    client: httpx.AsyncClient,
+) -> Path:
+    current_url = url
+    redirects = 0
+    while True:
+        async with client.stream("GET", current_url) as resp:
+            if 300 <= resp.status_code < 400:
+                if redirects >= MAX_REDIRECTS:
+                    raise BackendError("too many CDN redirects")
+                redirects += 1
+                location = resp.headers.get("location")
+                if not location:
+                    raise BackendError("CDN redirect without Location header")
+                current_url = _validate_redirect(current_url, location)
+                continue
+
+            if resp.status_code != 200:
+                raise BackendError(f"CDN GET failed: HTTP {resp.status_code}")
+
+            declared_ct = _normalize_ct(resp.headers.get("content-type")) or _normalize_ct(
+                content_type_hint
+            )
+            return await _stream_response(
+                resp=resp,
+                dest=dest,
+                declared_ct=declared_ct,
+                byte_budget=byte_budget,
+                taken_at=taken_at,
+            )
+
+
+async def _stream_response(
+    *,
+    resp: httpx.Response,
+    dest: Path,
+    declared_ct: str | None,
+    byte_budget: int,
+    taken_at: float | None,
+) -> Path:
+    parent = dest.parent
+    tmp_path = parent / (dest.name + ".part")
+    if tmp_path.exists():
+        tmp_path.unlink()
+
+    sniff_buf = bytearray()
+    sniffed: tuple[str, str] | None = None
+    total = 0
+
+    try:
+        with open(tmp_path, "wb") as fh:
+            async for chunk in resp.aiter_bytes(CHUNK_SIZE):
+                if not chunk:
+                    continue
+                if sniffed is None and len(sniff_buf) < SNIFF_SIZE:
+                    sniff_buf.extend(chunk[: SNIFF_SIZE - len(sniff_buf)])
+                    if len(sniff_buf) >= SNIFF_SIZE:
+                        sniffed = _sniff(bytes(sniff_buf))
+                        if sniffed is None:
+                            raise BackendError("CDN content-type sniff failed: unknown magic bytes")
+                        if declared_ct is not None and not _ct_compatible(declared_ct, sniffed[1]):
+                            raise BackendError(
+                                "CDN content-type mismatch: "
+                                f"header={declared_ct} sniff={sniffed[1]}"
+                            )
+                if total + len(chunk) > byte_budget:
+                    raise BackendError(f"CDN response exceeded byte budget {byte_budget}")
+                fh.write(chunk)
+                total += len(chunk)
+
+            if sniffed is None:
+                if not sniff_buf:
+                    raise BackendError("CDN response was empty")
+                sniffed = _sniff(bytes(sniff_buf))
+                if sniffed is None:
+                    raise BackendError("CDN content-type sniff failed: unknown magic bytes")
+            fh.flush()
+            os.fsync(fh.fileno())
+
+        ext = _decide_extension(declared_ct, sniffed)
+        final_base = parent / (dest.name + ext)
+        final = _resolve_collision(final_base)
+        os.rename(tmp_path, final)
+
+        if taken_at is not None:
+            os.utime(final, (taken_at, taken_at))
+
+        _set_macos_tag(final)
+        return final
+    except BaseException:
+        if tmp_path.exists():
+            with contextlib.suppress(OSError):
+                tmp_path.unlink()
+        raise