lorebase-scribe 0.2.3__py3-none-any.whl

lorebase_scribe/sources/access.py

@@ -0,0 +1,241 @@
+"""Guarded access to Source locations: local files under allowed roots, pinned HTTP requests.
+
+- Local files: the path is resolved (symbolic links included) and must lie under a resolved
+  root; the file is then opened from the root one component at a time without following
+  symbolic links, so a link swapped in after the check is refused instead of followed, and it
+  must be a regular file.
+- HTTP: the URL needs an allowed scheme and host; the host must resolve to global unicast
+  addresses only (unless private targets are allowed), checked as for webhooks, and requests
+  connect to those checked addresses, never to a new lookup (one lookup per host and client).
+  Redirects are never followed and response bodies are read in bounded steps.
+
+Messages name the settings section (``scribe.sources.csv``...) and never the resolved value of
+a connection, the host it resolves to or a header value.
+"""
+
+from __future__ import annotations
+
+import errno
+import os
+import stat
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path, PurePath
+from typing import Any
+from urllib.parse import urlsplit
+
+import httpx
+
+from lorebase_core.model import Source
+from lorebase_platform.secrets import SecretResolutionError, resolve_secret
+from lorebase_scribe.sources.mapping import SourceDefinitionError
+from lorebase_scribe.webhooks import pin
+from lorebase_store.webhooks import WebhookSettings
+from lorebase_store.webhooks.delivery import Resolver, check_target
+
+READ_CHUNK = 1024 * 1024
+"""Bytes read from a local file or a response per step; totals are bounded by settings."""
+
+
+class SourceReadError(RuntimeError):
+    """The Source could not be read (network, file system or database); a retry may work."""
+
+
+def resolve_location(source: Source) -> str:
+    """The value behind the Source's connection reference."""
+    try:
+        return resolve_secret(source.connection)
+    except SecretResolutionError as error:
+        raise SourceDefinitionError(f"connection: {error}") from None
+
+
+# Local files ----------------------------------------------------------------------------------
+
+
+def local_path(location: str, kind: str) -> Path:
+    """The path of a local location: a plain path or a ``file:`` URL."""
+    parts = urlsplit(location)
+    if parts.scheme == "file":
+        return Path(parts.path)
+    if parts.scheme:
+        raise SourceDefinitionError(f"connection: unsupported {kind} location scheme")
+    return Path(location)
+
+
+def _open_beneath(root: Path, relative: PurePath) -> int:
+    """A descriptor of `root / relative`, opened without following any symbolic link.
+
+    Each directory below `root` is opened relative to its parent with `O_NOFOLLOW`, and so is
+    the file itself (non-blocking, so a FIFO cannot hold the worker): a component replaced by
+    a symbolic link after the path was checked fails with ELOOP or ENOTDIR.
+    """
+    directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
+    try:
+        for part in relative.parts[:-1]:
+            child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory)
+            os.close(directory)
+            directory = child
+        return os.open(
+            relative.parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory
+        )
+    finally:
+        os.close(directory)
+
+
+def _beneath_root(path: Path, roots: Sequence[str], section: str) -> tuple[Path, PurePath]:
+    """The allowed root `path` resolves under, and the resolved path relative to it.
+
+    Symbolic links are resolved here, so a link inside a root that points inside a root keeps
+    working; the file is then opened along the resolved path, which has no links, without
+    following any (`_open_beneath`)."""
+    try:
+        resolved = path.resolve()
+        resolved_roots = [Path(root).resolve() for root in roots]
+    except (OSError, RuntimeError):
+        raise SourceReadError(f"{path.name}: the path cannot be resolved") from None
+    for root in resolved_roots:
+        if resolved != root and resolved.is_relative_to(root):
+            return root, resolved.relative_to(root)
+    raise SourceDefinitionError(f"connection: the path is outside {section}.allowed_roots")
+
+
+def open_local(path: Path, roots: Sequence[str], section: str) -> int:
+    """A read-only descriptor of the regular file `path` under one of `roots`."""
+    root, relative = _beneath_root(path, roots, section)
+    try:
+        descriptor = _open_beneath(root, relative)
+    except OSError as error:
+        if error.errno in (errno.ELOOP, errno.ENOTDIR):
+            raise SourceDefinitionError(
+                "connection: the path meets a symbolic link or a non-directory when opened "
+                f"under {section}.allowed_roots"
+            ) from None
+        raise SourceReadError(f"{path.name}: {error.strerror}") from None
+    try:
+        if not stat.S_ISREG(os.fstat(descriptor).st_mode):
+            raise SourceDefinitionError(f"{path.name}: not a regular file")
+    except BaseException:
+        os.close(descriptor)
+        raise
+    return descriptor
+
+
+def read_bounded(descriptor: int, name: str, limit: int, setting: str) -> bytes:
+    """At most `limit` bytes of the file behind `descriptor` (`setting` names the limit)."""
+    size = os.fstat(descriptor).st_size
+    if size > limit:
+        raise SourceDefinitionError(
+            f"{name}: {size} bytes, more than the {limit} allowed by {setting}"
+        )
+    chunks: list[bytes] = []
+    size = 0
+    while chunk := os.read(descriptor, min(READ_CHUNK, limit + 1 - size)):
+        size += len(chunk)
+        if size > limit:  # the file grew after fstat
+            raise SourceDefinitionError(f"{name}: more than the {limit} bytes allowed by {setting}")
+        chunks.append(chunk)
+    return b"".join(chunks)
+
+
+# HTTP -----------------------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class HttpRules:
+    """What a section of settings allows of URLs and responses."""
+
+    section: str
+    allowed_schemes: tuple[str, ...]
+    allowed_hosts: tuple[str, ...]
+    allow_private_targets: bool
+    max_bytes: int
+
+
+@dataclass(frozen=True)
+class Fetched:
+    """A response: its status, headers and, for a 200, its body."""
+
+    status: int
+    headers: httpx.Headers
+    body: bytes = b""
+
+
+@dataclass
+class PinnedClient:
+    """GET requests to checked URLs over an `httpx.AsyncClient` built with `trust_env=False`
+    and `follow_redirects=False` (owned by the caller)."""
+
+    client: httpx.AsyncClient
+    rules: HttpRules
+    resolve: Resolver
+    _addresses: dict[str, tuple[str, ...]] = field(default_factory=dict)
+
+    def check(self, url: httpx.URL, what: str) -> None:
+        """Refuse a URL whose scheme or host is not allowed (`what` starts the message)."""
+        if url.scheme not in self.rules.allowed_schemes:
+            raise SourceDefinitionError(
+                f"{what}: the URL scheme is not in {self.rules.section}.allowed_schemes"
+            )
+        if url.host not in self.rules.allowed_hosts:
+            raise SourceDefinitionError(
+                f"{what}: the URL host is not in {self.rules.section}.allowed_hosts"
+            )
+
+    async def addresses(self, url: httpx.URL, what: str) -> tuple[str, ...]:
+        """The checked addresses of the URL host, looked up once per client.
+
+        Empty when private targets are allowed: the client then resolves the host itself."""
+        if url.host in self._addresses:
+            return self._addresses[url.host]
+        checks = WebhookSettings(
+            allowed_schemes=self.rules.allowed_schemes,
+            allow_private_targets=self.rules.allow_private_targets,
+        )
+        target = await check_target(str(url), checks, self.resolve)
+        if target.problem is not None:
+            if target.problem.startswith("cannot resolve"):
+                raise SourceReadError(f"{what}: the URL host cannot be resolved")
+            # `check_target` problems never contain the host name or the URL.
+            raise SourceDefinitionError(
+                f"{what}: the URL host {target.problem}, refused unless "
+                f"{self.rules.section}.allow_private_targets"
+            )
+        self._addresses[url.host] = target.addresses
+        return target.addresses
+
+    async def get(self, url: httpx.URL, headers: Mapping[str, str], what: str) -> Fetched:
+        """GET `url` from its checked addresses; the body is read only for a 200."""
+        self.check(url, what)
+        addresses = await self.addresses(url, what)
+        if not addresses:
+            return await self._get(url, dict(headers), {})
+        failure: httpx.TransportError | None = None
+        for address in addresses:
+            pinned, host, extensions = pin(url, address)
+            try:
+                return await self._get(pinned, {**headers, **host}, extensions)
+            except (httpx.ConnectError, httpx.ConnectTimeout) as error:
+                failure = error  # the next checked address, never a new lookup
+        assert failure is not None
+        raise failure
+
+    async def _get(
+        self, target: httpx.URL, headers: dict[str, str], extensions: dict[str, Any]
+    ) -> Fetched:
+        limit = self.rules.max_bytes
+        chunks: list[bytes] = []
+        size = 0
+        async with self.client.stream(
+            "GET", target, headers=headers, extensions=extensions
+        ) as response:
+            if response.status_code != 200:
+                return Fetched(response.status_code, response.headers)
+            async for chunk in response.aiter_bytes():
+                size += len(chunk)
+                if size > limit:
+                    raise SourceDefinitionError(
+                        f"download larger than the {limit} bytes allowed by "
+                        f"{self.rules.section}.max_bytes"
+                    )
+                chunks.append(chunk)
+            return Fetched(200, response.headers, b"".join(chunks))

lorebase_scribe/sources/connectors.py

@@ -0,0 +1,320 @@
+"""Readers of Source rows (design 9.1): CSV, JSON, JSON Lines and Parquet files, JSON APIs,
+Postgres, MySQL/MariaDB and SQL Server tables and queries.
+
+A connector yields `(locator, row)` pairs; the locator (`name:row number`) becomes the row
+locator of the provenance entries. The connection of a Source is a secret reference; its
+value is the file location, the URL or the Postgres DSN.
+
+- CSV: a local path under one of `scribe.sources.csv.allowed_roots`, or an `http(s)` URL whose
+  scheme and host are allowed (`allowed_schemes`, `allowed_hosts`), both read through the
+  guards of `access` (no symbolic link followed after the roots check, regular files only,
+  hosts resolving to global unicast addresses only unless `allow_private_targets`, requests
+  pinned to the checked addresses, no redirects). Files and downloads are read in bounded
+  steps and refused past `max_bytes`. The delimiter is sniffed, the first decodable encoding
+  of `encodings` wins, and the first row is the header.
+- JSON, JSON Lines and Parquet files (`records`) and JSON APIs (`http_json`): see those
+  modules.
+- Postgres: the `table` (default: the last segment of the Source api name) or the `query` of
+  the Source, filtered by `scope_filter`, read in one read-only transaction with a statement
+  timeout and fetched `fetch_rows` at a time.
+- MySQL/MariaDB (`mysql_table`, `mysql_query`) and SQL Server (`mssql_table`, `mssql_query`):
+  the same, see `sql`; their drivers come with the `mysql` and `mssql` extras.
+
+Incremental Sources read only rows whose watermark column is greater than the highest watermark
+of the previous successful run (compared in SQL with the column's own type for Postgres, as text
+for CSV, so CSV watermarks should be ISO timestamps or zero-padded numbers, and natively, or as
+text when the types differ, for the other Sources).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import csv
+import io
+import os
+from collections.abc import AsyncIterator, Callable
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Protocol
+from urllib.parse import urlsplit
+
+import httpx
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import NullPool
+
+from lorebase_core.model import Source
+from lorebase_core.model.source import SourceType
+from lorebase_scribe.settings import CsvSourceSettings, PostgresSourceSettings, SourceSettings
+from lorebase_scribe.sources.access import (
+    HttpRules,
+    PinnedClient,
+    SourceReadError,
+    local_path,
+    open_local,
+    read_bounded,
+    resolve_location,
+)
+from lorebase_scribe.sources.http_json import HttpJsonConnector
+from lorebase_scribe.sources.mapping import Row, SourceDefinitionError
+from lorebase_scribe.sources.records import JsonConnector, JsonLinesConnector, ParquetConnector
+from lorebase_scribe.sources.sql import MssqlConnector, MysqlConnector, quote_identifier
+from lorebase_store.db import async_url
+from lorebase_store.webhooks.delivery import Resolver, resolve_host
+
+__all__ = [
+    "Connector",
+    "ConnectorFactory",
+    "CsvConnector",
+    "HttpJsonConnector",
+    "JsonConnector",
+    "JsonLinesConnector",
+    "MssqlConnector",
+    "MysqlConnector",
+    "ParquetConnector",
+    "PostgresConnector",
+    "SourceReadError",
+    "check_connection",
+    "default_connector",
+    "quote_identifier",
+]
+
+APPLICATION_NAME = "lorebase-scribe"
+CSV_SECTION = "scribe.sources.csv"
+
+
+class Connector(Protocol):
+    """Reads the rows of one Source."""
+
+    @property
+    def columns(self) -> tuple[str, ...]:
+        """Column names, known after the first row was read (header for CSV). Empty for
+        Sources without fixed columns (JSON records): a key missing from a row is null there,
+        and the mapped columns are checked against every key the run saw."""
+        ...
+
+    def rows(self, since: Any) -> AsyncIterator[tuple[str, Row]]:
+        """Rows in source order; with `since`, only rows whose watermark is greater."""
+        ...
+
+
+ConnectorFactory = Callable[[Source, SourceSettings], Connector]
+
+
+def check_connection(source: Source, settings: SourceSettings) -> None:
+    """Refuse a Source whose connection is not an allowed secret reference.
+
+    Sources come from the schema, written by API callers; resolving any reference they name
+    would let them read whatever the host can reach with it (the Lorebase database DSN
+    included). Messages here and below name the reference at most, never its value.
+    """
+    if source.connection not in settings.allowed_connections:
+        raise SourceDefinitionError(
+            f"connection {source.connection!r} is not in scribe.sources.allowed_connections"
+        )
+
+
+# CSV ----------------------------------------------------------------------------------------
+
+
+@dataclass
+class CsvConnector:
+    source: Source
+    settings: CsvSourceSettings
+    transport: httpx.AsyncBaseTransport | None = None
+    resolve: Resolver = resolve_host
+    _columns: tuple[str, ...] = ()
+
+    @property
+    def columns(self) -> tuple[str, ...]:
+        return self._columns
+
+    async def _load(self) -> tuple[str, bytes]:
+        location = resolve_location(self.source)
+        parts = urlsplit(location)
+        if parts.scheme in ("http", "https"):
+            return Path(parts.path).name or parts.netloc, await self._download(location)
+        return self._read_file(local_path(location, "CSV"))
+
+    def _read_file(self, path: Path) -> tuple[str, bytes]:
+        descriptor = open_local(path, self.settings.allowed_roots, CSV_SECTION)
+        try:
+            return path.name, read_bounded(
+                descriptor, path.name, self.settings.max_bytes, f"{CSV_SECTION}.max_bytes"
+            )
+        except OSError as error:
+            raise SourceReadError(f"{path.name}: {error.strerror}") from None
+        finally:
+            os.close(descriptor)
+
+    async def _download(self, location: str) -> bytes:
+        settings = self.settings
+        try:
+            url = httpx.URL(location)
+        except httpx.InvalidURL:
+            raise SourceDefinitionError("connection: not a valid URL") from None
+        rules = HttpRules(
+            CSV_SECTION,
+            settings.allowed_schemes,
+            settings.allowed_hosts,
+            settings.allow_private_targets,
+            settings.max_bytes,
+        )
+        try:
+            async with (
+                asyncio.timeout(settings.timeout_s),
+                httpx.AsyncClient(
+                    trust_env=False,
+                    follow_redirects=False,
+                    timeout=settings.timeout_s,
+                    transport=self.transport,
+                ) as client,
+            ):
+                fetched = await PinnedClient(client, rules, self.resolve).get(url, {}, "connection")
+        except TimeoutError:
+            raise SourceReadError(
+                f"GET {url.host}: not complete within {CSV_SECTION}.timeout_s"
+            ) from None
+        except httpx.HTTPError as error:
+            raise SourceReadError(f"GET {url.host}: {type(error).__name__}") from None
+        if fetched.status != 200:
+            raise SourceReadError(f"GET {url.host}: HTTP {fetched.status}")
+        return fetched.body
+
+    def _decode(self, name: str, data: bytes) -> str:
+        for encoding in self.settings.encodings:
+            try:
+                return data.decode(encoding)
+            except (UnicodeDecodeError, LookupError):
+                continue
+        raise SourceDefinitionError(
+            f"{name}: not decodable as {', '.join(self.settings.encodings)}"
+        )
+
+    def _delimiter(self, sample: str, name: str) -> str:
+        if name.lower().endswith((".tsv", ".tab")):
+            return "\t"
+        try:
+            return csv.Sniffer().sniff(sample, delimiters=self.settings.delimiters).delimiter
+        except csv.Error:
+            return ","
+
+    async def rows(self, since: Any) -> AsyncIterator[tuple[str, Row]]:
+        name, data = await self._load()
+        content = self._decode(name, data)
+        delimiter = self._delimiter(content[: self.settings.sniff_bytes], name)
+        reader = csv.reader(io.StringIO(content, newline=""), delimiter=delimiter)
+        try:
+            header = next(reader, None)
+            if header is None:
+                raise SourceDefinitionError(f"{name}: the file is empty")
+            self._columns = tuple(cell.strip() for cell in header)
+            watermark = self.source.watermark_column
+            for number, cells in enumerate(reader, start=2):
+                if not any(cell.strip() for cell in cells):
+                    continue
+                row = {
+                    c: (cells[i] if i < len(cells) else None) for i, c in enumerate(self._columns)
+                }
+                if since is not None and watermark is not None:
+                    value = row.get(watermark)
+                    if value is None or value <= str(since):
+                        continue
+                yield f"{name}:{number}", row
+        except csv.Error as error:
+            raise SourceDefinitionError(f"{name}: not valid delimited text ({error})") from None
+
+
+# Postgres -----------------------------------------------------------------------------------
+
+
+@dataclass
+class PostgresConnector:
+    source: Source
+    settings: PostgresSourceSettings
+    _columns: tuple[str, ...] = ()
+
+    @property
+    def columns(self) -> tuple[str, ...]:
+        return self._columns
+
+    def statement(self, since: Any) -> tuple[str, dict[str, Any], str]:
+        """SQL text, parameters and the relation name used in row locators."""
+        source = self.source
+        if source.type is SourceType.POSTGRES_QUERY:
+            if not (source.query or "").strip():
+                raise SourceDefinitionError("postgres_query Sources need a query")
+            body = f"SELECT * FROM ({source.query.strip().rstrip(';')}) AS lorebase_source"  # type: ignore[union-attr]
+            name = "query"
+        else:
+            name = source.table or source.api_name.rsplit(".", 1)[-1]
+            body = f"SELECT * FROM {quote_identifier(name)} AS lorebase_source"
+        conditions: list[str] = []
+        parameters: dict[str, Any] = {}
+        if source.scope_filter:
+            conditions.append(f"({source.scope_filter})")
+        order = ""
+        if source.watermark_column is not None:
+            column = quote_identifier(source.watermark_column)
+            if since is not None:
+                conditions.append(f"{column} > :since")
+                parameters["since"] = since
+            order = f" ORDER BY {column}"
+        where = f" WHERE {' AND '.join(conditions)}" if conditions else ""
+        return body + where + order, parameters, name
+
+    async def rows(self, since: Any) -> AsyncIterator[tuple[str, Row]]:
+        sql, parameters, name = self.statement(since)
+        try:
+            url = async_url(resolve_location(self.source))
+        except ValueError as error:
+            raise SourceDefinitionError(f"connection: {error}") from None
+        engine = create_async_engine(
+            url,
+            poolclass=NullPool,
+            connect_args={
+                "timeout": self.settings.connect_timeout_s,
+                "server_settings": {"application_name": APPLICATION_NAME},
+            },
+        )
+        try:
+            async with engine.connect() as connection, connection.begin():
+                await connection.execute(text("SET TRANSACTION READ ONLY"))
+                await connection.execute(
+                    text(f"SET LOCAL statement_timeout = {int(self.settings.statement_timeout_ms)}")
+                )
+                result = await connection.stream(text(sql), parameters)
+                self._columns = tuple(result.keys())
+                number = 0
+                async for partition in result.mappings().partitions(self.settings.fetch_rows):
+                    for row in partition:
+                        number += 1
+                        yield f"{name}:{number}", dict(row)
+        except (OSError, TimeoutError) as error:
+            raise SourceReadError(f"{name}: {type(error).__name__}: {error}") from None
+        finally:
+            await engine.dispose()
+
+
+def default_connector(source: Source, settings: SourceSettings) -> Connector:
+    """The connector of a Source type.
+
+    The connection must be one of `settings.allowed_connections`."""
+    check_connection(source, settings)
+    if source.type is SourceType.CSV:
+        return CsvConnector(source, settings.csv)
+    if source.type is SourceType.JSON:
+        return JsonConnector(source, settings.json_files)
+    if source.type is SourceType.JSONL:
+        return JsonLinesConnector(source, settings.json_files)
+    if source.type is SourceType.PARQUET:
+        return ParquetConnector(source, settings.parquet)
+    if source.type is SourceType.HTTP_JSON:
+        return HttpJsonConnector(source, settings.http_json, settings.allowed_connections)
+    if source.type in (SourceType.POSTGRES_TABLE, SourceType.POSTGRES_QUERY):
+        return PostgresConnector(source, settings.postgres)
+    if source.type in (SourceType.MYSQL_TABLE, SourceType.MYSQL_QUERY):
+        return MysqlConnector(source, settings.mysql)
+    if source.type in (SourceType.MSSQL_TABLE, SourceType.MSSQL_QUERY):
+        return MssqlConnector(source, settings.mssql)
+    raise SourceDefinitionError(f"Source type {source.type.value!r} is not supported")