prombed 0.1.0__py3-none-any.whl

prombed/__init__.py

@@ -0,0 +1,50 @@
+from .errors import PrombedError as PrombedError
+from .errors import PrombedErrorCode as PrombedErrorCode
+from .exposition import parse_exposition as parse_exposition
+from .prombed import Fetch as Fetch
+from .prombed import Prombed as Prombed
+from .promql import PromQLEngine as PromQLEngine
+from .promql import parse_duration as parse_duration
+from .promql import parse_promql as parse_promql
+from .storage import AppendResult as AppendResult
+from .storage import InMemoryStorage as InMemoryStorage
+from .storage import StorageAppender as StorageAppender
+from .storage import StoredSeries as StoredSeries
+from .storage import label_key as label_key
+from .types import (
+    ExpositionSample as ExpositionSample,
+)
+from .types import (
+    LabelMatcher as LabelMatcher,
+)
+from .types import (
+    LabelMatcherOperator as LabelMatcherOperator,
+)
+from .types import (
+    LabelSet as LabelSet,
+)
+from .types import (
+    MetricMetadata as MetricMetadata,
+)
+from .types import (
+    MetricType as MetricType,
+)
+from .types import (
+    ParsedExposition as ParsedExposition,
+)
+from .types import (
+    Sample as Sample,
+)
+from .types import (
+    ScrapeResult as ScrapeResult,
+)
+from .types import (
+    ScrapeTarget as ScrapeTarget,
+)
+from .types import (
+    ScrapeTargetHealth as ScrapeTargetHealth,
+)
+from .types import (
+    ScrapeTargetStatus as ScrapeTargetStatus,
+)
+from .value import STALE_NAN_BITS as STALE_NAN_BITS

prombed/errors.py

@@ -0,0 +1,20 @@
+from typing import Literal
+
+PrombedErrorCode = Literal[
+    "invalid_exposition",
+    "invalid_query",
+    "unsupported_promql_feature",
+    "query_execution_error",
+    "out_of_order_sample",
+    "duplicate_sample",
+    "storage_capacity_exceeded",
+    "scrape_failed",
+]
+
+
+class PrombedError(Exception):
+    """A stable, machine-readable failure from Prombed."""
+
+    def __init__(self, code: PrombedErrorCode, message: str) -> None:
+        super().__init__(message)
+        self.code = code

prombed/exposition.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import math
+import re
+
+from .errors import PrombedError
+from .types import ExpositionSample, MetricMetadata, ParsedExposition
+
+_METRIC_NAME = re.compile(r"^[A-Za-z_:][A-Za-z0-9_:]*$")
+_LABEL_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+_FLOAT = re.compile(r"^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$")
+_METRIC_TYPES: set[str] = {
+    "counter",
+    "gauge",
+    "histogram",
+    "summary",
+    "untyped",
+    "info",
+    "stateset",
+    "gaugehistogram",
+    "unknown",
+}
+
+
+def _invalid(line_number: int, message: str) -> None:
+    raise PrombedError(
+        "invalid_exposition",
+        f"invalid Prometheus exposition at line {line_number}: {message}",
+    )
+
+
+def _skip_whitespace(value: str, start: int) -> int:
+    index = start
+    while index < len(value) and value[index].isspace():
+        index += 1
+    return index
+
+
+def _parse_escaped_string(value: str, start: int, line_number: int) -> tuple[str, int]:
+    if start >= len(value) or value[start] != '"':
+        _invalid(line_number, "expected quoted label value")
+    output: list[str] = []
+    index = start + 1
+    while index < len(value):
+        char = value[index]
+        if char == '"':
+            return "".join(output), index + 1
+        if char != "\\":
+            output.append(char)
+            index += 1
+            continue
+        index += 1
+        if index >= len(value):
+            _invalid(line_number, "unterminated quoted label value")
+        escaped = value[index]
+        replacement = {"\\": "\\", '"': '"', "n": "\n"}.get(escaped)
+        if replacement is None:
+            _invalid(line_number, f"unsupported escape sequence \\{escaped}")
+        output.append(replacement)
+        index += 1
+    _invalid(line_number, "unterminated quoted label value")
+    raise AssertionError("unreachable")
+
+
+def _parse_labels(value: str, start: int, line_number: int) -> tuple[dict[str, str], int]:
+    labels: dict[str, str] = {}
+    index = start + 1
+    while True:
+        index = _skip_whitespace(value, index)
+        if index < len(value) and value[index] == "}":
+            return labels, index + 1
+        name_start = index
+        while index < len(value) and (value[index].isalnum() or value[index] == "_"):
+            index += 1
+        name = value[name_start:index]
+        if not _LABEL_NAME.fullmatch(name):
+            _invalid(line_number, f"invalid label name {name!r}")
+        if name in labels:
+            _invalid(line_number, f"duplicate label {name!r}")
+        index = _skip_whitespace(value, index)
+        if index >= len(value) or value[index] != "=":
+            _invalid(line_number, f"expected '=' after label {name!r}")
+        index = _skip_whitespace(value, index + 1)
+        parsed, index = _parse_escaped_string(value, index, line_number)
+        labels[name] = parsed
+        index = _skip_whitespace(value, index)
+        if index < len(value) and value[index] == ",":
+            index += 1
+            continue
+        if index < len(value) and value[index] == "}":
+            return labels, index + 1
+        _invalid(line_number, "expected ',' or '}' after label value")
+
+
+def _parse_float(token: str, line_number: int) -> float:
+    if token == "NaN":
+        return math.nan
+    if token in {"+Inf", "Inf"}:
+        return math.inf
+    if token == "-Inf":
+        return -math.inf
+    if not _FLOAT.fullmatch(token):
+        _invalid(line_number, f"invalid sample value {token!r}")
+    return float(token)
+
+
+def _parse_sample_line(line: str, line_number: int) -> ExpositionSample:
+    index = 0
+    while index < len(line) and not line[index].isspace() and line[index] != "{":
+        index += 1
+    name = line[:index]
+    if not _METRIC_NAME.fullmatch(name):
+        _invalid(line_number, f"invalid metric name {name!r}")
+    labels: dict[str, str] = {}
+    if index < len(line) and line[index] == "{":
+        labels, index = _parse_labels(line, index, line_number)
+    index = _skip_whitespace(line, index)
+    value_start = index
+    while index < len(line) and not line[index].isspace():
+        index += 1
+    sample_value = _parse_float(line[value_start:index], line_number)
+    index = _skip_whitespace(line, index)
+    timestamp: int | None = None
+    if index < len(line) and line[index] != "#":
+        timestamp_start = index
+        while index < len(line) and not line[index].isspace():
+            index += 1
+        token = line[timestamp_start:index]
+        if not re.fullmatch(r"-?\d+", token):
+            _invalid(line_number, f"invalid sample timestamp {token!r}")
+        timestamp = int(token)
+        if abs(timestamp) > 2**53 - 1:
+            _invalid(line_number, f"sample timestamp is outside the safe integer range: {token}")
+        index = _skip_whitespace(line, index)
+    if index < len(line) and line[index] != "#":
+        _invalid(line_number, f"unexpected trailing content {line[index:]!r}")
+    return ExpositionSample(name, labels, sample_value, timestamp)
+
+
+def _unescape_help(value: str, line_number: int) -> str:
+    output: list[str] = []
+    index = 0
+    while index < len(value):
+        char = value[index]
+        if char != "\\":
+            output.append(char)
+            index += 1
+            continue
+        index += 1
+        if index >= len(value):
+            _invalid(line_number, "unsupported HELP escape sequence \\")
+        escaped = value[index]
+        if escaped == "\\":
+            output.append("\\")
+        elif escaped == "n":
+            output.append("\n")
+        else:
+            _invalid(line_number, f"unsupported HELP escape sequence \\{escaped}")
+        index += 1
+    return "".join(output)
+
+
+def parse_exposition(value: str) -> ParsedExposition:
+    """Parse Prometheus text format 0.0.4 and its classic OpenMetrics subset."""
+
+    samples: list[ExpositionSample] = []
+    metadata: dict[str, MetricMetadata] = {}
+    descriptor_pattern = re.compile(r"^#\s*(HELP|TYPE|UNIT)\s+([^\s]+)\s+(.+)$")
+    for line_number, raw in enumerate(value.splitlines(), 1):
+        line = raw.strip()
+        if not line or line == "# EOF":
+            continue
+        if not line.startswith("#"):
+            samples.append(_parse_sample_line(line, line_number))
+            continue
+        match = descriptor_pattern.fullmatch(line)
+        if match is None:
+            continue
+        kind, name, raw_value = match.groups()
+        if not _METRIC_NAME.fullmatch(name):
+            _invalid(line_number, f"invalid metadata metric name {name!r}")
+        entry = metadata.setdefault(name, MetricMetadata())
+        if kind == "TYPE":
+            metric_type = raw_value.lower()
+            if metric_type not in _METRIC_TYPES:
+                _invalid(line_number, f"unsupported metric type {raw_value!r}")
+            entry.type = metric_type  # type: ignore[assignment]
+        elif kind == "HELP":
+            entry.help = _unescape_help(raw_value, line_number)
+        else:
+            entry.unit = raw_value
+    return ParsedExposition(samples, metadata)

prombed/prombed.py

@@ -0,0 +1,435 @@
+from __future__ import annotations
+
+import asyncio
+import math
+import time
+import urllib.request
+from collections.abc import Awaitable, Callable
+from dataclasses import replace
+from urllib.parse import urlsplit
+
+from .errors import PrombedError
+from .exposition import parse_exposition
+from .promql.evaluator import PromQLEngine
+from .storage import InMemoryStorage, StorageAppender, label_key
+from .types import (
+    ExpositionSample,
+    LabelSet,
+    Sample,
+    ScrapeResult,
+    ScrapeTarget,
+    ScrapeTargetStatus,
+)
+
+Fetch = Callable[[str, dict[str, str], float, int], Awaitable[bytes]]
+ScrapeErrorHandler = Callable[[PrombedError], None]
+
+_DEFAULT_SCRAPE_INTERVAL_MS = 5_000
+_DEFAULT_SCRAPE_TIMEOUT_MS = 5_000
+_DEFAULT_MAX_SCRAPE_BYTES = 16 * 1024 * 1024
+_FAMILY_SUFFIXES = ("_bucket", "_sum", "_count", "_created")
+_EXPORTED_LABEL_PREFIX = "exported_"
+_SCRAPE_REPORT_METRICS = {
+    "up": "up",
+    "duration": "scrape_duration_seconds",
+    "samples": "scrape_samples_scraped",
+    "samples_after_filter": "scrape_samples_post_metric_relabeling",
+    "series_added": "scrape_series_added",
+    "body_size": "scrape_body_size_bytes",
+}
+
+
+def _now_ms() -> int:
+    return time.time_ns() // 1_000_000
+
+
+def _allowed_metric(name: str, allowlist: tuple[str, ...]) -> bool:
+    if not allowlist:
+        return True
+    return any(
+        name == family or name in {family + suffix for suffix in _FAMILY_SUFFIXES}
+        for family in allowlist
+    )
+
+
+def _scrape_error(target: ScrapeTarget, error: Exception) -> PrombedError:
+    if isinstance(error, PrombedError):
+        return error
+    return PrombedError("scrape_failed", f"failed to scrape {target.url}: {error}")
+
+
+def _read_url(url: str, headers: dict[str, str], timeout: float, max_bytes: int) -> bytes:
+    if max_bytes <= 0:
+        raise PrombedError("scrape_failed", "scrape body limit must be positive")
+    request = urllib.request.Request(url, headers=headers)
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        declared = response.headers.get("content-length")
+        if declared is not None and declared.isdigit() and int(declared) > max_bytes:
+            raise PrombedError(
+                "scrape_failed",
+                f"metrics response declares {declared} bytes, exceeding the limit of {max_bytes}",
+            )
+        body = response.read(max_bytes + 1)
+    if len(body) > max_bytes:
+        raise PrombedError(
+            "scrape_failed", f"metrics response exceeds the limit of {max_bytes} bytes"
+        )
+    return body
+
+
+async def _default_fetch(
+    url: str, headers: dict[str, str], timeout: float, max_bytes: int
+) -> bytes:
+    return await asyncio.to_thread(_read_url, url, headers, timeout, max_bytes)
+
+
+# Port of Prometheus scrape.mutateSampleLabels with honor_labels=false as the default.
+def _mutate_sample_labels(exposed: LabelSet, target: LabelSet, honor: bool) -> LabelSet:
+    labels = dict(exposed)
+    if honor:
+        for name, value in target.items():
+            labels.setdefault(name, value)
+        return labels
+    conflicts: list[tuple[str, str]] = []
+    for name, value in target.items():
+        existing = labels.get(name, "")
+        if existing:
+            conflicts.append((name, existing))
+        labels[name] = value
+    conflicts.sort(key=lambda item: len(item[0]))
+    for name, value in conflicts:
+        exported = name
+        while labels.get(exported, ""):
+            exported = _EXPORTED_LABEL_PREFIX + exported
+        labels[exported] = value
+    return labels
+
+
+def _target_key(target: ScrapeTarget) -> str:
+    return label_key({**target.labels, "__url__": target.url})
+
+
+def _report_labels(target_labels: LabelSet, name: str) -> LabelSet:
+    return {**target_labels, "__name__": name}
+
+
+class Prombed:
+    """Embedded scrape, short-retention storage, and PromQL query runtime."""
+
+    def __init__(
+        self,
+        *,
+        targets: list[ScrapeTarget] | None = None,
+        retention_ms: int = 10 * 60_000,
+        max_series: int = 20_000,
+        max_samples_per_series: int = 10_000,
+        lookback_delta_ms: int = 5 * 60_000,
+        max_range_points: int = 11_000,
+        max_samples: int = 50_000_000,
+        scrape_interval_ms: int = _DEFAULT_SCRAPE_INTERVAL_MS,
+        scrape_timeout_ms: int = _DEFAULT_SCRAPE_TIMEOUT_MS,
+        max_scrape_bytes: int = _DEFAULT_MAX_SCRAPE_BYTES,
+        fetch: Fetch | None = None,
+        on_scrape_error: ScrapeErrorHandler | None = None,
+    ) -> None:
+        if scrape_interval_ms <= 0 or scrape_timeout_ms <= 0 or max_scrape_bytes <= 0:
+            raise PrombedError(
+                "scrape_failed", "scrape interval, timeout, and body limit must be positive"
+            )
+        self.storage = InMemoryStorage(
+            retention_ms=retention_ms,
+            max_series=max_series,
+            max_samples_per_series=max_samples_per_series,
+        )
+        self._engine = PromQLEngine(
+            self.storage,
+            lookback_delta_ms=lookback_delta_ms,
+            max_range_points=max_range_points,
+            max_samples=max_samples,
+        )
+        self._targets = list(targets or [])
+        self._scrape_interval_ms = scrape_interval_ms
+        self._scrape_timeout_ms = scrape_timeout_ms
+        self._max_scrape_bytes = max_scrape_bytes
+        self._fetch = fetch or _default_fetch
+        self._on_scrape_error = on_scrape_error
+        self._target_series: dict[str, dict[str, LabelSet]] = {}
+        self._target_statuses: dict[str, ScrapeTargetStatus] = {}
+        self._watch_task: asyncio.Task | None = None
+        for target in self._targets:
+            self._ensure_target(target)
+
+    def ingest(
+        self,
+        exposition: str,
+        *,
+        timestamp_ms: int | None = None,
+        labels: LabelSet | None = None,
+        metric_names: tuple[str, ...] = (),
+    ) -> int:
+        parsed = parse_exposition(exposition)
+        timestamp = timestamp_ms if timestamp_ms is not None else _now_ms()
+        appender = self.storage.appender(timestamp)
+        count = 0
+        try:
+            for sample in parsed.samples:
+                if not _allowed_metric(sample.name, metric_names):
+                    continue
+                appender.append(
+                    {**(labels or {}), **sample.labels, "__name__": sample.name},
+                    Sample(
+                        sample.timestamp if sample.timestamp is not None else timestamp,
+                        sample.value,
+                    ),
+                )
+                count += 1
+            appender.commit()
+            return count
+        except Exception:
+            appender.rollback()
+            raise
+
+    async def scrape_once(self, target: ScrapeTarget | None = None) -> list[ScrapeResult]:
+        selected = self._targets if target is None else [target]
+        return list(await asyncio.gather(*(self._scrape_target(item) for item in selected)))
+
+    async def start(self) -> list[ScrapeResult]:
+        if self._watch_task is not None:
+            return []
+        initial = await self.scrape_once()
+        self._watch_task = asyncio.create_task(self._watch(), name="prombed-scrape")
+        return initial
+
+    def stop(self) -> None:
+        if self._watch_task is not None:
+            self._watch_task.cancel()
+            self._watch_task = None
+
+    def targets(self) -> list[ScrapeTargetStatus]:
+        return [
+            replace(
+                status,
+                target=replace(
+                    status.target,
+                    labels=dict(status.target.labels),
+                    headers=dict(status.target.headers),
+                ),
+            )
+            for status in self._target_statuses.values()
+        ]
+
+    def query(self, expression: str, time_ms: int | None = None) -> dict:
+        return self._engine.query(expression, time_ms)
+
+    def query_range(self, expression: str, start_ms: int, end_ms: int, step_ms: int) -> dict:
+        return self._engine.query_range(expression, start_ms, end_ms, step_ms)
+
+    async def _watch(self) -> None:
+        try:
+            while True:
+                await asyncio.sleep(self._scrape_interval_ms / 1_000)
+                try:
+                    await self.scrape_once()
+                except Exception as exc:  # each target status already records the concrete failure
+                    if self._on_scrape_error is not None:
+                        failure = (
+                            exc
+                            if isinstance(exc, PrombedError)
+                            else PrombedError("scrape_failed", str(exc))
+                        )
+                        self._on_scrape_error(failure)
+        except asyncio.CancelledError:
+            return
+
+    async def _scrape_target(self, target: ScrapeTarget) -> ScrapeResult:
+        started_at = _now_ms()
+        scraped_at = started_at
+        key = self._ensure_target(target)
+        target_labels = dict(target.labels)
+        body_size = 0
+        try:
+            parsed_url = urlsplit(target.url)
+            if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
+                raise ValueError(f"invalid target URL {target.url!r}")
+            target_labels = {"instance": parsed_url.netloc, **target.labels}
+            timeout_ms = target.timeout_ms or self._scrape_timeout_ms
+            max_bytes = target.max_body_bytes or self._max_scrape_bytes
+            body = await self._fetch(
+                target.url,
+                {
+                    "Accept": "text/plain;version=0.0.4;q=1.0,application/openmetrics-text;q=0.5",
+                    **target.headers,
+                },
+                timeout_ms / 1_000,
+                max_bytes,
+            )
+            body_size = len(body)
+            if body_size > max_bytes:
+                raise PrombedError(
+                    "scrape_failed", f"metrics response exceeds the limit of {max_bytes} bytes"
+                )
+            parsed = parse_exposition(body.decode())
+            added = self._commit_scrape(
+                target, key, target_labels, parsed.samples, scraped_at, body_size
+            )
+            duration_ms = _now_ms() - started_at
+            self._report_target(target, key, started_at, duration_ms)
+            return ScrapeResult(target, scraped_at, added, duration_ms)
+        except Exception as exc:
+            failure = _scrape_error(target, exc)
+            duration_ms = _now_ms() - started_at
+            try:
+                self._commit_failed_scrape(key, target_labels, scraped_at, body_size, duration_ms)
+            except Exception as report_error:
+                failure = PrombedError(
+                    "scrape_failed",
+                    f"{failure}; additionally failed to record scrape failure: {report_error}",
+                )
+            self._report_target(target, key, started_at, duration_ms, failure)
+            if failure is exc:
+                raise
+            raise failure from exc
+
+    def _ensure_target(self, target: ScrapeTarget) -> str:
+        key = _target_key(target)
+        self._target_statuses.setdefault(key, ScrapeTargetStatus(target, "unknown"))
+        self._target_series.setdefault(key, {})
+        return key
+
+    def _commit_scrape(
+        self,
+        target: ScrapeTarget,
+        key: str,
+        target_labels: LabelSet,
+        samples: list[ExpositionSample],
+        timestamp: int,
+        body_size: int,
+    ) -> int:
+        appender = self.storage.appender(timestamp)
+        current: dict[str, LabelSet] = {}
+        added = 0
+        series_added = 0
+        try:
+            for sample in samples:
+                if not _allowed_metric(sample.name, target.metric_names):
+                    continue
+                labels = _mutate_sample_labels(
+                    {**sample.labels, "__name__": sample.name},
+                    target_labels,
+                    target.honor_labels,
+                )
+                result = appender.append(
+                    labels,
+                    Sample(
+                        sample.timestamp if sample.timestamp is not None else timestamp,
+                        sample.value,
+                    ),
+                )
+                series_added += int(result.created)
+                added += 1
+                if sample.timestamp is None or target.track_timestamps_staleness:
+                    current[label_key(labels)] = labels
+            self._append_stale_markers(
+                appender, self._target_series.get(key, {}), current, timestamp
+            )
+            self._append_scrape_report(
+                appender,
+                target_labels,
+                timestamp,
+                _now_ms() - timestamp,
+                len(samples),
+                added,
+                series_added,
+                body_size,
+                healthy=True,
+            )
+            appender.commit()
+            self._target_series[key] = current
+            return added
+        except Exception:
+            appender.rollback()
+            raise
+
+    def _commit_failed_scrape(
+        self,
+        key: str,
+        target_labels: LabelSet,
+        timestamp: int,
+        body_size: int,
+        duration_ms: int,
+    ) -> None:
+        appender = self.storage.appender(timestamp)
+        try:
+            self._append_stale_markers(appender, self._target_series.get(key, {}), {}, timestamp)
+            self._append_scrape_report(
+                appender,
+                target_labels,
+                timestamp,
+                duration_ms,
+                0,
+                0,
+                0,
+                body_size,
+                healthy=False,
+            )
+            appender.commit()
+            self._target_series[key] = {}
+        except Exception:
+            appender.rollback()
+            raise
+
+    @staticmethod
+    def _append_stale_markers(
+        appender: StorageAppender,
+        previous: dict[str, LabelSet],
+        current: dict[str, LabelSet],
+        timestamp: int,
+    ) -> None:
+        for key, labels in previous.items():
+            if key in current:
+                continue
+            try:
+                appender.append(labels, Sample(timestamp, math.nan, stale=True))
+            except PrombedError as exc:
+                if exc.code not in {"out_of_order_sample", "duplicate_sample"}:
+                    raise
+
+    @staticmethod
+    def _append_scrape_report(
+        appender: StorageAppender,
+        target_labels: LabelSet,
+        timestamp: int,
+        duration_ms: int,
+        scraped: int,
+        added: int,
+        series_added: int,
+        body_size: int,
+        *,
+        healthy: bool,
+    ) -> None:
+        values = (
+            (_SCRAPE_REPORT_METRICS["up"], 1.0 if healthy else 0.0),
+            (_SCRAPE_REPORT_METRICS["duration"], duration_ms / 1_000),
+            (_SCRAPE_REPORT_METRICS["samples"], float(scraped)),
+            (_SCRAPE_REPORT_METRICS["samples_after_filter"], float(added)),
+            (_SCRAPE_REPORT_METRICS["series_added"], float(series_added)),
+            (_SCRAPE_REPORT_METRICS["body_size"], float(body_size)),
+        )
+        for name, value in values:
+            appender.append(_report_labels(target_labels, name), Sample(timestamp, value))
+
+    def _report_target(
+        self,
+        target: ScrapeTarget,
+        key: str,
+        started_at: int,
+        duration_ms: int,
+        error: PrombedError | None = None,
+    ) -> None:
+        self._target_statuses[key] = ScrapeTargetStatus(
+            target,
+            "down" if error else "up",
+            str(error) if error else None,
+            started_at,
+            duration_ms,
+        )

prombed/promql/__init__.py

@@ -0,0 +1,3 @@
+from .evaluator import PromQLEngine as PromQLEngine
+from .parser import parse_duration as parse_duration
+from .parser import parse_promql as parse_promql