pymongoftdc 0.1.0__py3-none-any.whl

pyftdc/__init__.py

@@ -0,0 +1,13 @@
+"""Read time-series metrics from MongoDB FTDC archives."""
+
+from pyftdc.exceptions import FTDCDecodeError, FTDCError, MetricNotFoundError
+from pyftdc.models import DataPoint
+from pyftdc.reader import FTDCReader
+
+__all__ = [
+    "DataPoint",
+    "FTDCDecodeError",
+    "FTDCError",
+    "FTDCReader",
+    "MetricNotFoundError",
+]

pyftdc/_codec.py

@@ -0,0 +1,214 @@
+"""Low-level BSON framing and FTDC metric chunk decoding."""
+
+from __future__ import annotations
+
+import math
+import struct
+import zlib
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, BinaryIO, cast
+
+from bson import BSON
+from bson.codec_options import CodecOptions
+from bson.decimal128 import Decimal128
+from bson.timestamp import Timestamp
+
+from pyftdc.exceptions import FTDCDecodeError
+from pyftdc.models import MetricValue
+
+_UINT64_MASK = (1 << 64) - 1
+_MIN_BSON_SIZE = 5
+_CODEC_OPTIONS: CodecOptions[Any] = CodecOptions(tz_aware=True, tzinfo=timezone.utc)
+
+
+@dataclass(frozen=True, slots=True)
+class MetricSlot:
+    """A compressed numeric field and its location in a reference document."""
+
+    path: str
+    initial: int
+    kind: str
+    part: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class DecodedChunk:
+    """A reference document and its decoded metric rows."""
+
+    reference: Mapping[str, Any]
+    slots: tuple[MetricSlot, ...]
+    rows: tuple[tuple[int, ...], ...]
+
+
+def iter_bson_documents(stream: BinaryIO, source: Path) -> Iterator[Mapping[str, Any]]:
+    """Yield the concatenated BSON documents in an FTDC file."""
+
+    while prefix := stream.read(4):
+        if len(prefix) != 4:
+            raise FTDCDecodeError(f"{source}: truncated BSON length")
+        (length,) = struct.unpack("<I", prefix)
+        if length == 0:  # Zero bytes terminate an interim file.
+            return
+        if length < _MIN_BSON_SIZE:
+            raise FTDCDecodeError(f"{source}: invalid BSON length {length}")
+        remainder = stream.read(length - 4)
+        if len(remainder) != length - 4:
+            raise FTDCDecodeError(f"{source}: truncated BSON document")
+        try:
+            yield BSON(prefix + remainder).decode(codec_options=_CODEC_OPTIONS)
+        except Exception as exc:
+            raise FTDCDecodeError(f"{source}: invalid BSON document") from exc
+
+
+def decode_metric_document(document: Mapping[str, Any]) -> DecodedChunk:
+    """Decode one outer FTDC document whose type is 1."""
+
+    raw = _decompress_payload(document)
+    if len(raw) < _MIN_BSON_SIZE:
+        raise FTDCDecodeError("metric chunk has no reference document")
+
+    (reference_size,) = struct.unpack_from("<I", raw)
+    if reference_size < _MIN_BSON_SIZE or reference_size + 8 > len(raw):
+        raise FTDCDecodeError("invalid reference document size")
+    try:
+        reference = BSON(raw[:reference_size]).decode(codec_options=_CODEC_OPTIONS)
+    except Exception as exc:
+        raise FTDCDecodeError("invalid metric reference document") from exc
+
+    metric_count, delta_count = struct.unpack_from("<II", raw, reference_size)
+    slots = tuple(_extract_slots(reference))
+    if len(slots) != metric_count:
+        raise FTDCDecodeError(
+            f"reference contains {len(slots)} numeric slots, chunk declares {metric_count}"
+        )
+
+    encoded = memoryview(raw)[reference_size + 8 :]
+    flat_deltas = _decode_deltas(encoded, metric_count * delta_count)
+    current = [slot.initial for slot in slots]
+    rows: list[tuple[int, ...]] = [tuple(current)]
+    for sample_index in range(delta_count):
+        for metric_index in range(metric_count):
+            offset = metric_index * delta_count + sample_index
+            current[metric_index] = (current[metric_index] + flat_deltas[offset]) & _UINT64_MASK
+        rows.append(tuple(current))
+
+    return DecodedChunk(reference, slots, tuple(rows))
+
+
+def _decompress_payload(document: Mapping[str, Any]) -> bytes:
+    """Validate and decompress the binary payload of a metric document."""
+
+    payload = document.get("data", document.get("doc"))
+    if not isinstance(payload, (bytes, bytearray, memoryview)):
+        raise FTDCDecodeError("metric document has no binary 'data' or 'doc' field")
+    data = payload.tobytes() if isinstance(payload, memoryview) else bytes(payload)
+    if len(data) < 5:
+        raise FTDCDecodeError("compressed metric chunk is too short")
+
+    (expected_size,) = struct.unpack_from("<I", data)
+    try:
+        raw = zlib.decompress(data[4:])
+    except zlib.error as exc:
+        raise FTDCDecodeError("invalid zlib metric payload") from exc
+    if len(raw) != expected_size:
+        raise FTDCDecodeError(
+            f"metric chunk size mismatch: expected {expected_size}, got {len(raw)}"
+        )
+    return raw
+
+
+def value_for_slot(slot: MetricSlot, raw_value: int) -> MetricValue:
+    """Restore a compressed integer to the reference field's useful Python type."""
+
+    signed = _as_signed(raw_value)
+    if slot.kind == "bool":
+        return bool(raw_value)
+    if slot.kind == "float":
+        return float(signed)
+    return signed
+
+
+def timestamp_for_row(chunk: DecodedChunk, row: Sequence[int]) -> datetime:
+    """Return the top-level collection start time for a decoded sample."""
+
+    for index, slot in enumerate(chunk.slots):
+        if slot.path == "start" and slot.kind == "datetime":
+            return datetime.fromtimestamp(_as_signed(row[index]) / 1000, tz=timezone.utc)
+    raise FTDCDecodeError("metric reference document has no top-level datetime 'start'")
+
+
+def _extract_slots(value: object, path: str = "") -> Iterator[MetricSlot]:
+    if isinstance(value, Mapping):
+        mapping = cast(Mapping[object, object], value)
+        for name, child in mapping.items():
+            child_path = f"{path}.{name}" if path else str(name)
+            yield from _extract_slots(child, child_path)
+        return
+    if isinstance(value, (list, tuple)):
+        sequence = cast(Sequence[object], value)
+        for index, child in enumerate(sequence):
+            child_path = f"{path}.{index}" if path else str(index)
+            yield from _extract_slots(child, child_path)
+        return
+
+    if isinstance(value, bool):
+        yield MetricSlot(path, int(value), "bool")
+    elif isinstance(value, int):
+        yield MetricSlot(path, value & _UINT64_MASK, "int")
+    elif isinstance(value, float):
+        number = (
+            0
+            if math.isnan(value)
+            else max(-(1 << 63), min((1 << 63) - 1, int(value)))
+        )
+        yield MetricSlot(path, number & _UINT64_MASK, "float")
+    elif isinstance(value, datetime):
+        moment = value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
+        yield MetricSlot(path, int(moment.timestamp() * 1000) & _UINT64_MASK, "datetime")
+    elif isinstance(value, Timestamp):
+        yield MetricSlot(path, value.time & _UINT64_MASK, "timestamp", 0)
+        yield MetricSlot(path, value.inc & _UINT64_MASK, "timestamp", 1)
+    elif isinstance(value, Decimal128):
+        low, high = struct.unpack("<QQ", value.bid)
+        yield MetricSlot(path, low, "decimal128", 0)
+        yield MetricSlot(path, high, "decimal128", 1)
+
+
+def _decode_deltas(data: memoryview, expected_count: int) -> list[int]:
+    values: list[int] = []
+    position = 0
+    while len(values) < expected_count:
+        value, position = _read_varint(data, position)
+        if value:
+            values.append(value)
+            continue
+        run_minus_one, position = _read_varint(data, position)
+        run_length = run_minus_one + 1
+        if len(values) + run_length > expected_count:
+            raise FTDCDecodeError("zero run exceeds declared metric data")
+        values.extend([0] * run_length)
+    if position != len(data):
+        raise FTDCDecodeError("unexpected bytes after compressed metric data")
+    return values
+
+
+def _read_varint(data: memoryview, position: int) -> tuple[int, int]:
+    value = 0
+    for shift in range(0, 70, 7):
+        if position >= len(data):
+            raise FTDCDecodeError("truncated varint metric data")
+        byte = data[position]
+        position += 1
+        value |= (byte & 0x7F) << shift
+        if not byte & 0x80:
+            if value > _UINT64_MASK:
+                raise FTDCDecodeError("varint exceeds uint64")
+            return value, position
+    raise FTDCDecodeError("varint exceeds uint64")
+
+
+def _as_signed(value: int) -> int:
+    return value if value < (1 << 63) else value - (1 << 64)

pyftdc/exceptions.py

@@ -0,0 +1,13 @@
+"""Exceptions raised by pyftdc."""
+
+
+class FTDCError(Exception):
+    """Base class for pyftdc errors."""
+
+
+class FTDCDecodeError(FTDCError):
+    """An FTDC file or compressed metric chunk is invalid."""
+
+
+class MetricNotFoundError(FTDCError, KeyError):
+    """The requested metric does not occur in the source."""

pyftdc/models.py

@@ -0,0 +1,15 @@
+"""Public value objects."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import TypeAlias
+
+MetricValue: TypeAlias = int | float | bool
+
+
+@dataclass(frozen=True, slots=True)
+class DataPoint:
+    """One metric observation."""
+
+    timestamp: datetime
+    value: MetricValue

pyftdc/reader.py

@@ -0,0 +1,163 @@
+"""High-level FTDC metric query API."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterator
+from datetime import datetime, timezone
+from pathlib import Path
+
+from pyftdc._codec import (
+    DecodedChunk,
+    decode_metric_document,
+    iter_bson_documents,
+    timestamp_for_row,
+    value_for_slot,
+)
+from pyftdc.exceptions import FTDCError, MetricNotFoundError
+from pyftdc.models import DataPoint
+
+
+class FTDCReader:
+    """Read metrics from one FTDC file or a ``diagnostic.data`` directory."""
+
+    def __init__(self, source: str | Path) -> None:
+        self.source = Path(source)
+        if not self.source.exists():
+            raise FileNotFoundError(self.source)
+        if not self.source.is_file() and not self.source.is_dir():
+            raise FTDCError(f"FTDC source is not a regular file or directory: {self.source}")
+
+    def get_metric(
+        self,
+        name: set[str],
+        start: datetime | None = None,
+        end: datetime | None = None,
+        sample_rate: float = 1.0,
+    ) -> dict[str, list[DataPoint]]:
+        """Return sampled observations by metric name in the inclusive UTC timespan."""
+
+        requested_names = set(name)
+        if "" in requested_names:
+            raise ValueError("metric names must not be empty")
+        start_utc = _as_utc(start, "start") if start is not None else None
+        end_utc = _as_utc(end, "end") if end is not None else None
+        if start_utc is not None and end_utc is not None and start_utc > end_utc:
+            raise ValueError("start must be before or equal to end")
+        if not math.isfinite(sample_rate) or not 0 < sample_rate <= 1:
+            raise ValueError("sample_rate must be greater than 0 and at most 1")
+
+        found_names: set[str] = set()
+        point_numbers: dict[str, int] = {}
+        points_by_name: dict[str, dict[datetime, DataPoint]] = {}
+        for chunk in self._metric_chunks(start_utc, end_utc):
+            matching_slots = {
+                slot.path: (index, slot)
+                for index, slot in enumerate(chunk.slots)
+                if slot.part == 0 and (not requested_names or slot.path in requested_names)
+            }
+            if not matching_slots:
+                continue
+            found_names.update(matching_slots)
+            for metric_name in matching_slots:
+                point_numbers.setdefault(metric_name, 0)
+                points_by_name.setdefault(metric_name, {})
+            for row in chunk.rows:
+                timestamp = timestamp_for_row(chunk, row)
+                if (start_utc is None or start_utc <= timestamp) and (
+                    end_utc is None or timestamp <= end_utc
+                ):
+                    for metric_name, (metric_index, slot) in matching_slots.items():
+                        point_number = point_numbers[metric_name] + 1
+                        point_numbers[metric_name] = point_number
+                        if int(point_number * sample_rate) == int((point_number - 1) * sample_rate):
+                            continue
+                        points_by_name[metric_name][timestamp] = DataPoint(
+                            timestamp=timestamp,
+                            value=value_for_slot(slot, row[metric_index]),
+                        )
+
+        missing_names = requested_names - found_names
+        if missing_names:
+            raise MetricNotFoundError(sorted(missing_names)[0])
+        return {
+            metric_name: [points[timestamp] for timestamp in sorted(points)]
+            for metric_name, points in sorted(points_by_name.items())
+        }
+
+    query = get_metric
+
+    def list_metrics(self) -> list[str]:
+        """Return sorted dotted names for numeric fields in the source."""
+
+        names: set[str] = set()
+        for chunk in self._metric_chunks():
+            names.update(slot.path for slot in chunk.slots)
+        return sorted(names)
+
+    def _metric_chunks(
+        self,
+        start: datetime | None = None,
+        end: datetime | None = None,
+    ) -> Iterator[DecodedChunk]:
+        for path in self._paths(start, end):
+            with path.open("rb") as stream:
+                for document in iter_bson_documents(stream, path):
+                    if document.get("type") == 1:
+                        yield decode_metric_document(document)
+
+    def _paths(
+        self,
+        start: datetime | None = None,
+        end: datetime | None = None,
+    ) -> list[Path]:
+        if self.source.is_file():
+            return [self.source]
+        paths = sorted(
+            path
+            for path in self.source.glob("metrics.*")
+            if path.is_file() and not path.name.endswith(".tmp")
+        )
+        times = {path: _time_from_filename(path) for path in paths}
+        timestamped = [file_time for file_time in times.values() if file_time is not None]
+        if not timestamped:
+            return paths
+
+        first_time = min(timestamped)
+        lower_file_time: datetime | None = None
+        if start is not None:
+            preceding = [file_time for file_time in timestamped if file_time <= start]
+            lower_file_time = max(preceding, default=first_time)
+
+        upper_file_time = end
+        if end is not None and end < first_time:
+            upper_file_time = first_time
+
+        return [
+            path
+            for path in paths
+            if (file_time := times[path]) is None
+            or (
+                (lower_file_time is None or lower_file_time <= file_time)
+                and (upper_file_time is None or file_time <= upper_file_time)
+            )
+        ]
+
+
+def _as_utc(value: datetime, label: str) -> datetime:
+    if value.tzinfo is None or value.utcoffset() is None:
+        raise ValueError(f"{label} must be timezone-aware")
+    return value.astimezone(timezone.utc)
+
+
+def _time_from_filename(path: Path) -> datetime | None:
+    name = path.name.removeprefix("metrics.")
+    timestamp, separator, sequence = name.rpartition("-")
+    if not separator or not sequence.isdigit():
+        return None
+    try:
+        return datetime.strptime(timestamp, "%Y-%m-%dT%H-%M-%SZ").replace(
+            tzinfo=timezone.utc
+        )
+    except ValueError:
+        return None

pymongoftdc-0.1.0.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: pymongoftdc
+Version: 0.1.0
+Summary: A typed reader for MongoDB FTDC metric archives
+License: MIT License
+        
+        Copyright (c) 2026 Yaoxing
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: pymongo<5,>=4.6
+Provides-Extra: test
+Requires-Dist: pylint>=3; extra == 'test'
+Requires-Dist: pyright>=1.1.400; extra == 'test'
+Requires-Dist: pytest-cov>=5; extra == 'test'
+Requires-Dist: pytest>=8; extra == 'test'
+Requires-Dist: ruff>=0.11; extra == 'test'
+Description-Content-Type: text/markdown
+
+# pymongoftdc
+
+[![CI](https://github.com/zhangyaoxing/pyftdc/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/zhangyaoxing/pyftdc/actions/workflows/ci.yml)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+`pymongoftdc` reads numeric time-series metrics directly from MongoDB Full-Time
+Diagnostic Data Capture (FTDC) archive files.
+
+## Install
+
+```bash
+python -m pip install -e .
+```
+
+For development:
+
+```bash
+python -m pip install -e '.[test]'
+pytest
+```
+
+## Use
+
+```python
+from datetime import datetime, timezone
+from pyftdc import FTDCReader
+
+reader = FTDCReader("/var/lib/mongo/diagnostic.data")
+metrics = reader.get_metric(
+    {"serverStatus.connections.current"},
+    start=datetime(2026, 1, 1, tzinfo=timezone.utc),
+    end=datetime(2026, 1, 1, 1, tzinfo=timezone.utc),
+    sample_rate=0.1,
+)
+points = metrics["serverStatus.connections.current"]
+```
+
+The source may be one `metrics.*` file or a `diagnostic.data` directory.
+Timespan endpoints are inclusive and must be timezone-aware. Omit `start` or
+`end` to use the earliest or latest timestamp in the source. The result maps each
+requested name to points ordered by UTC timestamp. Pass an empty set to read every
+metric. `sample_rate` must be greater than 0 and at most 1;
+for example, `0.1` returns approximately 10% of points. Its default is `1.0`.
+`query()` is an alias for `get_metric()`.
+
+Use `reader.list_metrics()` to discover dotted metric paths. A missing requested
+metric raises `MetricNotFoundError`; an invalid archive raises `FTDCDecodeError`.
+
+## Project layout
+
+```text
+src/pyftdc/
+  _codec.py       BSON framing and FTDC decompression
+  reader.py       public query API
+  models.py       returned value objects
+  exceptions.py   library-specific errors
+tests/             pytest tests and fixture builders
+```
+
+The reader supports BSON-framed type-1 metric chunks using MongoDB's
+delta/RLE/varint/zlib encoding. Metadata documents are safely skipped.

pymongoftdc-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+pyftdc/__init__.py,sha256=HslKWPYP4hlF9n9dtu0gbrdY8drEJdT_qkIvxzjNKr8,328
+pyftdc/_codec.py,sha256=-Az4Eus9LK9NhTmBB2LByFAZDQDIe4_gNIJfGLVm4X4,8163
+pyftdc/exceptions.py,sha256=NuUABt86Y1td4nkoUv3wr0AUCzfi2v0T3zxzsct_Vlo,314
+pyftdc/models.py,sha256=scUG4n9Sb6lvscbmgWnOB6lRqUfx4aKEAKtUOHD5Hhc,304
+pyftdc/reader.py,sha256=Eqad2u9d-utlUUfdeU36Cig3wJHo6casxlhatOHPtQw,6216
+pymongoftdc-0.1.0.dist-info/METADATA,sha256=Z7O2HsCffJ_COt_M3fdcy5UP6QAKrVxggV1Oe9jWj1A,3791
+pymongoftdc-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+pymongoftdc-0.1.0.dist-info/licenses/LICENSE,sha256=Tv9yMSkndKN-OnOBs1yvmrW4fF4M7ZyvyulsTV1D2As,1064
+pymongoftdc-0.1.0.dist-info/RECORD,,

pymongoftdc-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

pymongoftdc-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yaoxing
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.