kinro-sdk 0.1.1__py3-none-any.whl

kinro/contract/__init__.py

@@ -0,0 +1,41 @@
+from kinro.contract.emit import HEARTBEAT_INTERVAL, ContractFormatter, GelfContractFilter, Heartbeat, start_heartbeat
+from kinro.contract.model import (
+    HEARTBEAT_EVENT,
+    MAX_LINE_BYTES,
+    MAX_PAYLOAD_BYTES,
+    SCHEMA_VERSION,
+    ContractError,
+    DbObsId,
+    FileObsId,
+    GraylogObsId,
+    JsonValue,
+    LogContractV1,
+    LokiObsId,
+    Observation,
+    ObsId,
+    TsSource,
+    obs_id_from_key,
+)
+
+__all__ = [
+    'HEARTBEAT_EVENT',
+    'HEARTBEAT_INTERVAL',
+    'MAX_LINE_BYTES',
+    'MAX_PAYLOAD_BYTES',
+    'SCHEMA_VERSION',
+    'ContractError',
+    'ContractFormatter',
+    'DbObsId',
+    'FileObsId',
+    'GelfContractFilter',
+    'GraylogObsId',
+    'Heartbeat',
+    'JsonValue',
+    'LogContractV1',
+    'LokiObsId',
+    'ObsId',
+    'Observation',
+    'TsSource',
+    'obs_id_from_key',
+    'start_heartbeat',
+]

kinro/contract/emit.py

@@ -0,0 +1,151 @@
+import json
+import logging
+import os
+import socket
+import threading
+from collections.abc import Mapping
+from dataclasses import replace
+from datetime import UTC, datetime, timedelta
+
+from kinro.contract.model import HEARTBEAT_EVENT, JsonValue, LogContractV1
+
+HEARTBEAT_INTERVAL = timedelta(seconds=2)
+
+
+class ContractFormatter(logging.Formatter):
+    """Records with `extra={'event': ...}` become LogContract v1 lines; others keep the template JsonFormatter shape.
+
+    Give every handler its own instance: each one numbers its lines, so a shared instance leaves `seq` gaps.
+    """
+
+    def __init__(self, service: str) -> None:
+        super().__init__()
+        self.service = service
+        self._sequence = _Sequence()
+
+    def format(self, record: logging.LogRecord) -> str:
+        event = getattr(record, 'event', None)
+        if not isinstance(event, str) or not event:
+            return self._plain(record)
+        contract = LogContractV1(
+            ts=datetime.fromtimestamp(record.created, tz=UTC),
+            svc=self.service,
+            event=event,
+            level=record.levelname,
+            cid=_as_text(getattr(record, 'cid', None)),
+            keys=_as_keys(getattr(record, 'keys', None)),
+            data=_as_data(getattr(record, 'data', None)),
+            msg=record.getMessage(),
+            exc=self.formatException(record.exc_info) if record.exc_info else None,
+        )
+        inst, seq = self._sequence.next()
+        contract = replace(contract, inst=inst, seq=seq)
+        # Once seq is taken the line must be emitted, otherwise Kinro sees a gap that never happened.
+        try:
+            return json.dumps(contract.to_dict(), ensure_ascii=False, default=str)
+        except (TypeError, ValueError):
+            return json.dumps(replace(contract, data={}).to_dict(), ensure_ascii=False, default=str)
+
+    def _plain(self, record: logging.LogRecord) -> str:
+        payload = {
+            'time': datetime.fromtimestamp(record.created, tz=UTC).isoformat(),
+            'level': record.levelname,
+            'logger': record.name,
+            'message': record.getMessage(),
+        }
+        if record.exc_info:
+            payload['exc_info'] = self.formatException(record.exc_info)
+        return json.dumps(payload, ensure_ascii=False)
+
+
+class GelfContractFilter(logging.Filter):
+    """Stamps `inst` and `seq` on records with `extra={'event': ...}` for a GELF handler such as graypy's,
+    which sends record attributes as additional fields and never calls a formatter for them.
+
+    Add one instance to each GELF handler (`handler.addFilter(GelfContractFilter())`); like ContractFormatter,
+    a shared instance leaves `seq` gaps in each output.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._sequence = _Sequence()
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        event = getattr(record, 'event', None)
+        if isinstance(event, str) and event:
+            record.inst, record.seq = self._sequence.next()
+        return True
+
+
+class _Sequence:
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._pid = -1
+        self._inst = ''
+        self._next_seq = 0
+
+    def next(self) -> tuple[str, int]:
+        with self._lock:
+            pid = os.getpid()
+            if pid != self._pid:
+                # A forked child inherits the parent's counter; it starts its own sequence under its own inst.
+                self._pid, self._inst, self._next_seq = pid, f'{socket.gethostname()}:{pid}', 0
+            seq = self._next_seq
+            self._next_seq += 1
+            return self._inst, seq
+
+
+class Heartbeat:
+    def __init__(self, logger: logging.Logger, every: timedelta) -> None:
+        self._logger = logger
+        self._every = every.total_seconds()
+        self._stopped = threading.Event()
+        self._thread = threading.Thread(target=self._run, name='kinro-heartbeat', daemon=True)
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stopped.set()
+        self._thread.join()
+
+    def _run(self) -> None:
+        while True:
+            self._beat()
+            if self._stopped.wait(self._every):
+                return
+
+    def _beat(self) -> None:
+        # Bypasses the logger level, so LOG_LEVEL=WARNING cannot silence the producer watermark.
+        record = self._logger.makeRecord(
+            self._logger.name, logging.INFO, __file__, 0, 'kinro heartbeat', (), None,
+            extra={'event': HEARTBEAT_EVENT},
+        )
+        self._logger.handle(record)
+
+
+def start_heartbeat(logger: logging.Logger, every: timedelta = HEARTBEAT_INTERVAL) -> Heartbeat:
+    """Emit a `kinro.heartbeat` record through `logger` now and then every `every` from a daemon thread.
+
+    Args:
+        logger: Must reach the handler carrying ContractFormatter, e.g. the root logger.
+    """
+    heartbeat = Heartbeat(logger, every)
+    heartbeat.start()
+    return heartbeat
+
+
+def _as_text(value: object) -> str | None:
+    return None if value is None else str(value)
+
+
+def _as_keys(value: object) -> dict[str, str]:
+    if not isinstance(value, Mapping):
+        return {}
+    return {str(name): str(key) for name, key in value.items() if key is not None}
+
+
+def _as_data(value: object) -> dict[str, JsonValue]:
+    if not isinstance(value, Mapping):
+        return {}
+    return {str(name): item for name, item in value.items()}

kinro/contract/model.py

@@ -0,0 +1,324 @@
+import json
+from collections.abc import Collection, Mapping
+from dataclasses import dataclass, field, replace
+from datetime import UTC, datetime
+from typing import Any, ClassVar, Literal, Self
+
+SCHEMA_VERSION = 1
+HEARTBEAT_EVENT = 'kinro.heartbeat'
+MAX_LINE_BYTES = 1024 * 1024
+MAX_PAYLOAD_BYTES = 2 * 1024
+REDACTED = '[redacted]'
+
+type JsonValue = str | int | float | bool | None | list[JsonValue] | dict[str, JsonValue]
+type TsSource = Literal['body', 'push', 'fetched']
+
+
+class ContractError(ValueError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class FileObsId:
+    KIND: ClassVar[str] = 'file'
+
+    source_id: str
+    file_uid: int
+    generation: int
+    byte_offset: int
+
+    @property
+    def key(self) -> str:
+        return _dump_key([self.KIND, self.source_id, self.file_uid, self.generation, self.byte_offset])
+
+
+@dataclass(frozen=True, slots=True)
+class LokiObsId:
+    KIND: ClassVar[str] = 'loki'
+
+    source_id: str
+    stream_labels_hash: str
+    ts_ns: int
+    line_hash: str
+    dup_index: int
+
+    @property
+    def key(self) -> str:
+        return _dump_key([self.KIND, self.source_id, self.stream_labels_hash, self.ts_ns, self.line_hash,
+                          self.dup_index])
+
+
+@dataclass(frozen=True, slots=True)
+class DbObsId:
+    KIND: ClassVar[str] = 'db'
+
+    source_id: str
+    query_name: str
+    poll_seq: int
+    row_index: int
+
+    @property
+    def key(self) -> str:
+        return _dump_key([self.KIND, self.source_id, self.query_name, self.poll_seq, self.row_index])
+
+
+@dataclass(frozen=True, slots=True)
+class GraylogObsId:
+    """`message_id` is Graylog's `gl2_message_id` (`_id` when absent); it stays the same across index rotation."""
+
+    KIND: ClassVar[str] = 'graylog'
+
+    source_id: str
+    ts_ms: int
+    message_id: str
+
+    @property
+    def key(self) -> str:
+        return _dump_key([self.KIND, self.source_id, self.ts_ms, self.message_id])
+
+
+type ObsId = FileObsId | LokiObsId | DbObsId | GraylogObsId
+
+_OBS_ID_TYPES: dict[str, type[FileObsId] | type[LokiObsId] | type[DbObsId] | type[GraylogObsId]] = {
+    kind.KIND: kind for kind in (FileObsId, LokiObsId, DbObsId, GraylogObsId)
+}
+
+
+def obs_id_from_key(key: str) -> ObsId:
+    kind, *parts = json.loads(key)
+    return _OBS_ID_TYPES[kind](*parts)
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class LogContractV1:
+    ts: datetime
+    svc: str
+    event: str
+    level: str | None = None
+    cid: str | None = None
+    keys: Mapping[str, str] = field(default_factory=dict)
+    data: Mapping[str, JsonValue] = field(default_factory=dict)
+    msg: str | None = None
+    exc: str | None = None
+    inst: str | None = None
+    seq: int | None = None
+
+    @property
+    def is_heartbeat(self) -> bool:
+        return self.event == HEARTBEAT_EVENT
+
+    def to_dict(self) -> dict[str, JsonValue]:
+        line: dict[str, JsonValue] = {
+            'kinro_v': SCHEMA_VERSION, 'ts': self.ts.isoformat(), 'svc': self.svc, 'event': self.event,
+        }
+        optional: dict[str, JsonValue] = {
+            'level': self.level,
+            'cid': self.cid,
+            'keys': dict(self.keys) or None,
+            'data': dict(self.data) or None,
+            'msg': self.msg,
+            'exc': self.exc,
+            'inst': self.inst,
+            'seq': self.seq,
+        }
+        line.update((name, value) for name, value in optional.items() if value is not None)
+        return line
+
+    @classmethod
+    def from_dict(cls, raw: Mapping[str, object]) -> Self:
+        version = raw.get('kinro_v')
+        if type(version) is not int or version != SCHEMA_VERSION:
+            raise ContractError(f'unsupported kinro_v {version!r}')
+        seq = raw.get('seq')
+        if seq is not None and (type(seq) is not int or seq < 0):
+            raise ContractError(f'seq must be a non-negative int, got {seq!r}')
+        return cls(
+            ts=parse_aware_ts(raw.get('ts')),
+            svc=_required_text(raw, 'svc'),
+            event=_required_text(raw, 'event'),
+            level=_optional_text(raw, 'level'),
+            cid=_optional_text(raw, 'cid'),
+            keys=_keys(raw.get('keys')),
+            data=_data(raw.get('data')),
+            msg=_optional_text(raw, 'msg', allow_empty=True),
+            exc=_optional_text(raw, 'exc', allow_empty=True),
+            inst=_optional_text(raw, 'inst'),
+            seq=seq,
+        )
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class Observation:
+    obs_id: ObsId
+    service: str
+    event: str
+    ts: datetime
+    ts_source: TsSource = 'body'
+    cid: str | None = None
+    keys: Mapping[str, str] = field(default_factory=dict)
+    payload: Mapping[str, JsonValue] = field(default_factory=dict)
+    level: str | None = None
+    inst: str | None = None
+    seq: int | None = None
+    truncated: bool = False
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        obs_id: ObsId,
+        service: str,
+        event: str,
+        ts: datetime,
+        ts_source: TsSource = 'body',
+        cid: str | None = None,
+        keys: Mapping[str, str] | None = None,
+        payload: Mapping[str, JsonValue] | None = None,
+        level: str | None = None,
+        inst: str | None = None,
+        seq: int | None = None,
+        payload_cap: int = MAX_PAYLOAD_BYTES,
+    ) -> Self:
+        """Build an observation with ts in UTC and payload fields dropped largest-first to fit `payload_cap`.
+
+        Args:
+            payload_cap: Byte budget for the serialized observation; identity fields are never dropped to meet it.
+        """
+        if ts.tzinfo is None:
+            raise ValueError('observation ts must carry a timezone')
+        ts = ts.astimezone(UTC)
+        keys = dict(keys or {})
+        fixed: dict[str, JsonValue] = {
+            'svc': service, 'event': event, 'ts': ts.isoformat(), 'cid': cid,
+            'keys': dict(keys),
+        }
+        capped, truncated = cap_payload(fixed, payload or {}, payload_cap)
+        return cls(
+            obs_id=obs_id, service=service, event=event, ts=ts, ts_source=ts_source, cid=cid, keys=keys,
+            payload=capped, level=level, inst=inst, seq=seq, truncated=truncated,
+        )
+
+    @property
+    def is_heartbeat(self) -> bool:
+        return self.event == HEARTBEAT_EVENT
+
+    def redact(self, fields: Collection[str]) -> Self:
+        """A copy whose top-level payload `fields` read `REDACTED`; nested values under such a name go too."""
+        if not any(name in self.payload for name in fields):
+            return self
+        payload = {name: REDACTED if name in fields else value for name, value in self.payload.items()}
+        return replace(self, payload=payload)
+
+    def lookup(self, name: str) -> JsonValue:
+        if name == 'cid':
+            return self.cid
+        if name in self.keys:
+            return self.keys[name]
+        return self.payload.get(name)
+
+    def to_dict(self) -> dict[str, JsonValue]:
+        return {
+            'obs_id': self.obs_id.key,
+            'service': self.service,
+            'event': self.event,
+            'ts': self.ts.isoformat(),
+            'ts_source': self.ts_source,
+            'cid': self.cid,
+            'keys': dict(self.keys),
+            'payload': dict(self.payload),
+            'level': self.level,
+            'inst': self.inst,
+            'seq': self.seq,
+            'truncated': self.truncated,
+        }
+
+    @classmethod
+    def from_dict(cls, raw: Mapping[str, Any]) -> Self:
+        return cls(
+            obs_id=obs_id_from_key(raw['obs_id']),
+            service=raw['service'],
+            event=raw['event'],
+            ts=datetime.fromisoformat(raw['ts']),
+            ts_source=raw['ts_source'],
+            cid=raw['cid'],
+            keys=raw['keys'],
+            payload=raw['payload'],
+            level=raw['level'],
+            inst=raw['inst'],
+            seq=raw['seq'],
+            truncated=raw['truncated'],
+        )
+
+
+def parse_aware_ts(value: object) -> datetime:
+    if not isinstance(value, str):
+        raise ContractError(f'ts must be an RFC 3339 string, got {value!r}')
+    try:
+        ts = datetime.fromisoformat(value)
+    except ValueError as exc:
+        raise ContractError(f'invalid ts {value!r}') from exc
+    if ts.tzinfo is None:
+        raise ContractError(f'ts {value!r} lacks a UTC offset')
+    return ts
+
+
+def cap_payload(
+    fixed: Mapping[str, JsonValue], payload: Mapping[str, JsonValue], limit: int,
+) -> tuple[dict[str, JsonValue], bool]:
+    kept = dict(payload)
+    total = _json_size({**fixed, 'data': kept})
+    if total <= limit:
+        return kept, False
+    # Size of `"name":value,` inside the data object.
+    sizes = sorted(((_json_size({name: value}) - 1, name) for name, value in kept.items()), reverse=True)
+    for size, name in sizes:
+        if total <= limit:
+            break
+        del kept[name]
+        total -= size
+    return kept, len(kept) < len(payload)
+
+
+def _json_size(value: JsonValue) -> int:
+    return len(json.dumps(value, ensure_ascii=False, separators=(',', ':'), default=str).encode())
+
+
+def _dump_key(parts: list[JsonValue]) -> str:
+    return json.dumps(parts, ensure_ascii=False, separators=(',', ':'))
+
+
+def _required_text(raw: Mapping[str, object], name: str) -> str:
+    value = raw.get(name)
+    if not isinstance(value, str) or not value:
+        raise ContractError(f'{name} must be a non-empty string, got {value!r}')
+    return value
+
+
+def _optional_text(raw: Mapping[str, object], name: str, *, allow_empty: bool = False) -> str | None:
+    value = raw.get(name)
+    if value is None:
+        return None
+    if not isinstance(value, str) or not (value or allow_empty):
+        raise ContractError(f'{name} must be a string, got {value!r}')
+    return value
+
+
+def _keys(value: object) -> dict[str, str]:
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ContractError(f'keys must be an object, got {value!r}')
+    keys: dict[str, str] = {}
+    for name, key in value.items():
+        if isinstance(key, bool) or not isinstance(key, str | int):
+            raise ContractError(f'key {name!r} must be a string or int, got {key!r}')
+        keys[str(name)] = str(key)
+    return keys
+
+
+def _data(value: object) -> dict[str, JsonValue]:
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ContractError(f'data must be an object, got {value!r}')
+    return {str(name): item for name, item in value.items()}

kinro/contract/schema.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Kinro LogContract v1",
+  "description": "One JSON object per log line. Unknown fields are ignored by Kinro.",
+  "type": "object",
+  "required": ["kinro_v", "ts", "svc", "event"],
+  "properties": {
+    "kinro_v": {"const": 1},
+    "ts": {"type": "string", "format": "date-time", "description": "RFC 3339 with an offset; event time"},
+    "svc": {"type": "string", "minLength": 1},
+    "event": {"type": "string", "minLength": 1, "description": "Dotted name, e.g. order.created; kinro.heartbeat is reserved"},
+    "level": {"type": "string", "minLength": 1},
+    "cid": {"type": "string", "minLength": 1, "description": "Propagated correlation id"},
+    "keys": {
+      "type": "object",
+      "additionalProperties": {"type": ["string", "integer"]},
+      "description": "Business keys; integers are read as strings"
+    },
+    "data": {"type": "object", "description": "Payload; fields are dropped largest-first over the 2 KiB cap"},
+    "msg": {"type": "string", "description": "Human-readable message; not used by rules"},
+    "exc": {"type": "string", "description": "Formatted exception; not used by rules"},
+    "inst": {"type": "string", "description": "host:pid of the producing process"},
+    "seq": {"type": "integer", "minimum": 0, "description": "Per-inst counter, gap-free from 0"}
+  }
+}

kinro/parsers/__init__.py

@@ -0,0 +1,17 @@
+from kinro.parsers.base import LineMeta, ParseError, Parser, ParseStats
+from kinro.parsers.contract import StandardParser
+from kinro.parsers.gelf import GelfParser
+from kinro.parsers.regex import RegexParser
+from kinro.parsers.template_json import MessageRule, TemplateJsonParser
+
+__all__ = [
+    'GelfParser',
+    'LineMeta',
+    'MessageRule',
+    'ParseError',
+    'ParseStats',
+    'Parser',
+    'RegexParser',
+    'StandardParser',
+    'TemplateJsonParser',
+]

kinro/parsers/base.py

@@ -0,0 +1,96 @@
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import datetime
+
+from kinro.contract.model import MAX_LINE_BYTES, JsonValue, Observation, ObsId
+
+logger = logging.getLogger(__name__)
+
+
+class ParseError(ValueError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class LineMeta:
+    """Where a line came from: the source assigns the id and service, the parser never derives them from content.
+
+    Args:
+        push_ts: Loki entry time; event time only for bodies that carry no ts of their own.
+    """
+
+    obs_id: ObsId
+    service: str
+    push_ts: datetime | None = None
+
+
+@dataclass(slots=True)
+class ParseStats:
+    lines: int = 0
+    parsed: int = 0
+    heartbeats: int = 0
+    truncated: int = 0
+    oversize: int = 0
+    no_contract: int = 0
+    unmatched: int = 0
+    malformed: int = 0
+
+    @property
+    def errors(self) -> int:
+        return self.malformed + self.oversize
+
+
+class Parser(ABC):
+    def __init__(self) -> None:
+        self.stats = ParseStats()
+
+    def parse(self, line: str, meta: LineMeta) -> Observation | None:
+        """Never raises: malformed, oversize and skipped lines only bump `stats`."""
+        self.stats.lines += 1
+        if _too_long(line):
+            self.stats.oversize += 1
+            return None
+        try:
+            obs = self._parse(line.rstrip('\r\n'), meta)
+        except Exception as exc:
+            self.stats.malformed += 1
+            logger.debug('Malformed %s line %s: %r', meta.service, meta.obs_id.key, exc)
+            return None
+        if obs is None:
+            return None
+        if obs.is_heartbeat:
+            self.stats.heartbeats += 1
+        else:
+            self.stats.parsed += 1
+        if obs.truncated:
+            self.stats.truncated += 1
+        return obs
+
+    @abstractmethod
+    def _parse(self, line: str, meta: LineMeta) -> Observation | None:
+        """Raise for a malformed line; count the reason in `stats` and return None for a deliberately skipped one."""
+
+
+def split_groups(
+    groups: Mapping[str, str | None], key_names: frozenset[str], reserved: frozenset[str],
+) -> tuple[str | None, dict[str, str], dict[str, JsonValue]]:
+    """Route named regex groups: `cid` to cid, `key_names` to keys, everything else not reserved to payload."""
+    keys: dict[str, str] = {}
+    payload: dict[str, JsonValue] = {}
+    for name, value in groups.items():
+        if value is None or name in reserved or name == 'cid':
+            continue
+        if name in key_names:
+            keys[name] = value
+        else:
+            payload[name] = value
+    return groups.get('cid'), keys, payload
+
+
+def _too_long(line: str) -> bool:
+    # A str of n chars encodes to at most 4n UTF-8 bytes, so short lines skip the encode.
+    return len(line) > MAX_LINE_BYTES or (
+        len(line) * 4 > MAX_LINE_BYTES and len(line.encode('utf-8', 'surrogatepass')) > MAX_LINE_BYTES
+    )

kinro/parsers/contract.py

@@ -0,0 +1,29 @@
+import json
+
+from kinro.contract.model import LogContractV1, Observation
+from kinro.parsers.base import LineMeta, ParseError, Parser
+
+
+class StandardParser(Parser):
+    """LogContract v1 lines. The service comes from the source, not from the body's `svc`."""
+
+    def _parse(self, line: str, meta: LineMeta) -> Observation | None:
+        raw = json.loads(line)
+        if not isinstance(raw, dict):
+            raise ParseError('line is not a JSON object')
+        if 'kinro_v' not in raw:
+            self.stats.no_contract += 1
+            return None
+        contract = LogContractV1.from_dict(raw)
+        return Observation.create(
+            obs_id=meta.obs_id,
+            service=meta.service,
+            event=contract.event,
+            ts=contract.ts,
+            cid=contract.cid,
+            keys=contract.keys,
+            payload=contract.data,
+            level=contract.level,
+            inst=contract.inst,
+            seq=contract.seq,
+        )