edutap.observability-settings 0.1.3__py3-none-any.whl

edutap/observability_settings/__init__.py

@@ -0,0 +1,31 @@
+"""How an eduTAP service reports what it is doing, and what it may say about people.
+
+Three backends with three jobs: Sentry takes errors, an OTLP collector takes traces
+and metrics, structlog produces the records that reach both. This package holds the
+settings for all three and the one call that applies them.
+
+It exists because the options are the point, not the code. Which Sentry options are
+on decides whether a bearer token or a person's identifier leaves the process, and
+each of them was chosen against a measurement rather than against a backend's
+recommendation. Separated from the ``sentry_sdk.init()`` call that applies them those
+measurements are worth nothing, which is why the settings live here and not in
+``edutap.data_models``.
+
+What must never appear here is a dependency on any eduTAP service. This is installed
+before a service resolves its own settings; it can know nothing about them.
+"""
+
+from .install import install_observability, logfire_options, sentry_options
+from .pseudonym import person_label, pseudonym
+from .settings import OTLP_ENDPOINT_VARIABLE, ObservabilitySettings, PersonUidMode
+
+__all__ = [
+    "OTLP_ENDPOINT_VARIABLE",
+    "ObservabilitySettings",
+    "PersonUidMode",
+    "install_observability",
+    "logfire_options",
+    "person_label",
+    "pseudonym",
+    "sentry_options",
+]

edutap/observability_settings/install.py

@@ -0,0 +1,153 @@
+"""Wiring the three backends the same way in every eduTAP service.
+
+Three systems with three jobs, and the split is deliberate rather than historical:
+**Sentry** takes errors, **the OTLP collector** takes traces and metrics, and
+**structlog** produces the records that go to both. Nothing travels two paths --
+Sentry's own tracing stays off, because the spans already go to the collector and
+Bugsink, the tracker this estate runs, states that it does not support traces.
+
+The options are returned by pure functions before they are applied. That is what lets
+a test assert the exact set rather than assert that *something* was configured, and it
+is the same shape ``edutap.data_provider`` uses, from whose observability design
+record the Sentry options and their measurements are taken.
+"""
+
+import logging
+import os
+from typing import Any
+
+import logfire
+import sentry_sdk
+import structlog
+
+from .settings import OTLP_ENDPOINT_VARIABLE, ObservabilitySettings
+
+
+def sentry_options(settings: ObservabilitySettings) -> dict[str, Any]:
+    """Return the options that decide what leaves the process.
+
+    Each contradicts the backend's own default, and each was chosen against a
+    measurement recorded in the data provider's observability design record.
+
+    ``include_local_variables=False`` -- with local variables on, a raw
+    ``Authorization`` header sits in the ASGI scope, which is a local in most frames of
+    an ASGI stack, and the bearer token then appears dozens of times in an event whose
+    rendered ``authorization`` header says ``[Filtered]``. Sentry's scrubber matches
+    key names; it does not walk a list of byte tuples.
+
+    ``max_request_body_size="never"`` -- for a service whose request body *is* the
+    identifying datum there is no partial version of this.
+
+    ``max_breadcrumbs=0`` -- Sentry's ``LoggingIntegration`` is on by default and turns
+    every WARNING/ERROR record into a breadcrumb carrying the record's formatted
+    message verbatim and unscrubbed, on a path none of the other options constrains.
+    A hook dropping only ``type == "log"`` breadcrumbs was the narrower alternative and
+    was rejected: it would need re-auditing against every breadcrumb-producing
+    integration added in the future, whereas turning them off closes the surface once.
+    The cost is real -- an event carries no timeline of what happened earlier -- and it
+    is the same trade already made for local variables and the request body.
+
+    ``Any`` rather than ``object`` in the return type, which is the one place this
+    package spends the escape hatch: the mapping is heterogeneous by nature and is
+    unpacked into a third-party signature, and ``object`` makes a type checker reject
+    every single key -- measured, 71 diagnostics for six options. Narrowing it to a
+    ``TypedDict`` would restate sentry-sdk's signature here and go stale with its next
+    release.
+    """
+    return {
+        "environment": settings.environment,
+        "traces_sample_rate": 0,
+        "send_default_pii": False,
+        "include_local_variables": False,
+        "max_request_body_size": "never",
+        "max_breadcrumbs": 0,
+    }
+
+
+def logfire_options(
+    settings: ObservabilitySettings,
+    *,
+    service_name: str,
+    service_version: str | None = None,
+) -> dict[str, Any]:
+    """Return how this service configures tracing, metrics and log export.
+
+    ``send_to_logfire=False`` is not a detail. The library defaults it to ``True``, so
+    leaving it unset would ship spans to a hosted third party the first time a token
+    happened to be present. This estate exports to its own collector.
+
+    The console stands in while there is no collector. Measured against logfire 4.40:
+    with ``send_to_logfire=False`` and no ``OTEL_EXPORTER_OTLP_ENDPOINT``, no exporter
+    is installed at all, so an instrumented service would be indistinguishable from an
+    uninstrumented one -- which is how instrumentation reaches production broken. Once
+    the endpoint is set the console stands down, because printing every span into the
+    container log is a development aid and not a production one.
+    """
+    exports_to_a_collector = bool(os.environ.get(OTLP_ENDPOINT_VARIABLE))
+    return {
+        "send_to_logfire": False,
+        "service_name": service_name,
+        "service_version": service_version,
+        "environment": settings.environment,
+        "console": False if exports_to_a_collector else logfire.ConsoleOptions(),
+    }
+
+
+def install_observability(
+    settings: ObservabilitySettings | None = None,
+    *,
+    service_name: str,
+    service_version: str | None = None,
+) -> None:
+    """Install error reporting, tracing and structured logging for this process.
+
+    Call it first, before the service resolves the settings it needs to run, so that a
+    process refusing to start is reported rather than silently absent.
+
+    Both backends are opt-in by configuration and not by code: no DSN means no error
+    tracker, ``telemetry_enabled=False`` means no tracing. Structured logging is set up
+    either way -- a service without a collector still has to be readable.
+    """
+    settings = settings or ObservabilitySettings()
+
+    if settings.telemetry_enabled:
+        logfire.configure(
+            **logfire_options(settings, service_name=service_name, service_version=service_version)
+        )
+
+    _configure_structlog(settings)
+
+    if settings.sentry_dsn is not None:
+        sentry_sdk.init(dsn=settings.sentry_dsn.get_secret_value(), **sentry_options(settings))
+
+
+def _configure_structlog(settings: ObservabilitySettings) -> None:
+    """Point structlog at the collector and render what is left as JSON.
+
+    ``logfire.StructlogProcessor`` is the bridge: it turns each event into a log
+    record on the OTel side, so a log line and the span it happened inside share a
+    trace id without the caller doing anything. It sits in the chain rather than at
+    the end, because the record still has to be rendered for the container log.
+
+    JSON rather than the developer console renderer: these lines are read out of
+    ``docker service logs`` and, later, out of the collector. A DLQ entry that cannot
+    be found by ``edutap-event-id`` is a DLQ entry nobody replays.
+    """
+    processors: list[structlog.typing.Processor] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.processors.add_log_level,
+        structlog.processors.TimeStamper(fmt="iso", utc=True),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+    ]
+    if settings.telemetry_enabled:
+        processors.append(logfire.StructlogProcessor())
+    processors.append(structlog.processors.JSONRenderer())
+
+    structlog.configure(
+        processors=processors,
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelNamesMapping()[settings.log_level]
+        ),
+        cache_logger_on_first_use=True,
+    )

edutap/observability_settings/pseudonym.py

@@ -0,0 +1,57 @@
+"""Standing in for a person, or refusing to.
+
+An error tracker is a machine that copies the state around a failure into a second
+system. A ``person_uid`` at a university is not an opaque handle -- at the LMU it is
+the LMU identifier with ``@lmu.de`` appended, and the circle able to turn that back
+into a human being is far wider than the circle with directory administration rights.
+Pseudonymised it is still personal data; the question a deployment answers here is not
+*whether* but *who may see it*.
+"""
+
+import hashlib
+import hmac
+
+from pydantic import SecretStr
+
+from .settings import ObservabilitySettings
+
+
+def pseudonym(person_uid: str, salt: SecretStr | None) -> str | None:
+    """Return a keyed, truncated stand-in for a person, or nothing without a key.
+
+    Keyed rather than a plain digest: a ``person_uid`` comes from a directory, so the
+    value space is small and enumerable, and an unsalted hash would be reversible by
+    anyone able to read the error tracker -- simply by hashing the directory.
+
+    Truncated to 12 hex characters, 48 bits: wide enough that two people in one
+    installation colliding is not a practical concern, short enough that the result
+    reads as a label rather than as an identifier worth storing.
+
+    An *empty* salt counts as no salt. ``compose.yml`` writes ``${VAR:-}``, which sets
+    a variable to the empty string rather than leaving it unset, and an HMAC under an
+    empty key is a well-defined, entirely unkeyed digest -- precisely the
+    hash-the-directory construction the key exists to prevent, and indistinguishable
+    from a real pseudonym by eye.
+    """
+    if salt is None or not salt.get_secret_value():
+        return None
+    digest = hmac.new(
+        salt.get_secret_value().encode("utf-8"),
+        person_uid.encode("utf-8"),
+        hashlib.sha256,
+    )
+    return digest.hexdigest()[:12]
+
+
+def person_label(person_uid: str, settings: ObservabilitySettings) -> str | None:
+    """Return what may stand for this person in what leaves the process.
+
+    ``None`` means: attach nothing. Callers must treat that as an answer rather than
+    as a reason to fall back to the raw value -- a deployment that asked for
+    pseudonyms and forgot the key has to lose the datum, not publish it.
+    """
+    if settings.person_uid_mode == "omit":
+        return None
+    if settings.person_uid_mode == "plain":
+        return person_uid
+    return pseudonym(person_uid, settings.pseudonym_salt)

edutap/observability_settings/settings.py

@@ -0,0 +1,72 @@
+"""Where a service reports to, and how much of a person may travel with it."""
+
+from typing import Literal
+
+from edutap.data_models.settings import ServiceSettings
+from pydantic import SecretStr
+from pydantic_settings import SettingsConfigDict
+
+#: What may stand in for a ``person_uid`` in an error report or a span.
+#:
+#: ``pseudonym`` keeps correlation without identification, ``plain`` is the honest
+#: choice where the tracker is read by exactly the people who may read the directory
+#: anyway, and ``omit`` drops the datum entirely.
+PersonUidMode = Literal["pseudonym", "plain", "omit"]
+
+
+class ObservabilitySettings(ServiceSettings):
+    """Everything the reporting side of a service needs, and nothing it can fail on.
+
+    Inherits :class:`~edutap.data_models.settings.ServiceSettings` rather than sitting
+    beside it: ``environment``, ``telemetry_enabled`` and ``log_level`` are read here,
+    and a service should not have to assemble two objects to start reporting.
+
+    **No field is required.** Observability is installed *before* a service resolves
+    the settings it needs to run, so that a process refusing to start is reported
+    rather than silently absent. That ordering only holds if reading this can never
+    fail for want of a value. A value that is present but not a legal one still
+    fails -- ignoring it would decide in silence what leaves the process.
+
+    The prefix is ``EDUTAP_`` and deliberately not per-service. These fields are
+    defined by an eduTAP package, another university deploying them should not have to
+    learn an LMU name, and the one field whose value genuinely differs per service --
+    the Sentry DSN, which identifies one project -- differs by being set in that
+    service's own environment block, not by carrying a different name.
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="EDUTAP_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    #: A DSN identifies one project, so every service gets its own rather than sharing
+    #: one. It is a write-only ingest key and thus low-sensitivity -- a plain
+    #: environment variable is enough and a Docker secret would be effort without
+    #: gain -- but it is still a credential, hence ``SecretStr``: ``BaseSettings``
+    #: prints every plain field verbatim, and a settings object reaches a log line
+    #: sooner or later. Unset means the integration stays off.
+    sentry_dsn: SecretStr | None = None
+
+    #: The HMAC key behind :func:`~edutap.observability_settings.pseudonym.pseudonym`.
+    #: Without it there is no pseudonym at all, rather than one computed from an empty
+    #: key -- that would be a plain digest of a small, enumerable value space, and
+    #: reversible by anyone able to hash the directory.
+    pseudonym_salt: SecretStr | None = None
+
+    #: Whether a person may be named in what leaves the process. The default is the
+    #: careful one, so that a deployment which has not thought about it does not
+    #: publish identifiers by omission.
+    person_uid_mode: PersonUidMode = "pseudonym"
+
+
+#: Where the OTLP endpoint comes from -- the OpenTelemetry specification's own
+#: variable, not a field of this class.
+#:
+#: An endpoint names a *receiver*, normally one per host or cluster, and which service
+#: sent a span is carried in the resource attributes rather than in the address. A
+#: field under ``EDUTAP_`` would be a second name for a value every OTel SDK already
+#: reads by itself. This is the asymmetry with the Sentry DSN, which names a project
+#: and therefore does belong to the service.
+OTLP_ENDPOINT_VARIABLE = "OTEL_EXPORTER_OTLP_ENDPOINT"

edutap_observability_settings-0.1.3.dist-info/METADATA

@@ -0,0 +1,145 @@
+Metadata-Version: 2.5
+Name: edutap.observability_settings
+Version: 0.1.3
+Summary: Error reporting, tracing and structured logging, wired the same way in every eduTAP service
+Project-URL: Source, https://github.com/edutap-collective/edutap.observability_settings
+Project-URL: Issues, https://github.com/edutap-collective/edutap.observability_settings/issues
+Author: eduTAP
+License-Expression: EUPL-1.2
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Monitoring
+Classifier: Typing :: Typed
+Requires-Python: >=3.13
+Requires-Dist: edutap-data-models<0.3,>=0.2.1
+Requires-Dist: logfire>=4.40
+Requires-Dist: pydantic-settings>=2.4
+Requires-Dist: pydantic>=2.8
+Requires-Dist: sentry-sdk>=2.67
+Requires-Dist: structlog>=25
+Provides-Extra: dev
+Requires-Dist: pdbp; extra == 'dev'
+Requires-Dist: pytest>=8.2; extra == 'dev'
+Requires-Dist: ruff<0.17,>=0.16; extra == 'dev'
+Requires-Dist: ty; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: myst-parser; extra == 'docs'
+Requires-Dist: sphinx>=8; extra == 'docs'
+Description-Content-Type: text/markdown
+
+# edutap.observability_settings
+
+Error reporting, tracing and structured logging, wired the same way in every eduTAP
+service — and one decision written down: what a service may say about a person.
+
+## Why this exists
+
+**Because the options are the point, not the code.** Three lines would start Sentry.
+Which options those three lines carry decides whether a bearer token or a person's
+identifier leaves the process, and each of the options below was chosen against a
+measurement rather than against a backend's recommendation. Separated from the
+`sentry_sdk.init()` call that applies them, those measurements are worth nothing —
+which is why this is a package and not a paragraph in a README.
+
+**Because a person's identifier is not an opaque handle.** At a university a
+`person_uid` resolves to a human being for far more people than hold directory
+administration rights; at the LMU it is the LMU identifier with `@lmu.de` appended.
+Pseudonymised it remains personal data. What a deployment decides here is not
+*whether* it is personal data but *who may see it*.
+
+## Usage
+
+```python
+from edutap.observability_settings import install_observability, person_label
+
+install_observability(service_name="lmu_edutap_worker")   # first thing in main()
+
+log.warning("no view for person", person=person_label(uid, settings))
+```
+
+Call it before the service resolves the settings it needs to run, so that a process
+refusing to start is reported rather than silently absent. Nothing here can fail for
+want of a value.
+
+## Configuration
+
+One prefix for the whole estate, `EDUTAP_`. These fields are defined by an eduTAP
+package, and another university deploying them should not have to learn an LMU name.
+
+| Variable | Default | Meaning |
+|---|---|---|
+| `EDUTAP_ENVIRONMENT` | `production` | Labels every event and every span. Unset must not masquerade as development. |
+| `EDUTAP_TELEMETRY_ENABLED` | `true` | The deliberate off switch for tracing, metrics and log export. |
+| `EDUTAP_LOG_LEVEL` | `INFO` | A closed set; a misspelled level fails at startup. |
+| `EDUTAP_SENTRY_DSN` | unset | Unset means no error tracker. One project per service. |
+| `EDUTAP_PSEUDONYM_SALT` | unset | The HMAC key behind the person pseudonym. Without it there is no pseudonym at all. |
+| `EDUTAP_PERSON_UID_MODE` | `pseudonym` | `pseudonym` · `plain` · `omit` |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | unset | **Not** an `EDUTAP_` field — see below. |
+
+### Why the endpoint is not one of ours
+
+A Sentry DSN names a **project**, so it belongs to the service and every service gets
+its own. An OTLP endpoint names a **receiver**, normally one per host or cluster, and
+which service sent a span rides in the resource attributes rather than in the address.
+Every OpenTelemetry SDK already reads `OTEL_EXPORTER_OTLP_ENDPOINT` by itself; a
+field under `EDUTAP_` would be a second name for the same value.
+
+```{note}
+A shared prefix means shared *names*, not shared *values*. `EDUTAP_SENTRY_DSN` differs
+per service by being set in that service's own compose `environment:` block. Only
+genuinely stack-wide values belong in the shared `.env`.
+```
+
+### The three modes for a person
+
+| Mode | What travels | When it is right |
+|---|---|---|
+| `pseudonym` | a keyed, 12-character label | The default. Correlation survives — forty errors about one person still read as one person — identification does not. |
+| `plain` | the `person_uid` itself | Where the error tracker is read by exactly the people who may read the directory anyway. A configured decision, not an accident. |
+| `omit` | nothing | Where the tracker is read more widely than the directory. |
+
+`pseudonym` without `EDUTAP_PSEUDONYM_SALT` yields **nothing**, never the raw value: a
+deployment that asked for pseudonyms and forgot the key has to lose the datum rather
+than publish it. An empty salt counts as no salt — compose writes `${VAR:-}`, which
+sets a variable to the empty string, and an HMAC under an empty key is a plain digest
+of a small, enumerable value space and reversible by anyone who can hash the directory.
+
+## What the three backends do
+
+Sentry takes errors. An OTLP collector takes traces and metrics. structlog produces
+the records that reach both, bridged by `logfire.StructlogProcessor`, so a log line
+and the span it happened inside share a trace id without the caller doing anything.
+
+Nothing travels two paths: Sentry's own tracing stays off (`traces_sample_rate=0`),
+because the spans already go to the collector and Bugsink — the tracker this estate
+runs — states that it does not support traces.
+
+```{important}
+`send_to_logfire=False` is not a detail. The library defaults it to `True`, so leaving
+it unset would ship spans to a hosted third party the first time a token happened to
+be present.
+```
+
+While no collector exists, the console stands in. Measured against logfire 4.40: with
+`send_to_logfire=False` and no `OTEL_EXPORTER_OTLP_ENDPOINT`, no exporter is installed
+at all, so an instrumented service would be indistinguishable from an uninstrumented
+one — which is how instrumentation reaches production broken. Once the endpoint is
+set, the console stands down.
+
+## Development
+
+```shell
+make venv
+make lint
+make test-local
+```
+
+`tox` runs the suite across every supported Python version.
+
+## Design records
+
+Under [`docs/superpowers/specs/`](docs/superpowers/specs/).

edutap_observability_settings-0.1.3.dist-info/RECORD

@@ -0,0 +1,8 @@
+edutap/observability_settings/__init__.py,sha256=f4m0YUPEnwo7yP2zN4YgzYr3Q8fPQTwButFmO-VWPEM,1327
+edutap/observability_settings/install.py,sha256=2dExNLEuQ5HW6kD1TKlnuaEgdG-rz_4ruvhmbQ_vDfY,6926
+edutap/observability_settings/pseudonym.py,sha256=rgJcQbOcbdC-bDqlOm3iYEoT9nQA0w7QWaF7BWq-jok,2458
+edutap/observability_settings/settings.py,sha256=TEfAQKoWmiVgfsUCZNn89JRtoOnFRkiwVd3AO6RWzn8,3653
+edutap_observability_settings-0.1.3.dist-info/METADATA,sha256=u1BebmYrIcOakN0deuhb1rz_HqwGzHOpOm1zvPh2oDo,6663
+edutap_observability_settings-0.1.3.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+edutap_observability_settings-0.1.3.dist-info/licenses/LICENSE,sha256=2UI9yv7y2X_WmlpE41IbjrcqqIrC3cAWMh40FWrSNLo,428
+edutap_observability_settings-0.1.3.dist-info/RECORD,,

edutap_observability_settings-0.1.3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

edutap_observability_settings-0.1.3.dist-info/licenses/LICENSE

@@ -0,0 +1,11 @@
+SPDX-License-Identifier: EUPL-1.2
+
+Licensed under the European Union Public Licence, Version 1.2 (the "EUPL").
+
+You may obtain a copy of the licence at:
+
+    https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the Licence is distributed on an "AS IS" basis, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.