bricklogger 0.2.3__tar.gz → 0.2.4__tar.gz

{bricklogger-0.2.3 → bricklogger-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bricklogger
-Version: 0.2.3
+Version: 0.2.4
 Summary: Data bridge for building automation: collects data based on a Brick model and writes it to time-series databases
 Keywords: brick,bacnet,building-automation,timescaledb,data-logger
 Author: Michael Rosenlund
@@ -84,7 +84,7 @@ uv tool install bricklogger
 uv brings a Python that fits, so the machine needs nothing else. Everything
 stays in that login's home directory: the program in uv's tool directory, the
 command in `~/.local/bin`, the configuration in `~/.config/bricklogger` and the
-data in `~/.local/share/bricklogger`. `uv tool install bricklogger==0.2.3`
+data in `~/.local/share/bricklogger`. `uv tool install bricklogger==0.2.4`
 installs a given version.
 
 **As a container** the same program is `ghcr.io/cx1-aps/bricklogger`, with a

{bricklogger-0.2.3 → bricklogger-0.2.4}/README.md

@@ -39,7 +39,7 @@ uv tool install bricklogger
 uv brings a Python that fits, so the machine needs nothing else. Everything
 stays in that login's home directory: the program in uv's tool directory, the
 command in `~/.local/bin`, the configuration in `~/.config/bricklogger` and the
-data in `~/.local/share/bricklogger`. `uv tool install bricklogger==0.2.3`
+data in `~/.local/share/bricklogger`. `uv tool install bricklogger==0.2.4`
 installs a given version.
 
 **As a container** the same program is `ghcr.io/cx1-aps/bricklogger`, with a

{bricklogger-0.2.3 → bricklogger-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bricklogger"
-version = "0.2.3"
+version = "0.2.4"
 description = "Data bridge for building automation: collects data based on a Brick model and writes it to time-series databases"
 readme = "README.md"
 license = "MIT"

{bricklogger-0.2.3 → bricklogger-0.2.4}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "bricklogger"
-version = "0.2.3"
+version = "0.2.4"
 description = "Data bridge for building automation: collects data based on a Brick model and writes it to time-series databases"
 readme = "README.md"
 license = "MIT"

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/__init__.py

@@ -14,4 +14,4 @@ The package is laid out after the architecture in ``docs/architecture.md``:
   external plugins.
 """
 
-__version__ = "0.2.3"
+__version__ = "0.2.4"

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/cli/model.py

@@ -299,11 +299,30 @@ def export(
     values: Annotated[
         bool, typer.Option("--values", help="Add the value overlay.")
     ] = False,
+    timeseries: Annotated[
+        bool,
+        typer.Option(
+            "--timeseries",
+            help="Add the time-series references of the destination that stores "
+            "the model.",
+        ),
+    ] = False,
+    destination: Annotated[
+        str | None,
+        typer.Option(
+            "--destination",
+            help="With --timeseries: the destination instance, when several "
+            "store the model.",
+        ),
+    ] = None,
     output: Annotated[
         Path | None, typer.Option("-o", "--output", help="Write to a file.")
     ] = None,
 ) -> None:
-    """The model as uploaded; the inferred graph and the values need the daemon."""
+    """The model as uploaded; the inferred graph, the values and the time-series
+    references need the daemon."""
+    if destination is not None and not timeseries:
+        raise fail("--destination goes with --timeseries")
     context = cli_context(ctx)
     client = reachable_client(context)
     if client is not None:
@@ -316,11 +335,13 @@ def export(
             f"/v1/models/{number}",
             inferred="true" if inferred else None,
             values="true" if values else None,
+            timeseries=(destination or "true") if timeseries else None,
         )
     else:
-        if inferred or values:
+        if inferred or values or timeseries:
             raise fail(
-                "--inferred and --values need the running daemon's working graph"
+                "--inferred, --values and --timeseries need the running daemon's "
+                "working graph"
             )
         store = _store(ctx)
         number = version if version is not None else store.active()

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/cli/plugins.py

@@ -327,6 +327,7 @@ def _echo_declaration(data: dict[str, Any]) -> None:
             echo_schema(tool.get("parameters"), indent="    ")
     else:
         typer.echo(f"stores metadata: {'yes' if data.get('stores_metadata') else 'no'}")
+        typer.echo(f"stores the model: {'yes' if data.get('stores_model') else 'no'}")
     typer.echo("configuration:")
     echo_schema(data.get("config_schema"), indent="  ")
 

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/cli/roles.py

@@ -335,6 +335,8 @@ def _row(role: str, name: str, type_name: str, state: str) -> dict[str, Any]:
         "written": 0,
         "last_write": None,
         "stores_metadata": None,
+        "stores_model": None,
+        "model_version": None,
         "spool": {
             "observations": 0,
             "bytes": 0,
@@ -438,7 +440,8 @@ def _echo_detail(role: str, row: Mapping[str, Any]) -> None:
     age = spool.get("oldest_age_seconds")
     typer.echo(
         f"{row['name']}: last write {row.get('last_write') or 'never'}, "
-        f"stores metadata {'yes' if row.get('stores_metadata') else 'no'}"
+        f"stores metadata {'yes' if row.get('stores_metadata') else 'no'}, "
+        f"stores the model {_model_note(row)}"
     )
     typer.echo(
         f"  spool: {spool.get('observations', 0)} observations, "
@@ -683,3 +686,11 @@ def _file(role: str) -> str:
 
 sources_app = _role_app("source", _SourcesGroup)
 destinations_app = _role_app("destination", _DestinationsGroup)
+
+
+def _model_note(row: Mapping[str, Any]) -> str:
+    """Whether a destination stores the model, and the version it last wrote."""
+    if not row.get("stores_model"):
+        return "no"
+    version = row.get("model_version")
+    return "yes" if version is None else f"yes (version {version})"

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/daemon/api.py

@@ -32,6 +32,7 @@ from bricklogger.daemon.core import (
 from bricklogger.daemon.jobs import Job
 from bricklogger.daemon.sparql import QueryError
 from bricklogger.model import ModelNotFound, ModelUnreadable
+from bricklogger.ops.status import live_answer
 
 PROBLEM = "application/problem+json"
 YAML = "application/yaml"
@@ -135,8 +136,8 @@ def create_app(
     # --- health and daemon ---------------------------------------------------
 
     @app.get("/health/live")
-    def live() -> dict[str, str]:
-        return {"live": "ok"}
+    def live() -> dict[str, Any]:
+        return live_answer(daemon.started_at)
 
     @app.get("/health")
     def health() -> JSONResponse:
@@ -483,12 +484,19 @@ def create_app(
 
     @app.get("/v1/models/{version}")
     def export_model(
-        version: int, inferred: bool = False, values: bool = False
+        version: int,
+        inferred: bool = False,
+        values: bool = False,
+        timeseries: str | None = None,
     ) -> Response:
         try:
-            data, fmt = daemon.export(version, inferred=inferred, values=values)
+            data, fmt = daemon.export(
+                version, inferred=inferred, values=values, timeseries=timeseries
+            )
         except ModelNotFound as exc:
             return problem(404, "Model version not found", str(exc))
+        except NotFound as exc:
+            return problem(404, "No time-series references", str(exc))
         except ExportUnavailable as exc:
             return problem(409, "Export unavailable", str(exc))
         return Response(

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/daemon/core.py

@@ -61,6 +61,7 @@ from bricklogger.model import (
     Inference,
     ModelDiff,
     ModelInvalid,
+    ModelNotFound,
     ModelReport,
     ModelStore,
     ModelUnreadable,
@@ -78,8 +79,14 @@ from bricklogger.model.prefixes import (
     declared_prefixes,
     expand,
 )
+from bricklogger.model.timeseries import (
+    REF,
+    TIMESERIES_GRAPH_PREFIX,
+    reference_triples,
+    timeseries_graph,
+)
 from bricklogger.model.working_graph import GRAPH_DIR
-from bricklogger.sdk.contract import AssignedPoint, Outcome
+from bricklogger.sdk.contract import AssignedPoint, ModelDocument, Outcome
 from bricklogger.sdk.registry import PluginRegistry
 
 log = logging.getLogger(__name__)
@@ -225,6 +232,7 @@ class Daemon:
         self.model_error: str | None = None
         self.started_at: datetime | None = None
         self._fallback_active: set[str] = set()
+        self._turtle: dict[int, str] = {}
         self._lock = threading.RLock()
         self.jobs = JobRunner()
 
@@ -375,6 +383,9 @@ class Daemon:
             self.graph.replace_model(
                 inference.model, inference.ontology, inference.inferred
             )
+            if number != self.active_version:
+                # another version's references until each destination writes this one
+                self._clear_timeseries_graphs()
             self.model_store.set_active(number)
             self.state.record_activation(number)
             self.active_version = number
@@ -488,10 +499,8 @@ class Daemon:
 
     def models(self) -> dict[str, Any]:
         """The stored versions, which one is active, and when each was activated."""
-        assert self.model_store is not None and self.state is not None
-        history: dict[int, list[str]] = {}
-        for row in self.state.activations():
-            history.setdefault(int(row["version"]), []).append(str(row["activated_at"]))
+        assert self.model_store is not None
+        history = self._activation_history()
         return {
             "active": self.active_version,
             "prefixes": dict(self.graph_access.prefixes) if self.graph_access else {},
@@ -512,20 +521,30 @@ class Daemon:
         return run_query(self.graph, prefixes, query, accept)
 
     def export(
-        self, version: int, *, inferred: bool = False, values: bool = False
+        self,
+        version: int,
+        *,
+        inferred: bool = False,
+        values: bool = False,
+        timeseries: str | None = None,
     ) -> tuple[bytes, str]:
         """A version as uploaded, with its format name.
 
-        With the inferred graph or the values added the result is Turtle built
-        from the working graph, which holds them for the active version only.
+        With the inferred graph, the values or a destination's time-series
+        references added the result is Turtle built from the working graph,
+        which holds them for the active version only. ``timeseries`` names the
+        destination instance, or is ``"true"`` for the one that stores the
+        model.
         """
         assert self.model_store is not None and self.graph is not None
         stored = self.model_store.get(version)
-        if not inferred and not values:
+        if not inferred and not values and timeseries is None:
             return self.model_store.read(version), stored.format
+        instance = None if timeseries is None else self._reference_instance(timeseries)
         if version != self.active_version:
             raise ExportUnavailable(
-                "the inferred graph and the values exist for the active version only"
+                "the inferred graph, the values and the time-series references "
+                "exist for the active version only"
             )
         graph = rdflib.Graph()
         graph.parse(data=self.graph.dump(MODEL_GRAPH), format="nt")
@@ -533,6 +552,9 @@ class Daemon:
             graph.parse(data=self.graph.dump(INFERRED_GRAPH), format="nt")
         if values:
             graph.parse(data=self.graph.dump(VALUES_GRAPH), format="nt")
+        if instance is not None:
+            graph.parse(data=self.graph.dump(timeseries_graph(instance)), format="nt")
+            graph.bind("ref", REF)
         prefixes = self.graph_access.prefixes if self.graph_access else WELL_KNOWN
         for prefix, iri in prefixes.items():
             graph.bind(prefix, iri, override=True)
@@ -551,6 +573,7 @@ class Daemon:
             last is None or last[0] != marker or self.graph.count(MODEL_GRAPH) == 0
         )
         if needs_build:
+            self._clear_timeseries_graphs()
             try:
                 activate_version(
                     self.model_store,
@@ -571,6 +594,101 @@ class Daemon:
         self.active_version = marker
         self._refresh_graph_access()
 
+    # --- the model in destinations -------------------------------------------
+
+    def _reference_instance(self, timeseries: str) -> str:
+        """The destination instance whose references an export adds: the one
+        named, or with ``"true"`` the only one that stores the model."""
+        storing = sorted(
+            name for name, runner in self.destinations.items() if runner.stores_model
+        )
+        if timeseries == "true":
+            if not storing:
+                raise NotFound("no destination stores the model")
+            if len(storing) > 1:
+                raise ExportUnavailable(
+                    "several destinations store the model; name one: "
+                    + ", ".join(storing)
+                )
+            return storing[0]
+        if timeseries not in storing:
+            raise NotFound(f"no destination {timeseries!r} stores the model")
+        return timeseries
+
+    def _clear_timeseries_graphs(self, instance: str | None = None) -> None:
+        assert self.graph is not None
+        for iri in self.graph.named_graphs():
+            if iri.startswith(TIMESERIES_GRAPH_PREFIX) and (
+                instance is None or iri == timeseries_graph(instance)
+            ):
+                self.graph.clear_graph(iri)
+
+    def _model_document(
+        self, version: int, activations: Sequence[str]
+    ) -> ModelDocument:
+        """A version as Turtle, without references; the runner adds them."""
+        assert self.model_store is not None
+        stored = self.model_store.get(version)
+        turtle = self._turtle.get(version)
+        if turtle is None:
+            model = parse_model(self.model_store.read(version), stored.format)
+            turtle = model.serialize(format="turtle")
+            self._turtle[version] = turtle
+        return ModelDocument(
+            version=version,
+            uploaded_at=stored.uploaded_at,
+            activations=tuple(datetime.fromisoformat(a) for a in activations),
+            turtle=turtle,
+        )
+
+    def _activation_history(self) -> dict[int, list[str]]:
+        assert self.state is not None
+        history: dict[int, list[str]] = {}
+        for row in self.state.activations():
+            history.setdefault(int(row["version"]), []).append(str(row["activated_at"]))
+        return history
+
+    def _offer_models(self, spools: Sequence[Spool], *, every: bool) -> None:
+        """Spool the active version, or every version that has been active,
+        for the destinations that store the model."""
+        assert self.model_store is not None
+        if not spools:
+            return
+        history = self._activation_history()
+        if every:
+            versions = sorted(history)
+        elif self.active_version is not None:
+            versions = [self.active_version]
+        else:
+            versions = []
+        for version in versions:
+            try:
+                document = self._model_document(version, history.get(version, []))
+            except (ModelNotFound, ModelUnreadable):
+                log.warning("model version %d cannot be offered", version)
+                continue
+            for spool in spools:
+                spool.append_model(document)
+
+    def _model_spools(self) -> list[Spool]:
+        return [
+            self.spools[name]
+            for name, runner in self.destinations.items()
+            if runner.stores_model
+        ]
+
+    def _on_model_written(
+        self, instance: str, version: int, ids: Mapping[str, str]
+    ) -> None:
+        """Called from a destination's thread once it stored a version; the
+        active version's keys become the instance's graph of references."""
+        if version != self.active_version or self.graph is None:
+            return
+        try:
+            self.graph.replace_graph(timeseries_graph(instance), reference_triples(ids))
+        except Exception:
+            log.exception("the time-series references of %s were not stored", instance)
+
     def _refresh_graph_access(self) -> None:
         assert self.graph is not None and self.model_store is not None
         prefixes = {**WELL_KNOWN, **self.registry.prefixes()}
@@ -615,8 +733,12 @@ class Daemon:
             spool_max_age=instance.spool.max_age,
             stop_timeout=self.configuration.daemon.stop_timeout,
             stores_metadata=declaration.stores_metadata,
+            stores_model=declaration.stores_model,
+            on_model_written=self._on_model_written,
         )
         self.destinations[name] = runner
+        if declaration.stores_model:
+            self._offer_models([spool], every=True)
         if self.state.stop_intent(name):
             runner.current_state = "stopped"
             self.state.set_instance(
@@ -693,6 +815,7 @@ class Daemon:
                 continue
             self.destinations.pop(name).stop()
             self.spools.pop(name).close()
+            self._clear_timeseries_graphs(name)
         for name, instance in new.items():
             if name in old and old[name] == instance:
                 continue
@@ -854,6 +977,8 @@ class Daemon:
         if assigned and self.sink is not None:
             entries = graph_metadata(self.graph, self.graph_access.prefixes, assigned)
             self.sink.metadata(entries.values())
+        # behind the metadata, so the destinations hold the keys the model needs
+        self._offer_models(self._model_spools(), every=False)
         for name, runner in self.sources.items():
             runner.assign(plan.assignments.get(name, []))
 
@@ -1047,6 +1172,8 @@ class Daemon:
                     "type": runner.type_name,
                     "state": runner.current_state,
                     "stores_metadata": runner.stores_metadata,
+                    "stores_model": runner.stores_model,
+                    "model_version": runner.model_version,
                     "last_error": row.get("last_error"),
                     "last_write": runner.last_write,
                     "written": self.state.counter(f"written:{name}"),
@@ -1067,6 +1194,8 @@ class Daemon:
                     "type": entry.type_name,
                     "state": "failed",
                     "stores_metadata": None,
+                    "stores_model": None,
+                    "model_version": None,
                     "last_error": entry.error,
                     "last_write": None,
                     "written": 0,

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/daemon/instances.py

@@ -7,11 +7,13 @@ from __future__ import annotations
 import logging
 import threading
 import time
-from collections.abc import Callable, Iterable, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from dataclasses import replace
 from datetime import timedelta
 
 from bricklogger.daemon.spool import Entry, Spool
 from bricklogger.daemon.state import RuntimeState
+from bricklogger.model.timeseries import with_references
 from bricklogger.sdk.contract import (
     AssignedPoint,
     Destination,
@@ -256,6 +258,8 @@ class DestinationRunner:
         spool_max_age: timedelta,
         stop_timeout: timedelta,
         stores_metadata: bool,
+        stores_model: bool = False,
+        on_model_written: Callable[[str, int, Mapping[str, str]], None] | None = None,
     ) -> None:
         self.name = name
         self.type_name = type_name
@@ -268,6 +272,9 @@ class DestinationRunner:
         self.spool_max_age = spool_max_age
         self.stop_timeout = stop_timeout
         self.stores_metadata = stores_metadata
+        self.stores_model = stores_model
+        self.on_model_written = on_model_written
+        self.model_version: int | None = None
         self._thread: threading.Thread | None = None
         self._stopping = threading.Event()
         self._failures = 0
@@ -375,6 +382,10 @@ class DestinationRunner:
                 if self.stores_metadata:
                     for entry in batch:
                         self.destination.write_metadata(entry.metadata())
+            elif batch[0].kind == "model":
+                if self.stores_model:
+                    for entry in batch:
+                        self._write_model(entry)
             else:
                 observations = [o for entry in batch for o in entry.observations()]
                 self.destination.write(observations)
@@ -397,6 +408,19 @@ class DestinationRunner:
         self.last_write = time.strftime("%Y-%m-%dT%H:%M:%S%z")
         return True
 
+    def _write_model(self, entry: Entry) -> None:
+        """Add the destination's references to a model version and store it.
+
+        The keys are asked for at write time, after the metadata ahead of the
+        model in the spool, so every point the plan offered already has one.
+        """
+        model = entry.model()
+        turtle, used = with_references(model.turtle, self.destination.timeseries_ids())
+        self.destination.write_model(replace(model, turtle=turtle))
+        self.model_version = model.version
+        if self.on_model_written is not None:
+            self.on_model_written(self.name, model.version, used)
+
     def _enforce_caps(self) -> None:
         dropped = self.spool.enforce_caps(self.spool_max_size, self.spool_max_age)
         if dropped:

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/daemon/plugins.py

@@ -141,6 +141,7 @@ def describe_plugin(
         ]
     elif destination is not None:
         data["stores_metadata"] = destination.stores_metadata
+        data["stores_model"] = destination.stores_model
     return data
 
 

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/daemon/sparql.py

@@ -80,7 +80,7 @@ def run_query(
     if is_update(query):
         raise QueryError("update requests are refused: the graph is read-only")
     try:
-        result = graph.query(with_prefixes(query, prefixes))
+        result = graph.query(with_prefixes(query, prefixes), references=True)
     except SyntaxError as exc:
         raise QueryError(f"the query does not parse: {exc}") from exc
     except (OSError, ValueError) as exc:

{bricklogger-0.2.3 → bricklogger-0.2.4}/src/bricklogger/daemon/spool.py

@@ -1,8 +1,8 @@
 """The persistent spool between the daemon and one destination.
 
-An on-disk queue that observations and metadata are appended to and drained
-from in order, in batches; it survives a daemon restart and is bounded by size
-and age, dropping the oldest entries when a cap is reached. See
+An on-disk queue that observations, metadata and model versions are appended
+to and drained from in order, in batches; it survives a daemon restart and is
+bounded by size and age, dropping the oldest entries when a cap is reached. See
 ``docs/architecture.md``, "Backpressure: the spool".
 """
 
@@ -18,16 +18,17 @@ from datetime import timedelta
 from pathlib import Path
 from typing import Any, Literal
 
-from bricklogger.sdk.contract import Observation, PointMetadata
+from bricklogger.sdk.contract import ModelDocument, Observation, PointMetadata
 
 SPOOL_DIR = "spool"
 
-EntryKind = Literal["observations", "metadata"]
+EntryKind = Literal["observations", "metadata", "model"]
 
 
 @dataclass(frozen=True)
 class Entry:
-    """One spooled batch: observations or metadata, as it was appended."""
+    """One spooled batch: observations, metadata or one model version, as it
+    was appended."""
 
     id: int
     kind: EntryKind
@@ -39,6 +40,9 @@ class Entry:
     def metadata(self) -> list[PointMetadata]:
         return [PointMetadata.from_dict(item) for item in self.payload]
 
+    def model(self) -> ModelDocument:
+        return ModelDocument.from_dict(self.payload[0])
+
 
 @dataclass(frozen=True)
 class SpoolStats:
@@ -82,6 +86,9 @@ class Spool:
         if entries:
             self._append("metadata", [e.as_dict() for e in entries])
 
+    def append_model(self, model: ModelDocument) -> None:
+        self._append("model", [model.as_dict()])
+
     def _append(self, kind: EntryKind, items: list[dict[str, Any]]) -> None:
         payload = json.dumps(items, separators=(",", ":"))
         with self._lock:
@@ -95,8 +102,9 @@ class Spool:
     def next_entries(self, max_observations: int) -> list[Entry]:
         """The oldest entries, in order, up to about ``max_observations`` observations.
 
-        Metadata entries break a run, so a destination learns about a point
-        before it sees the point's values, in the order things were appended.
+        Metadata and model entries break a run, so a destination learns about
+        a point before it sees the point's values, and holds its key before
+        the model names it, in the order things were appended.
         """
         entries: list[Entry] = []
         collected = 0
@@ -110,7 +118,7 @@ class Spool:
                 break
             entries.append(Entry(int(row["id"]), kind, json.loads(row["payload"])))
             collected += int(row["items"])
-            if kind == "metadata" or collected >= max_observations:
+            if kind != "observations" or collected >= max_observations:
                 break
         return entries