opendataframework 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opendataframework-0.3.0 → opendataframework-0.4.0}/PKG-INFO +1 -1
- {opendataframework-0.3.0 → opendataframework-0.4.0}/pyproject.toml +1 -1
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/__init__.py +5 -1
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/context.py +146 -66
- opendataframework-0.4.0/src/opendataframework/events.py +188 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/function.py +23 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/view.py +16 -5
- {opendataframework-0.3.0 → opendataframework-0.4.0}/LICENSE +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/README.md +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/agent.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/component.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/config.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/entity.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/layer.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/logger.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/namespace.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/pipeline.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/project.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/py.typed +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/repository.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/service.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/task.py +0 -0
- {opendataframework-0.3.0 → opendataframework-0.4.0}/src/opendataframework/utils.py +0 -0
|
@@ -5,7 +5,8 @@ from opendataframework.component import Component
|
|
|
5
5
|
from opendataframework.config import Config
|
|
6
6
|
from opendataframework.context import Context, Lifecycle, Resolver
|
|
7
7
|
from opendataframework.entity import Entity
|
|
8
|
-
from opendataframework.
|
|
8
|
+
from opendataframework.events import Event, Events
|
|
9
|
+
from opendataframework.function import Function, FunctionProtocol
|
|
9
10
|
from opendataframework.layer import Analytics, Api, Layer, Messaging, Monitoring, Security, Storage
|
|
10
11
|
from opendataframework.logger import Logger
|
|
11
12
|
from opendataframework.namespace import Namespace
|
|
@@ -51,7 +52,10 @@ __all__ = [
|
|
|
51
52
|
"DataViewProtocol",
|
|
52
53
|
"DocumentView",
|
|
53
54
|
"Entity",
|
|
55
|
+
"Event",
|
|
56
|
+
"Events",
|
|
54
57
|
"Function",
|
|
58
|
+
"FunctionProtocol",
|
|
55
59
|
"ImageView",
|
|
56
60
|
"Layer",
|
|
57
61
|
"Lifecycle",
|
|
@@ -7,8 +7,11 @@ dispatch: setup/run/stop for services, on_start/on_stop for components),
|
|
|
7
7
|
and ``StreamManager`` (background threads for ``StreamableProtocol``
|
|
8
8
|
repositories). ``Context`` is the only public entry point of the three —
|
|
9
9
|
``Resolver``/``Lifecycle``/``StreamManager`` are internal collaborators.
|
|
10
|
+
Every protocol call the three make is logged and also reported as an
|
|
11
|
+
``Event`` to the ``Context``'s ``Events`` (see ``opendataframework.events``).
|
|
10
12
|
"""
|
|
11
13
|
|
|
14
|
+
import contextvars
|
|
12
15
|
import functools
|
|
13
16
|
import inspect
|
|
14
17
|
import threading
|
|
@@ -18,13 +21,14 @@ from pathlib import Path
|
|
|
18
21
|
from queue import Empty, Full, Queue
|
|
19
22
|
from typing import Annotated, TypeAliasType, TypeVar, cast, get_origin, overload
|
|
20
23
|
|
|
21
|
-
from opendataframework.agent import Agent
|
|
24
|
+
from opendataframework.agent import Agent, AgentProtocol
|
|
22
25
|
from opendataframework.component import Component, OnStartProtocol, OnStopProtocol
|
|
23
26
|
from opendataframework.config import Config
|
|
24
|
-
from opendataframework.
|
|
27
|
+
from opendataframework.events import Events
|
|
28
|
+
from opendataframework.function import Function, FunctionProtocol, bind, string_hint
|
|
25
29
|
from opendataframework.logger import Logger, LogManager
|
|
26
30
|
from opendataframework.namespace import Namespace, parse, qualified
|
|
27
|
-
from opendataframework.pipeline import Pipeline
|
|
31
|
+
from opendataframework.pipeline import Pipeline, PipelineProtocol
|
|
28
32
|
from opendataframework.repository import (
|
|
29
33
|
CloseStreamProtocol,
|
|
30
34
|
OpenStreamProtocol,
|
|
@@ -32,7 +36,7 @@ from opendataframework.repository import (
|
|
|
32
36
|
StreamableProtocol,
|
|
33
37
|
)
|
|
34
38
|
from opendataframework.service import Service, ServiceProtocol
|
|
35
|
-
from opendataframework.task import Task
|
|
39
|
+
from opendataframework.task import Task, TaskProtocol
|
|
36
40
|
from opendataframework.utils import normalize
|
|
37
41
|
|
|
38
42
|
_STREAM_ENDED = object()
|
|
@@ -54,27 +58,31 @@ def _is_async(func: object) -> bool:
|
|
|
54
58
|
return inspect.iscoroutinefunction(func) or inspect.isasyncgenfunction(func)
|
|
55
59
|
|
|
56
60
|
|
|
57
|
-
def _logged_execute(
|
|
61
|
+
def _logged_execute(
|
|
62
|
+
execute, name: str, log_manager: LogManager, events: Events, protocol: type, method: str
|
|
63
|
+
):
|
|
58
64
|
"""Wrap a bound ``execute`` method to log its start, duration, and errors.
|
|
59
65
|
|
|
60
|
-
``
|
|
61
|
-
serves an ``Agent``'s ``invoke()`` and a ``Function``
|
|
62
|
-
``Context``.
|
|
66
|
+
``protocol`` and ``method`` name the protocol call being wrapped, so the
|
|
67
|
+
same wrapper also serves an ``Agent``'s ``invoke()`` and a ``Function``
|
|
68
|
+
called through the ``Context``. Each call is also reported to ``events``.
|
|
63
69
|
"""
|
|
70
|
+
label = "call" if protocol is FunctionProtocol else f"{method}()"
|
|
64
71
|
|
|
65
72
|
@functools.wraps(execute)
|
|
66
73
|
def wrapped(*args, **kwargs):
|
|
67
74
|
"""Call the wrapped ``execute``, logging its start, duration, and errors."""
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
with events.notify(name, protocol, method):
|
|
76
|
+
log_manager.write(name, "INFO", f"{label} started")
|
|
77
|
+
start = time.monotonic()
|
|
78
|
+
try:
|
|
79
|
+
result = execute(*args, **kwargs)
|
|
80
|
+
except Exception as exc:
|
|
81
|
+
log_manager.write(name, "ERROR", f"{label} raised {exc!r}")
|
|
82
|
+
raise
|
|
83
|
+
elapsed_ms = (time.monotonic() - start) * 1000
|
|
84
|
+
log_manager.write(name, "INFO", f"{label} completed in {elapsed_ms:.1f}ms")
|
|
85
|
+
return result
|
|
78
86
|
|
|
79
87
|
return wrapped
|
|
80
88
|
|
|
@@ -113,16 +121,21 @@ class Resolver:
|
|
|
113
121
|
derives the dependency graph from their signatures, sorts them
|
|
114
122
|
topologically, and instantiates each class — or binds each function —
|
|
115
123
|
with its dependencies injected. Has no knowledge
|
|
116
|
-
of lifecycle or framework protocols — the
|
|
124
|
+
of lifecycle or framework protocols — the exceptions are ``Logger``
|
|
117
125
|
(see ``opendataframework.logger``), which is resolved contextually: a ``logger: Logger``
|
|
118
126
|
constructor parameter receives an instance bound to the *consuming*
|
|
119
127
|
class's own key rather than a single shared instance, and every
|
|
120
|
-
instantiation is logged to that component's file
|
|
128
|
+
instantiation is logged to that component's file — and ``Events``
|
|
129
|
+
(see ``opendataframework.events``): an ``events: Events`` parameter
|
|
130
|
+
receives the ``Context``'s own instance, and every call of a bound
|
|
131
|
+
function is reported to it as a ``FunctionProtocol`` call.
|
|
121
132
|
|
|
122
133
|
Args:
|
|
123
134
|
namespaces: Namespace subclasses to collect registered classes from.
|
|
124
135
|
log_manager: Owns per-component log files. Defaults to a disabled
|
|
125
136
|
``LogManager`` (writes become no-ops) when omitted.
|
|
137
|
+
events: Receives an event for every call of a bound function.
|
|
138
|
+
Defaults to an ``Events`` with no listeners when omitted.
|
|
126
139
|
|
|
127
140
|
Example:
|
|
128
141
|
>>> class NS(Namespace): ...
|
|
@@ -144,6 +157,7 @@ class Resolver:
|
|
|
144
157
|
namespaces: Iterable[type[Namespace]],
|
|
145
158
|
extra_instances: dict[type, object] | None = None,
|
|
146
159
|
log_manager: LogManager | None = None,
|
|
160
|
+
events: Events | None = None,
|
|
147
161
|
) -> None:
|
|
148
162
|
"""See class docstring for ``Args``."""
|
|
149
163
|
self._namespaces = set(namespaces)
|
|
@@ -154,6 +168,7 @@ class Resolver:
|
|
|
154
168
|
self._keys: dict[type | Callable[..., object], str] = {}
|
|
155
169
|
self._by_key: dict[str, type | Callable[..., object]] = {}
|
|
156
170
|
self._log_manager = log_manager if log_manager is not None else LogManager()
|
|
171
|
+
self._events = events if events is not None else Events(self._log_manager)
|
|
157
172
|
|
|
158
173
|
@property
|
|
159
174
|
def instances(self) -> dict[type, object]:
|
|
@@ -254,7 +269,9 @@ class Resolver:
|
|
|
254
269
|
bind(node, self._injected[node]),
|
|
255
270
|
self._keys[node],
|
|
256
271
|
self._log_manager,
|
|
257
|
-
|
|
272
|
+
self._events,
|
|
273
|
+
FunctionProtocol,
|
|
274
|
+
"__call__",
|
|
258
275
|
)
|
|
259
276
|
|
|
260
277
|
def _index(self) -> None:
|
|
@@ -470,7 +487,8 @@ class Resolver:
|
|
|
470
487
|
|
|
471
488
|
A ``dep`` annotated exactly as ``Logger`` is special-cased: instead of
|
|
472
489
|
a shared instance, ``target`` receives a ``Logger`` bound to its own
|
|
473
|
-
key (see ``opendataframework.logger.LogManager.logger_for``).
|
|
490
|
+
key (see ``opendataframework.logger.LogManager.logger_for``). One
|
|
491
|
+
annotated exactly as ``Events`` receives the ``Context``'s ``Events``. A class is
|
|
474
492
|
supplied only if already present in ``self._instances``, and a
|
|
475
493
|
function only if already bound — unregistered ones in the signature
|
|
476
494
|
are silently ignored.
|
|
@@ -479,6 +497,8 @@ class Resolver:
|
|
|
479
497
|
for name, dep in self.dependencies(target).items():
|
|
480
498
|
if dep is Logger:
|
|
481
499
|
supplied[name] = self._log_manager.logger_for(self._keys[target])
|
|
500
|
+
elif dep is Events:
|
|
501
|
+
supplied[name] = self._events
|
|
482
502
|
elif dep in self._instances:
|
|
483
503
|
supplied[name] = self._instances[dep]
|
|
484
504
|
elif dep in self._functions:
|
|
@@ -495,7 +515,8 @@ class Lifecycle:
|
|
|
495
515
|
|
|
496
516
|
Every stage transition is also logged to the component's own log file via
|
|
497
517
|
``log_manager`` (see ``opendataframework.logger``), giving every ``Service`` a log
|
|
498
|
-
record with no code changes required
|
|
518
|
+
record with no code changes required, and every protocol call is reported
|
|
519
|
+
to ``events`` (see ``opendataframework.events``).
|
|
499
520
|
|
|
500
521
|
Args:
|
|
501
522
|
instances: Class-to-instance mapping produced by ``Resolver``.
|
|
@@ -504,6 +525,8 @@ class Lifecycle:
|
|
|
504
525
|
``Resolver.keys``. Names each class's log file.
|
|
505
526
|
log_manager: Owns per-component log files. Defaults to a disabled
|
|
506
527
|
``LogManager`` (writes become no-ops) when omitted.
|
|
528
|
+
events: Receives an event for every protocol call. Defaults to an
|
|
529
|
+
``Events`` with no listeners when omitted.
|
|
507
530
|
|
|
508
531
|
Example:
|
|
509
532
|
>>> lifecycle = Lifecycle(instances={...}, order=[...], keys={...})
|
|
@@ -517,12 +540,14 @@ class Lifecycle:
|
|
|
517
540
|
order: list[type],
|
|
518
541
|
keys: dict[type, str],
|
|
519
542
|
log_manager: LogManager | None = None,
|
|
543
|
+
events: Events | None = None,
|
|
520
544
|
) -> None:
|
|
521
545
|
"""See class docstring for ``Args``."""
|
|
522
546
|
self._instances = instances
|
|
523
547
|
self._order = order
|
|
524
548
|
self._keys = keys
|
|
525
549
|
self._log_manager = log_manager if log_manager is not None else LogManager()
|
|
550
|
+
self._events = events if events is not None else Events(self._log_manager)
|
|
526
551
|
self._threads: list[threading.Thread] = []
|
|
527
552
|
self._running: dict[type, bool] = {}
|
|
528
553
|
|
|
@@ -552,9 +577,10 @@ class Lifecycle:
|
|
|
552
577
|
def _activate_one(self, cls: type, instance: object) -> None:
|
|
553
578
|
"""Call a non-``Service`` instance's ``on_start()`` hook."""
|
|
554
579
|
name = self._keys[cls]
|
|
555
|
-
self.
|
|
556
|
-
|
|
557
|
-
|
|
580
|
+
with self._events.notify(name, OnStartProtocol, "on_start"):
|
|
581
|
+
self._log_manager.write(name, "INFO", "on_start() called")
|
|
582
|
+
instance.on_start()
|
|
583
|
+
self._log_manager.write(name, "INFO", "on_start() complete")
|
|
558
584
|
|
|
559
585
|
def start_one(self, cls: type) -> None:
|
|
560
586
|
"""Start a single ``ServiceProtocol`` instance. No-op if already running.
|
|
@@ -571,10 +597,15 @@ class Lifecycle:
|
|
|
571
597
|
return
|
|
572
598
|
instance = self._instances[cls]
|
|
573
599
|
name = self._keys[cls]
|
|
574
|
-
self.
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
600
|
+
with self._events.notify(name, ServiceProtocol, "setup"):
|
|
601
|
+
self._log_manager.write(name, "INFO", "setup() starting")
|
|
602
|
+
instance.setup()
|
|
603
|
+
self._log_manager.write(name, "INFO", "setup() complete")
|
|
604
|
+
# An empty context, so run() is a top-level call rather than a child
|
|
605
|
+
# of whatever call started the service.
|
|
606
|
+
t = threading.Thread(
|
|
607
|
+
target=self._run, args=(cls, instance), daemon=True, context=contextvars.Context()
|
|
608
|
+
)
|
|
578
609
|
t.start()
|
|
579
610
|
self._threads.append(t)
|
|
580
611
|
self._running[cls] = True
|
|
@@ -594,9 +625,10 @@ class Lifecycle:
|
|
|
594
625
|
return
|
|
595
626
|
instance = self._instances[cls]
|
|
596
627
|
name = self._keys[cls]
|
|
597
|
-
self.
|
|
598
|
-
|
|
599
|
-
|
|
628
|
+
with self._events.notify(name, ServiceProtocol, "stop"):
|
|
629
|
+
self._log_manager.write(name, "INFO", "stop() called")
|
|
630
|
+
instance.stop()
|
|
631
|
+
self._log_manager.write(name, "INFO", "stop() complete")
|
|
600
632
|
self._running[cls] = False
|
|
601
633
|
|
|
602
634
|
def is_running(self, cls: type) -> bool:
|
|
@@ -606,13 +638,14 @@ class Lifecycle:
|
|
|
606
638
|
def _run(self, cls: type, instance: ServiceProtocol) -> None:
|
|
607
639
|
"""Run ``instance.run()``, logging its start, return, and any exception."""
|
|
608
640
|
name = self._keys[cls]
|
|
609
|
-
self.
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
641
|
+
with self._events.notify(name, ServiceProtocol, "run"):
|
|
642
|
+
self._log_manager.write(name, "INFO", "run() started")
|
|
643
|
+
try:
|
|
644
|
+
instance.run()
|
|
645
|
+
self._log_manager.write(name, "INFO", "run() returned")
|
|
646
|
+
except Exception as exc:
|
|
647
|
+
self._log_manager.write(name, "ERROR", f"run() raised {exc!r}")
|
|
648
|
+
raise
|
|
616
649
|
|
|
617
650
|
def on_stop(self) -> None:
|
|
618
651
|
"""Deactivate all instances in reverse dependency order.
|
|
@@ -638,9 +671,10 @@ class Lifecycle:
|
|
|
638
671
|
def _deactivate_one(self, cls: type, instance: object) -> None:
|
|
639
672
|
"""Call a non-``Service`` instance's ``on_stop()`` hook."""
|
|
640
673
|
name = self._keys[cls]
|
|
641
|
-
self.
|
|
642
|
-
|
|
643
|
-
|
|
674
|
+
with self._events.notify(name, OnStopProtocol, "on_stop"):
|
|
675
|
+
self._log_manager.write(name, "INFO", "on_stop() called")
|
|
676
|
+
instance.on_stop()
|
|
677
|
+
self._log_manager.write(name, "INFO", "on_stop() complete")
|
|
644
678
|
|
|
645
679
|
|
|
646
680
|
def _publish_latest(q: Queue, item: object) -> None:
|
|
@@ -694,15 +728,21 @@ class StreamManager:
|
|
|
694
728
|
"""
|
|
695
729
|
|
|
696
730
|
def __init__(
|
|
697
|
-
self,
|
|
731
|
+
self,
|
|
732
|
+
keys: dict[type, str] | None = None,
|
|
733
|
+
log_manager: LogManager | None = None,
|
|
734
|
+
events: Events | None = None,
|
|
698
735
|
) -> None:
|
|
699
736
|
"""See class docstring; ``log_manager`` defaults to a disabled ``LogManager``.
|
|
700
737
|
|
|
701
738
|
``keys`` maps each class to the ``<namespace>.<name>`` key its log
|
|
702
|
-
lines go under, as returned by ``Resolver.keys``.
|
|
739
|
+
lines go under, as returned by ``Resolver.keys``. ``events`` receives
|
|
740
|
+
an event for every ``open_stream()``/``stream()``/``close_stream()``
|
|
741
|
+
call, and defaults to an ``Events`` with no listeners.
|
|
703
742
|
"""
|
|
704
743
|
self._keys = keys if keys is not None else {}
|
|
705
744
|
self._log_manager = log_manager if log_manager is not None else LogManager()
|
|
745
|
+
self._events = events if events is not None else Events(self._log_manager)
|
|
706
746
|
self._threads: dict[type, threading.Thread] = {}
|
|
707
747
|
self._stop_events: dict[type, threading.Event] = {}
|
|
708
748
|
self._subscribers: dict[type, list[Queue]] = {}
|
|
@@ -724,15 +764,19 @@ class StreamManager:
|
|
|
724
764
|
name = self._keys[cls]
|
|
725
765
|
try:
|
|
726
766
|
if isinstance(instance, OpenStreamProtocol):
|
|
727
|
-
self.
|
|
728
|
-
|
|
729
|
-
|
|
767
|
+
with self._events.notify(name, OpenStreamProtocol, "open_stream"):
|
|
768
|
+
self._log_manager.write(name, "INFO", "open_stream() called")
|
|
769
|
+
instance.open_stream()
|
|
770
|
+
self._log_manager.write(name, "INFO", "open_stream() complete")
|
|
730
771
|
with self._lock:
|
|
731
772
|
self._subscribers[cls] = []
|
|
732
773
|
event = threading.Event()
|
|
733
774
|
self._stop_events[cls] = event
|
|
734
775
|
thread = threading.Thread(
|
|
735
|
-
target=self._run,
|
|
776
|
+
target=self._run,
|
|
777
|
+
args=(cls, instance, event),
|
|
778
|
+
daemon=True,
|
|
779
|
+
context=contextvars.Context(),
|
|
736
780
|
)
|
|
737
781
|
self._threads[cls] = thread
|
|
738
782
|
finally:
|
|
@@ -789,11 +833,12 @@ class StreamManager:
|
|
|
789
833
|
"""Drive ``instance.stream()``, fanning entities out to subscribers until done."""
|
|
790
834
|
name = self._keys[cls]
|
|
791
835
|
try:
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
836
|
+
with self._events.notify(name, StreamableProtocol, "stream"):
|
|
837
|
+
for entity in instance.stream():
|
|
838
|
+
if stop_event.is_set():
|
|
839
|
+
break
|
|
840
|
+
for q in list(self._subscribers.get(cls, ())):
|
|
841
|
+
_publish_latest(q, entity)
|
|
797
842
|
except Exception as exc:
|
|
798
843
|
self._log_manager.write(name, "ERROR", f"stream raised {exc!r}")
|
|
799
844
|
finally:
|
|
@@ -802,9 +847,10 @@ class StreamManager:
|
|
|
802
847
|
# exception — close_stream() releases the resource in all three
|
|
803
848
|
# cases, not just an explicit stop_stream() call.
|
|
804
849
|
if isinstance(instance, CloseStreamProtocol):
|
|
805
|
-
self._log_manager.write(name, "INFO", "close_stream() called")
|
|
806
850
|
try:
|
|
807
|
-
|
|
851
|
+
with self._events.notify(name, CloseStreamProtocol, "close_stream"):
|
|
852
|
+
self._log_manager.write(name, "INFO", "close_stream() called")
|
|
853
|
+
instance.close_stream()
|
|
808
854
|
except Exception as exc:
|
|
809
855
|
self._log_manager.write(name, "ERROR", f"close_stream() raised {exc!r}")
|
|
810
856
|
else:
|
|
@@ -885,13 +931,15 @@ class Context:
|
|
|
885
931
|
"""See class docstring for ``Args``."""
|
|
886
932
|
extra = {Config: Config(config)} if config is not None else {}
|
|
887
933
|
self._log_manager = LogManager(log_dir)
|
|
934
|
+
self._events = Events(self._log_manager)
|
|
888
935
|
self._resolver = Resolver(
|
|
889
936
|
namespaces if namespaces is not None else _DEFAULT_NAMESPACES,
|
|
890
937
|
extra_instances=extra,
|
|
891
938
|
log_manager=self._log_manager,
|
|
939
|
+
events=self._events,
|
|
892
940
|
)
|
|
893
941
|
self._lifecycle: Lifecycle | None = None
|
|
894
|
-
self._stream_manager = StreamManager(log_manager=self._log_manager)
|
|
942
|
+
self._stream_manager = StreamManager(log_manager=self._log_manager, events=self._events)
|
|
895
943
|
self._functions: dict[Callable[..., object], Callable[..., object]] = {}
|
|
896
944
|
|
|
897
945
|
@property
|
|
@@ -904,6 +952,22 @@ class Context:
|
|
|
904
952
|
"""
|
|
905
953
|
return self._resolver.instances
|
|
906
954
|
|
|
955
|
+
@property
|
|
956
|
+
def events(self) -> Events:
|
|
957
|
+
"""The ``Events`` every protocol call this ``Context`` makes is reported to.
|
|
958
|
+
|
|
959
|
+
The same instance an ``events: Events`` constructor parameter
|
|
960
|
+
receives. Usable before ``open()``, so a listener subscribed first
|
|
961
|
+
also sees the startup calls ``open()`` makes. See
|
|
962
|
+
``opendataframework.events``.
|
|
963
|
+
|
|
964
|
+
Example:
|
|
965
|
+
>>> unsubscribe = context.events.subscribe(print, protocol=ServiceProtocol)
|
|
966
|
+
>>> context.open()
|
|
967
|
+
Event(key='service.postgres', protocol=<class '...ServiceProtocol'>, ...)
|
|
968
|
+
"""
|
|
969
|
+
return self._events
|
|
970
|
+
|
|
907
971
|
@property
|
|
908
972
|
def functions(self) -> dict[Callable[..., object], Callable[..., object]]:
|
|
909
973
|
"""Registered ``Function`` → bound callable mapping, populated by ``open()``.
|
|
@@ -988,9 +1052,9 @@ class Context:
|
|
|
988
1052
|
keys = self._resolver.keys
|
|
989
1053
|
self._instrument(keys)
|
|
990
1054
|
self._functions = self._resolver.functions
|
|
991
|
-
self._stream_manager = StreamManager(keys, self._log_manager)
|
|
1055
|
+
self._stream_manager = StreamManager(keys, self._log_manager, self._events)
|
|
992
1056
|
self._lifecycle = Lifecycle(
|
|
993
|
-
self._resolver.instances, self._resolver.order, keys, self._log_manager
|
|
1057
|
+
self._resolver.instances, self._resolver.order, keys, self._log_manager, self._events
|
|
994
1058
|
)
|
|
995
1059
|
self._lifecycle.on_start()
|
|
996
1060
|
|
|
@@ -1291,18 +1355,34 @@ class Context:
|
|
|
1291
1355
|
|
|
1292
1356
|
Logs start, completion (with duration), and any exception to that
|
|
1293
1357
|
component's own log file — with no code changes required in the
|
|
1294
|
-
classes themselves
|
|
1295
|
-
|
|
1296
|
-
|
|
1358
|
+
classes themselves — and reports each call to ``events`` as a
|
|
1359
|
+
``TaskProtocol``, ``PipelineProtocol`` or ``AgentProtocol`` call. The
|
|
1360
|
+
wrapper is set on the instance, so a call made directly (from a
|
|
1361
|
+
``Pipeline``, say) is logged the same as one made through the
|
|
1362
|
+
``Context``.
|
|
1297
1363
|
"""
|
|
1298
|
-
|
|
1364
|
+
tasks = set(dict(Task.items()).values())
|
|
1365
|
+
pipelines = set(dict(Pipeline.items()).values())
|
|
1299
1366
|
agents = set(dict(Agent.items()).values())
|
|
1300
1367
|
for cls, instance in self._resolver.instances.items():
|
|
1301
|
-
if cls in
|
|
1302
|
-
|
|
1368
|
+
if cls in tasks | pipelines and hasattr(instance, "execute"):
|
|
1369
|
+
protocol = PipelineProtocol if cls in pipelines else TaskProtocol
|
|
1370
|
+
instance.execute = _logged_execute(
|
|
1371
|
+
instance.execute,
|
|
1372
|
+
keys[cls],
|
|
1373
|
+
self._log_manager,
|
|
1374
|
+
self._events,
|
|
1375
|
+
protocol,
|
|
1376
|
+
"execute",
|
|
1377
|
+
)
|
|
1303
1378
|
if cls in agents:
|
|
1304
1379
|
instance.invoke = _logged_execute(
|
|
1305
|
-
instance.invoke,
|
|
1380
|
+
instance.invoke,
|
|
1381
|
+
keys[cls],
|
|
1382
|
+
self._log_manager,
|
|
1383
|
+
self._events,
|
|
1384
|
+
AgentProtocol,
|
|
1385
|
+
"invoke",
|
|
1306
1386
|
)
|
|
1307
1387
|
|
|
1308
1388
|
def __enter__(self) -> Context:
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Protocol events — an Observer over the protocol calls the framework drives.
|
|
2
|
+
|
|
3
|
+
Every time the ``Context`` calls a lifecycle or execution protocol method on
|
|
4
|
+
a component — a ``Service``'s ``setup()``/``run()``/``stop()``, an
|
|
5
|
+
``on_start()``/``on_stop()`` hook, a ``Task``'s or ``Pipeline``'s
|
|
6
|
+
``execute()``, an ``Agent``'s ``invoke()``, a ``@Function`` call, a
|
|
7
|
+
repository's ``open_stream()``/``stream()``/``close_stream()`` — it emits an
|
|
8
|
+
``Event`` when the call starts and when it completes or fails. Nothing else
|
|
9
|
+
emits one.
|
|
10
|
+
|
|
11
|
+
``Events`` is the subject, one per ``Context`` — no process-global state, so
|
|
12
|
+
multiple ``Project`` instances in the same process never share listeners
|
|
13
|
+
(see ``CLAUDE.md``'s "no global mutable state" constraint). Like ``Logger``,
|
|
14
|
+
it is injectable: a constructor parameter annotated ``events: Events``
|
|
15
|
+
receives the ``Context``'s own instance. Code holding the ``Context`` reaches
|
|
16
|
+
it as ``context.events``.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
from collections.abc import Callable, Iterator
|
|
22
|
+
from contextlib import contextmanager
|
|
23
|
+
from contextvars import ContextVar
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from uuid import uuid4
|
|
26
|
+
|
|
27
|
+
from opendataframework.logger import LogManager
|
|
28
|
+
|
|
29
|
+
# The ``call_id`` of the protocol call in progress in this thread (or task).
|
|
30
|
+
# A ``ContextVar`` holds a separate value per thread, so it is not shared state.
|
|
31
|
+
_current_call: ContextVar[str | None] = ContextVar("opendataframework_call", default=None)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class Event:
|
|
36
|
+
"""One state change of a protocol call the framework made.
|
|
37
|
+
|
|
38
|
+
Attributes:
|
|
39
|
+
key: The ``<namespace>.<name>`` key of the component called, as
|
|
40
|
+
``Context.key()`` returns it.
|
|
41
|
+
protocol: The protocol whose method was called — ``ServiceProtocol``,
|
|
42
|
+
``OnStartProtocol``, ``TaskProtocol``, ``FunctionProtocol``, ...
|
|
43
|
+
method: The method called — ``"setup"``, ``"on_start"``,
|
|
44
|
+
``"execute"``, ``"__call__"``, ``"stream"``, ...
|
|
45
|
+
state: ``"started"``, ``"completed"``, or ``"failed"``.
|
|
46
|
+
call_id: Identifies the call; shared by its ``started`` and its
|
|
47
|
+
``completed``/``failed`` event.
|
|
48
|
+
parent: ``call_id`` of the call in progress in this thread when this
|
|
49
|
+
one started — a pipeline for its steps, a task for a function it
|
|
50
|
+
calls — or ``None`` for a top-level call.
|
|
51
|
+
ts: ``time.time()`` when the event was emitted.
|
|
52
|
+
elapsed_ms: Duration of the call, on ``completed``/``failed`` only.
|
|
53
|
+
error: ``repr()`` of the exception raised, on ``failed`` only.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
key: str
|
|
57
|
+
protocol: type
|
|
58
|
+
method: str
|
|
59
|
+
state: str
|
|
60
|
+
call_id: str
|
|
61
|
+
parent: str | None
|
|
62
|
+
ts: float
|
|
63
|
+
elapsed_ms: float | None = None
|
|
64
|
+
error: str | None = None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Events:
|
|
68
|
+
"""Delivers every ``Event`` of a single ``Context`` to its listeners.
|
|
69
|
+
|
|
70
|
+
Not constructed by user code — declare ``events: Events`` in a
|
|
71
|
+
constructor to receive the ``Context``'s instance, or use
|
|
72
|
+
``context.events``. A listener is any callable taking an ``Event``; it
|
|
73
|
+
runs synchronously, in the thread that made the call, so it must be quick
|
|
74
|
+
and thread-safe — hand slow work off to a queue instead.
|
|
75
|
+
|
|
76
|
+
Args:
|
|
77
|
+
log_manager: Where a listener's exception is logged, as a ``WARNING``
|
|
78
|
+
on the event's component. Defaults to a disabled ``LogManager``.
|
|
79
|
+
|
|
80
|
+
Example:
|
|
81
|
+
>>> @Component
|
|
82
|
+
... class FailureAlerts:
|
|
83
|
+
... def __init__(self, events: Events) -> None:
|
|
84
|
+
... self.queue: Queue[Event | None] = Queue()
|
|
85
|
+
... self._unsubscribe = events.subscribe(self.queue.put)
|
|
86
|
+
...
|
|
87
|
+
... def on_stop(self) -> None:
|
|
88
|
+
... self._unsubscribe()
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
def __init__(self, log_manager: LogManager | None = None) -> None:
|
|
92
|
+
"""See class docstring for ``Args``."""
|
|
93
|
+
self._log_manager = log_manager if log_manager is not None else LogManager()
|
|
94
|
+
self._listeners: list[tuple[Callable[[Event], None], type | None]] = []
|
|
95
|
+
self._lock = threading.Lock()
|
|
96
|
+
|
|
97
|
+
def subscribe(
|
|
98
|
+
self, listener: Callable[[Event], None], protocol: type | None = None
|
|
99
|
+
) -> Callable[[], None]:
|
|
100
|
+
"""Call ``listener`` with every event, or only with calls of ``protocol``'s methods.
|
|
101
|
+
|
|
102
|
+
Args:
|
|
103
|
+
listener: Called with each matching ``Event``.
|
|
104
|
+
protocol: Deliver only events whose ``protocol`` is exactly this
|
|
105
|
+
one — ``TaskProtocol`` does not match a ``Pipeline``'s
|
|
106
|
+
``execute()``. ``None`` (the default) delivers every event.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
A function that unsubscribes ``listener``. Calling it more than
|
|
110
|
+
once is a no-op.
|
|
111
|
+
|
|
112
|
+
Example:
|
|
113
|
+
>>> unsubscribe = events.subscribe(on_service, protocol=ServiceProtocol)
|
|
114
|
+
>>> unsubscribe()
|
|
115
|
+
"""
|
|
116
|
+
entry = (listener, protocol)
|
|
117
|
+
with self._lock:
|
|
118
|
+
self._listeners.append(entry)
|
|
119
|
+
|
|
120
|
+
def unsubscribe() -> None:
|
|
121
|
+
"""Stop delivering events to ``listener``."""
|
|
122
|
+
with self._lock:
|
|
123
|
+
if entry in self._listeners:
|
|
124
|
+
self._listeners.remove(entry)
|
|
125
|
+
|
|
126
|
+
return unsubscribe
|
|
127
|
+
|
|
128
|
+
@contextmanager
|
|
129
|
+
def notify(self, key: str, protocol: type, method: str) -> Iterator[None]:
|
|
130
|
+
"""Emit ``started``, then ``completed`` or ``failed``, around one protocol call.
|
|
131
|
+
|
|
132
|
+
Used by the framework at each place it calls a protocol method; not
|
|
133
|
+
meant for user code. While the call runs, it is the current call, so
|
|
134
|
+
calls made inside it get it as their ``parent``. An exception from
|
|
135
|
+
the call is re-raised unchanged.
|
|
136
|
+
|
|
137
|
+
Example:
|
|
138
|
+
>>> with events.notify("service.postgres", ServiceProtocol, "setup"):
|
|
139
|
+
... instance.setup()
|
|
140
|
+
"""
|
|
141
|
+
call_id = uuid4().hex
|
|
142
|
+
parent = _current_call.get()
|
|
143
|
+
self._dispatch(Event(key, protocol, method, "started", call_id, parent, time.time()))
|
|
144
|
+
token = _current_call.set(call_id)
|
|
145
|
+
start = time.monotonic()
|
|
146
|
+
try:
|
|
147
|
+
yield
|
|
148
|
+
except Exception as exc:
|
|
149
|
+
elapsed_ms = (time.monotonic() - start) * 1000
|
|
150
|
+
failed = Event(
|
|
151
|
+
key, protocol, method, "failed", call_id, parent, time.time(), elapsed_ms, repr(exc)
|
|
152
|
+
)
|
|
153
|
+
self._dispatch(failed)
|
|
154
|
+
raise
|
|
155
|
+
else:
|
|
156
|
+
elapsed_ms = (time.monotonic() - start) * 1000
|
|
157
|
+
completed = Event(
|
|
158
|
+
key, protocol, method, "completed", call_id, parent, time.time(), elapsed_ms
|
|
159
|
+
)
|
|
160
|
+
self._dispatch(completed)
|
|
161
|
+
finally:
|
|
162
|
+
_current_call.reset(token)
|
|
163
|
+
|
|
164
|
+
def _dispatch(self, event: Event) -> None:
|
|
165
|
+
"""Call every listener subscribed to ``event``'s protocol, isolating their failures.
|
|
166
|
+
|
|
167
|
+
Listeners run with no current call, so a call one of them makes is a
|
|
168
|
+
top-level call rather than a child of the call that emitted
|
|
169
|
+
``event``. An exception from a listener is logged and never reaches
|
|
170
|
+
the call.
|
|
171
|
+
"""
|
|
172
|
+
with self._lock:
|
|
173
|
+
listeners = [
|
|
174
|
+
listener
|
|
175
|
+
for listener, protocol in self._listeners
|
|
176
|
+
if protocol is None or protocol is event.protocol
|
|
177
|
+
]
|
|
178
|
+
if not listeners:
|
|
179
|
+
return
|
|
180
|
+
token = _current_call.set(None)
|
|
181
|
+
try:
|
|
182
|
+
for listener in listeners:
|
|
183
|
+
try:
|
|
184
|
+
listener(event)
|
|
185
|
+
except Exception as exc:
|
|
186
|
+
self._log_manager.write(event.key, "WARNING", f"listener raised {exc!r}")
|
|
187
|
+
finally:
|
|
188
|
+
_current_call.reset(token)
|
|
@@ -10,6 +10,7 @@ to call it by name with its managed dependencies filled in.
|
|
|
10
10
|
import functools
|
|
11
11
|
import inspect
|
|
12
12
|
from collections.abc import Callable, ItemsView, Iterable
|
|
13
|
+
from typing import Protocol
|
|
13
14
|
|
|
14
15
|
from opendataframework.namespace import Namespace, describe
|
|
15
16
|
from opendataframework.utils import normalize
|
|
@@ -18,6 +19,28 @@ _POSITIONAL = (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_O
|
|
|
18
19
|
_VARIADIC = (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
|
|
19
20
|
|
|
20
21
|
|
|
22
|
+
class FunctionProtocol(Protocol):
|
|
23
|
+
"""Expected interface for functions registered with ``@Function``.
|
|
24
|
+
|
|
25
|
+
A function is called with arguments and returns a result — the
|
|
26
|
+
function-shaped counterpart of ``TaskProtocol``. It is a label for type
|
|
27
|
+
hints and for events, not a way to detect a function: any callable would
|
|
28
|
+
match it structurally, so it is deliberately not ``runtime_checkable``
|
|
29
|
+
and ``isinstance()`` against it raises ``TypeError``. The ``Context``
|
|
30
|
+
knows a function by its ``@Function`` registration instead.
|
|
31
|
+
|
|
32
|
+
Every call of a registered function made through the ``Context`` —
|
|
33
|
+
injected, ``context.call()``, or ``context.get(func)`` — emits events
|
|
34
|
+
with this protocol (see ``opendataframework.events``)::
|
|
35
|
+
|
|
36
|
+
events.subscribe(on_call, protocol=FunctionProtocol)
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __call__(self, *args, **kwargs) -> object:
|
|
40
|
+
"""Run the function with its arguments and return a result."""
|
|
41
|
+
...
|
|
42
|
+
|
|
43
|
+
|
|
21
44
|
class Function(Namespace):
|
|
22
45
|
"""Decorator that registers a plain function with the Context.
|
|
23
46
|
|
|
@@ -12,7 +12,9 @@ on the other end (e.g. the sibling ``odf`` package's UI).
|
|
|
12
12
|
|
|
13
13
|
Every variant points at its field(s) generically (``field``/``fields``),
|
|
14
14
|
never with a domain-specific attribute name — the meaning comes from which
|
|
15
|
-
``View`` subclass it is, not from the attribute name.
|
|
15
|
+
``View`` subclass it is, not from the attribute name. A variant may also
|
|
16
|
+
name a field by its role in the representation (``TimeseriesView.group``),
|
|
17
|
+
never by what it means in the domain (``run_id``).
|
|
16
18
|
|
|
17
19
|
A view says nothing about the content itself, though. A repository whose
|
|
18
20
|
view points at content (``ImageView``/``VideoView``/``AudioView``/
|
|
@@ -129,14 +131,23 @@ class LocationView:
|
|
|
129
131
|
|
|
130
132
|
@dataclass
|
|
131
133
|
class TimeseriesView:
|
|
132
|
-
"""A line chart — the entity's
|
|
133
|
-
|
|
134
|
+
"""A line chart — the entity's numeric fields plotted against an
|
|
135
|
+
ordering field, optionally one line per group.
|
|
134
136
|
|
|
135
137
|
Attributes:
|
|
136
|
-
field: The entity's
|
|
138
|
+
field: The entity's ordering field, plotted as the x axis — a
|
|
139
|
+
timestamp, or any field records are ordered by (e.g. a
|
|
140
|
+
training ``step``).
|
|
141
|
+
fields: The numeric fields to plot, in order. ``None`` means every
|
|
142
|
+
numeric entity field other than ``field`` and ``group``.
|
|
143
|
+
group: A field whose distinct values each get their own line (e.g.
|
|
144
|
+
``run_id``), so groups can be compared on one chart. ``None``
|
|
145
|
+
means all records form a single line.
|
|
137
146
|
"""
|
|
138
147
|
|
|
139
148
|
field: str
|
|
149
|
+
fields: tuple[str, ...] | None = None
|
|
150
|
+
group: str | None = None
|
|
140
151
|
|
|
141
152
|
|
|
142
153
|
DataView = (
|
|
@@ -206,7 +217,7 @@ class ReplayProtocol(Protocol):
|
|
|
206
217
|
``field()`` gets no such affordance — this only matters for
|
|
207
218
|
``LocationView``/``VideoView``/``AudioView``-backed repositories; a
|
|
208
219
|
``TimeseriesView`` repository is always replayable since it already
|
|
209
|
-
names its own
|
|
220
|
+
names its own ordering field via ``field``.
|
|
210
221
|
"""
|
|
211
222
|
|
|
212
223
|
def field(self) -> str:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|