opendataframework 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opendataframework-0.4.0 → opendataframework-0.5.0}/PKG-INFO +40 -4
- {opendataframework-0.4.0 → opendataframework-0.5.0}/README.md +39 -3
- {opendataframework-0.4.0 → opendataframework-0.5.0}/pyproject.toml +1 -1
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/__init__.py +3 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/context.py +118 -66
- opendataframework-0.5.0/src/opendataframework/events.py +360 -0
- opendataframework-0.5.0/src/opendataframework/logger.py +408 -0
- opendataframework-0.5.0/src/opendataframework/notifications.py +304 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/project.py +7 -4
- opendataframework-0.4.0/src/opendataframework/events.py +0 -188
- opendataframework-0.4.0/src/opendataframework/logger.py +0 -140
- {opendataframework-0.4.0 → opendataframework-0.5.0}/LICENSE +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/agent.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/component.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/config.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/entity.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/function.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/layer.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/namespace.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/pipeline.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/py.typed +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/repository.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/service.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/task.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/utils.py +0 -0
- {opendataframework-0.4.0 → opendataframework-0.5.0}/src/opendataframework/view.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opendataframework
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Lightweight dependency-injection framework for data applications.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -26,7 +26,7 @@ Lightweight dependency-injection framework for data applications.
|
|
|
26
26
|
This package provides structural abstractions —
|
|
27
27
|
`Entity`, `Repository`, `Component`, `Service`, `Task`, `Pipeline`,
|
|
28
28
|
`Agent`, `Function`, `Layer`, `Namespace`, `Context`, `Project`, `Config`,
|
|
29
|
-
`Logger`, `View` — with zero third-party dependencies. No UI, no CLI, no MCP server.
|
|
29
|
+
`Logger`, `Events`, `Notifications`, `View` — with zero third-party dependencies. No UI, no CLI, no MCP server.
|
|
30
30
|
|
|
31
31
|
> Need the CLI, UI, or MCP server? That's the
|
|
32
32
|
> [`odf`](https://opendataframework.github.io/odf/) package, which
|
|
@@ -307,7 +307,10 @@ class Postgres:
|
|
|
307
307
|
|
|
308
308
|
Per-component logging handle, injectable via constructor DI like any
|
|
309
309
|
other dependency — writes land in that component's own log file with no
|
|
310
|
-
name argument needed.
|
|
310
|
+
name argument needed. Logs are structured: one JSON record per line by
|
|
311
|
+
default, keyword arguments become fields, and every protocol call the
|
|
312
|
+
framework makes is logged with its call and parent ids, so a run can be
|
|
313
|
+
traced across files.
|
|
311
314
|
|
|
312
315
|
```python
|
|
313
316
|
@Task
|
|
@@ -317,7 +320,40 @@ class ExportUsers:
|
|
|
317
320
|
self.logger = logger
|
|
318
321
|
|
|
319
322
|
def execute(self) -> None:
|
|
320
|
-
self.logger.info("export starting")
|
|
323
|
+
self.logger.info("export starting", users=len(self.users.all()))
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
## Events
|
|
327
|
+
|
|
328
|
+
Reports every protocol call the framework makes — a `Service`'s `setup()`,
|
|
329
|
+
a `Task`'s `execute()`, a `@Function` call — as `started`, then `completed`
|
|
330
|
+
or `failed`, with timing and the `parent` call it ran inside.
|
|
331
|
+
|
|
332
|
+
```python
|
|
333
|
+
@Component
|
|
334
|
+
class FailureAlerts:
|
|
335
|
+
def __init__(self, events: Events):
|
|
336
|
+
self._unsubscribe = events.subscribe(self.on_event)
|
|
337
|
+
|
|
338
|
+
def on_event(self, event: Event) -> None:
|
|
339
|
+
if event.state == "failed":
|
|
340
|
+
...
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
## Notifications
|
|
344
|
+
|
|
345
|
+
Lets a component announce that something happened — a file arrived, an
|
|
346
|
+
order came in — so others can react without the publisher knowing who
|
|
347
|
+
they are. Never emitted by the framework itself.
|
|
348
|
+
|
|
349
|
+
```python
|
|
350
|
+
@Task
|
|
351
|
+
class TakeOrders:
|
|
352
|
+
def __init__(self, notifications: Notifications):
|
|
353
|
+
self.notifications = notifications
|
|
354
|
+
|
|
355
|
+
def execute(self) -> None:
|
|
356
|
+
self.notifications.publish("orders.received", {"id": 7}, summary="Le Petit: 2 x baguette")
|
|
321
357
|
```
|
|
322
358
|
|
|
323
359
|
## Context
|
|
@@ -10,7 +10,7 @@ Lightweight dependency-injection framework for data applications.
|
|
|
10
10
|
This package provides structural abstractions —
|
|
11
11
|
`Entity`, `Repository`, `Component`, `Service`, `Task`, `Pipeline`,
|
|
12
12
|
`Agent`, `Function`, `Layer`, `Namespace`, `Context`, `Project`, `Config`,
|
|
13
|
-
`Logger`, `View` — with zero third-party dependencies. No UI, no CLI, no MCP server.
|
|
13
|
+
`Logger`, `Events`, `Notifications`, `View` — with zero third-party dependencies. No UI, no CLI, no MCP server.
|
|
14
14
|
|
|
15
15
|
> Need the CLI, UI, or MCP server? That's the
|
|
16
16
|
> [`odf`](https://opendataframework.github.io/odf/) package, which
|
|
@@ -291,7 +291,10 @@ class Postgres:
|
|
|
291
291
|
|
|
292
292
|
Per-component logging handle, injectable via constructor DI like any
|
|
293
293
|
other dependency — writes land in that component's own log file with no
|
|
294
|
-
name argument needed.
|
|
294
|
+
name argument needed. Logs are structured: one JSON record per line by
|
|
295
|
+
default, keyword arguments become fields, and every protocol call the
|
|
296
|
+
framework makes is logged with its call and parent ids, so a run can be
|
|
297
|
+
traced across files.
|
|
295
298
|
|
|
296
299
|
```python
|
|
297
300
|
@Task
|
|
@@ -301,7 +304,40 @@ class ExportUsers:
|
|
|
301
304
|
self.logger = logger
|
|
302
305
|
|
|
303
306
|
def execute(self) -> None:
|
|
304
|
-
self.logger.info("export starting")
|
|
307
|
+
self.logger.info("export starting", users=len(self.users.all()))
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
## Events
|
|
311
|
+
|
|
312
|
+
Reports every protocol call the framework makes — a `Service`'s `setup()`,
|
|
313
|
+
a `Task`'s `execute()`, a `@Function` call — as `started`, then `completed`
|
|
314
|
+
or `failed`, with timing and the `parent` call it ran inside.
|
|
315
|
+
|
|
316
|
+
```python
|
|
317
|
+
@Component
|
|
318
|
+
class FailureAlerts:
|
|
319
|
+
def __init__(self, events: Events):
|
|
320
|
+
self._unsubscribe = events.subscribe(self.on_event)
|
|
321
|
+
|
|
322
|
+
def on_event(self, event: Event) -> None:
|
|
323
|
+
if event.state == "failed":
|
|
324
|
+
...
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
## Notifications
|
|
328
|
+
|
|
329
|
+
Lets a component announce that something happened — a file arrived, an
|
|
330
|
+
order came in — so others can react without the publisher knowing who
|
|
331
|
+
they are. Never emitted by the framework itself.
|
|
332
|
+
|
|
333
|
+
```python
|
|
334
|
+
@Task
|
|
335
|
+
class TakeOrders:
|
|
336
|
+
def __init__(self, notifications: Notifications):
|
|
337
|
+
self.notifications = notifications
|
|
338
|
+
|
|
339
|
+
def execute(self) -> None:
|
|
340
|
+
self.notifications.publish("orders.received", {"id": 7}, summary="Le Petit: 2 x baguette")
|
|
305
341
|
```
|
|
306
342
|
|
|
307
343
|
## Context
|
|
@@ -10,6 +10,7 @@ from opendataframework.function import Function, FunctionProtocol
|
|
|
10
10
|
from opendataframework.layer import Analytics, Api, Layer, Messaging, Monitoring, Security, Storage
|
|
11
11
|
from opendataframework.logger import Logger
|
|
12
12
|
from opendataframework.namespace import Namespace
|
|
13
|
+
from opendataframework.notifications import Notification, Notifications
|
|
13
14
|
from opendataframework.pipeline import Pipeline, PipelineProtocol
|
|
14
15
|
from opendataframework.project import Project
|
|
15
16
|
from opendataframework.repository import (
|
|
@@ -64,6 +65,8 @@ __all__ = [
|
|
|
64
65
|
"Messaging",
|
|
65
66
|
"Monitoring",
|
|
66
67
|
"Namespace",
|
|
68
|
+
"Notification",
|
|
69
|
+
"Notifications",
|
|
67
70
|
"Pipeline",
|
|
68
71
|
"PipelineProtocol",
|
|
69
72
|
"Project",
|
|
@@ -11,12 +11,12 @@ Every protocol call the three make is logged and also reported as an
|
|
|
11
11
|
``Event`` to the ``Context``'s ``Events`` (see ``opendataframework.events``).
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
+
import contextlib
|
|
14
15
|
import contextvars
|
|
15
16
|
import functools
|
|
16
17
|
import inspect
|
|
17
18
|
import threading
|
|
18
|
-
import
|
|
19
|
-
from collections.abc import Callable, Hashable, Iterable, Iterator
|
|
19
|
+
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
from queue import Empty, Full, Queue
|
|
22
22
|
from typing import Annotated, TypeAliasType, TypeVar, cast, get_origin, overload
|
|
@@ -24,10 +24,11 @@ from typing import Annotated, TypeAliasType, TypeVar, cast, get_origin, overload
|
|
|
24
24
|
from opendataframework.agent import Agent, AgentProtocol
|
|
25
25
|
from opendataframework.component import Component, OnStartProtocol, OnStopProtocol
|
|
26
26
|
from opendataframework.config import Config
|
|
27
|
-
from opendataframework.events import Events
|
|
27
|
+
from opendataframework.events import Events, build_trace
|
|
28
28
|
from opendataframework.function import Function, FunctionProtocol, bind, string_hint
|
|
29
29
|
from opendataframework.logger import Logger, LogManager
|
|
30
30
|
from opendataframework.namespace import Namespace, parse, qualified
|
|
31
|
+
from opendataframework.notifications import Notifications
|
|
31
32
|
from opendataframework.pipeline import Pipeline, PipelineProtocol
|
|
32
33
|
from opendataframework.repository import (
|
|
33
34
|
CloseStreamProtocol,
|
|
@@ -37,7 +38,7 @@ from opendataframework.repository import (
|
|
|
37
38
|
)
|
|
38
39
|
from opendataframework.service import Service, ServiceProtocol
|
|
39
40
|
from opendataframework.task import Task, TaskProtocol
|
|
40
|
-
from opendataframework.utils import normalize
|
|
41
|
+
from opendataframework.utils import kebab, normalize
|
|
41
42
|
|
|
42
43
|
_STREAM_ENDED = object()
|
|
43
44
|
|
|
@@ -58,31 +59,20 @@ def _is_async(func: object) -> bool:
|
|
|
58
59
|
return inspect.iscoroutinefunction(func) or inspect.isasyncgenfunction(func)
|
|
59
60
|
|
|
60
61
|
|
|
61
|
-
def
|
|
62
|
-
|
|
63
|
-
):
|
|
64
|
-
"""Wrap a bound ``execute`` method to log its start, duration, and errors.
|
|
62
|
+
def _notified_call(execute, name: str, events: Events, protocol: type, method: str):
|
|
63
|
+
"""Wrap a bound ``execute`` method so each call is reported to ``events``.
|
|
65
64
|
|
|
66
65
|
``protocol`` and ``method`` name the protocol call being wrapped, so the
|
|
67
66
|
same wrapper also serves an ``Agent``'s ``invoke()`` and a ``Function``
|
|
68
|
-
called through the ``Context``.
|
|
67
|
+
called through the ``Context``. ``events`` also writes the call's start,
|
|
68
|
+
duration, and any error to ``name``'s log.
|
|
69
69
|
"""
|
|
70
|
-
label = "call" if protocol is FunctionProtocol else f"{method}()"
|
|
71
70
|
|
|
72
71
|
@functools.wraps(execute)
|
|
73
72
|
def wrapped(*args, **kwargs):
|
|
74
|
-
"""Call the wrapped ``execute
|
|
73
|
+
"""Call the wrapped ``execute`` inside ``events.notify()``."""
|
|
75
74
|
with events.notify(name, protocol, method):
|
|
76
|
-
|
|
77
|
-
start = time.monotonic()
|
|
78
|
-
try:
|
|
79
|
-
result = execute(*args, **kwargs)
|
|
80
|
-
except Exception as exc:
|
|
81
|
-
log_manager.write(name, "ERROR", f"{label} raised {exc!r}")
|
|
82
|
-
raise
|
|
83
|
-
elapsed_ms = (time.monotonic() - start) * 1000
|
|
84
|
-
log_manager.write(name, "INFO", f"{label} completed in {elapsed_ms:.1f}ms")
|
|
85
|
-
return result
|
|
75
|
+
return execute(*args, **kwargs)
|
|
86
76
|
|
|
87
77
|
return wrapped
|
|
88
78
|
|
|
@@ -128,7 +118,10 @@ class Resolver:
|
|
|
128
118
|
instantiation is logged to that component's file — and ``Events``
|
|
129
119
|
(see ``opendataframework.events``): an ``events: Events`` parameter
|
|
130
120
|
receives the ``Context``'s own instance, and every call of a bound
|
|
131
|
-
function is reported to it as a ``FunctionProtocol`` call
|
|
121
|
+
function is reported to it as a ``FunctionProtocol`` call — and
|
|
122
|
+
``Notifications`` (see ``opendataframework.notifications``), resolved
|
|
123
|
+
contextually like ``Logger``: a ``notifications: Notifications``
|
|
124
|
+
parameter receives a view of the hub bound to the consuming class's key.
|
|
132
125
|
|
|
133
126
|
Args:
|
|
134
127
|
namespaces: Namespace subclasses to collect registered classes from.
|
|
@@ -136,6 +129,9 @@ class Resolver:
|
|
|
136
129
|
``LogManager`` (writes become no-ops) when omitted.
|
|
137
130
|
events: Receives an event for every call of a bound function.
|
|
138
131
|
Defaults to an ``Events`` with no listeners when omitted.
|
|
132
|
+
notifications: The hub a ``notifications: Notifications`` parameter
|
|
133
|
+
gets a bound view of. Defaults to a ``Notifications`` with no
|
|
134
|
+
subscribers when omitted.
|
|
139
135
|
|
|
140
136
|
Example:
|
|
141
137
|
>>> class NS(Namespace): ...
|
|
@@ -158,6 +154,7 @@ class Resolver:
|
|
|
158
154
|
extra_instances: dict[type, object] | None = None,
|
|
159
155
|
log_manager: LogManager | None = None,
|
|
160
156
|
events: Events | None = None,
|
|
157
|
+
notifications: Notifications | None = None,
|
|
161
158
|
) -> None:
|
|
162
159
|
"""See class docstring for ``Args``."""
|
|
163
160
|
self._namespaces = set(namespaces)
|
|
@@ -169,6 +166,18 @@ class Resolver:
|
|
|
169
166
|
self._by_key: dict[str, type | Callable[..., object]] = {}
|
|
170
167
|
self._log_manager = log_manager if log_manager is not None else LogManager()
|
|
171
168
|
self._events = events if events is not None else Events(self._log_manager)
|
|
169
|
+
self._notifications = (
|
|
170
|
+
notifications if notifications is not None else Notifications(self._log_manager)
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def namespaces(self) -> set[type[Namespace]]:
|
|
175
|
+
"""The namespaces registered classes and functions are collected from.
|
|
176
|
+
|
|
177
|
+
Returns:
|
|
178
|
+
A copy of the set.
|
|
179
|
+
"""
|
|
180
|
+
return set(self._namespaces)
|
|
172
181
|
|
|
173
182
|
@property
|
|
174
183
|
def instances(self) -> dict[type, object]:
|
|
@@ -265,10 +274,9 @@ class Resolver:
|
|
|
265
274
|
self._instances[node] = self._instantiate(node)
|
|
266
275
|
else:
|
|
267
276
|
self._injected[node] = self._inject(node)
|
|
268
|
-
self._functions[node] =
|
|
277
|
+
self._functions[node] = _notified_call(
|
|
269
278
|
bind(node, self._injected[node]),
|
|
270
279
|
self._keys[node],
|
|
271
|
-
self._log_manager,
|
|
272
280
|
self._events,
|
|
273
281
|
FunctionProtocol,
|
|
274
282
|
"__call__",
|
|
@@ -488,7 +496,9 @@ class Resolver:
|
|
|
488
496
|
A ``dep`` annotated exactly as ``Logger`` is special-cased: instead of
|
|
489
497
|
a shared instance, ``target`` receives a ``Logger`` bound to its own
|
|
490
498
|
key (see ``opendataframework.logger.LogManager.logger_for``). One
|
|
491
|
-
annotated exactly as ``Events`` receives the ``Context``'s ``Events
|
|
499
|
+
annotated exactly as ``Events`` receives the ``Context``'s ``Events``,
|
|
500
|
+
and one annotated exactly as ``Notifications`` a view of the hub bound
|
|
501
|
+
to ``target``'s key (see ``Notifications.bound_to``). A class is
|
|
492
502
|
supplied only if already present in ``self._instances``, and a
|
|
493
503
|
function only if already bound — unregistered ones in the signature
|
|
494
504
|
are silently ignored.
|
|
@@ -499,6 +509,8 @@ class Resolver:
|
|
|
499
509
|
supplied[name] = self._log_manager.logger_for(self._keys[target])
|
|
500
510
|
elif dep is Events:
|
|
501
511
|
supplied[name] = self._events
|
|
512
|
+
elif dep is Notifications:
|
|
513
|
+
supplied[name] = self._notifications.bound_to(self._keys[target])
|
|
502
514
|
elif dep in self._instances:
|
|
503
515
|
supplied[name] = self._instances[dep]
|
|
504
516
|
elif dep in self._functions:
|
|
@@ -578,9 +590,7 @@ class Lifecycle:
|
|
|
578
590
|
"""Call a non-``Service`` instance's ``on_start()`` hook."""
|
|
579
591
|
name = self._keys[cls]
|
|
580
592
|
with self._events.notify(name, OnStartProtocol, "on_start"):
|
|
581
|
-
self._log_manager.write(name, "INFO", "on_start() called")
|
|
582
593
|
instance.on_start()
|
|
583
|
-
self._log_manager.write(name, "INFO", "on_start() complete")
|
|
584
594
|
|
|
585
595
|
def start_one(self, cls: type) -> None:
|
|
586
596
|
"""Start a single ``ServiceProtocol`` instance. No-op if already running.
|
|
@@ -598,9 +608,7 @@ class Lifecycle:
|
|
|
598
608
|
instance = self._instances[cls]
|
|
599
609
|
name = self._keys[cls]
|
|
600
610
|
with self._events.notify(name, ServiceProtocol, "setup"):
|
|
601
|
-
self._log_manager.write(name, "INFO", "setup() starting")
|
|
602
611
|
instance.setup()
|
|
603
|
-
self._log_manager.write(name, "INFO", "setup() complete")
|
|
604
612
|
# An empty context, so run() is a top-level call rather than a child
|
|
605
613
|
# of whatever call started the service.
|
|
606
614
|
t = threading.Thread(
|
|
@@ -626,9 +634,7 @@ class Lifecycle:
|
|
|
626
634
|
instance = self._instances[cls]
|
|
627
635
|
name = self._keys[cls]
|
|
628
636
|
with self._events.notify(name, ServiceProtocol, "stop"):
|
|
629
|
-
self._log_manager.write(name, "INFO", "stop() called")
|
|
630
637
|
instance.stop()
|
|
631
|
-
self._log_manager.write(name, "INFO", "stop() complete")
|
|
632
638
|
self._running[cls] = False
|
|
633
639
|
|
|
634
640
|
def is_running(self, cls: type) -> bool:
|
|
@@ -636,16 +642,9 @@ class Lifecycle:
|
|
|
636
642
|
return self._running.get(cls, False)
|
|
637
643
|
|
|
638
644
|
def _run(self, cls: type, instance: ServiceProtocol) -> None:
|
|
639
|
-
"""Run ``instance.run()
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
self._log_manager.write(name, "INFO", "run() started")
|
|
643
|
-
try:
|
|
644
|
-
instance.run()
|
|
645
|
-
self._log_manager.write(name, "INFO", "run() returned")
|
|
646
|
-
except Exception as exc:
|
|
647
|
-
self._log_manager.write(name, "ERROR", f"run() raised {exc!r}")
|
|
648
|
-
raise
|
|
645
|
+
"""Run ``instance.run()``; ``events`` logs its start, return, and any exception."""
|
|
646
|
+
with self._events.notify(self._keys[cls], ServiceProtocol, "run"):
|
|
647
|
+
instance.run()
|
|
649
648
|
|
|
650
649
|
def on_stop(self) -> None:
|
|
651
650
|
"""Deactivate all instances in reverse dependency order.
|
|
@@ -672,9 +671,7 @@ class Lifecycle:
|
|
|
672
671
|
"""Call a non-``Service`` instance's ``on_stop()`` hook."""
|
|
673
672
|
name = self._keys[cls]
|
|
674
673
|
with self._events.notify(name, OnStopProtocol, "on_stop"):
|
|
675
|
-
self._log_manager.write(name, "INFO", "on_stop() called")
|
|
676
674
|
instance.on_stop()
|
|
677
|
-
self._log_manager.write(name, "INFO", "on_stop() complete")
|
|
678
675
|
|
|
679
676
|
|
|
680
677
|
def _publish_latest(q: Queue, item: object) -> None:
|
|
@@ -765,9 +762,7 @@ class StreamManager:
|
|
|
765
762
|
try:
|
|
766
763
|
if isinstance(instance, OpenStreamProtocol):
|
|
767
764
|
with self._events.notify(name, OpenStreamProtocol, "open_stream"):
|
|
768
|
-
self._log_manager.write(name, "INFO", "open_stream() called")
|
|
769
765
|
instance.open_stream()
|
|
770
|
-
self._log_manager.write(name, "INFO", "open_stream() complete")
|
|
771
766
|
with self._lock:
|
|
772
767
|
self._subscribers[cls] = []
|
|
773
768
|
event = threading.Event()
|
|
@@ -833,28 +828,28 @@ class StreamManager:
|
|
|
833
828
|
"""Drive ``instance.stream()``, fanning entities out to subscribers until done."""
|
|
834
829
|
name = self._keys[cls]
|
|
835
830
|
try:
|
|
836
|
-
|
|
831
|
+
# A raising stream() is logged by events as a failed call.
|
|
832
|
+
with (
|
|
833
|
+
contextlib.suppress(Exception),
|
|
834
|
+
self._events.notify(name, StreamableProtocol, "stream"),
|
|
835
|
+
):
|
|
837
836
|
for entity in instance.stream():
|
|
838
837
|
if stop_event.is_set():
|
|
839
838
|
break
|
|
840
839
|
for q in list(self._subscribers.get(cls, ())):
|
|
841
840
|
_publish_latest(q, entity)
|
|
842
|
-
except Exception as exc:
|
|
843
|
-
self._log_manager.write(name, "ERROR", f"stream raised {exc!r}")
|
|
844
841
|
finally:
|
|
845
842
|
# Runs whether the loop above ended via stop_event, the source
|
|
846
843
|
# exhausting itself (e.g. a device disconnecting), or an
|
|
847
844
|
# exception — close_stream() releases the resource in all three
|
|
848
845
|
# cases, not just an explicit stop_stream() call.
|
|
849
846
|
if isinstance(instance, CloseStreamProtocol):
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
else:
|
|
857
|
-
self._log_manager.write(name, "INFO", "close_stream() complete")
|
|
847
|
+
# A raising close_stream() is logged by events as a failed call.
|
|
848
|
+
with (
|
|
849
|
+
contextlib.suppress(Exception),
|
|
850
|
+
self._events.notify(name, CloseStreamProtocol, "close_stream"),
|
|
851
|
+
):
|
|
852
|
+
instance.close_stream()
|
|
858
853
|
with self._lock:
|
|
859
854
|
subscribers = self._subscribers.pop(cls, [])
|
|
860
855
|
self._threads.pop(cls, None)
|
|
@@ -879,11 +874,15 @@ class Context:
|
|
|
879
874
|
Defaults to all of the built-in ones listed above.
|
|
880
875
|
config: Configuration dict, pre-seeded as ``Config`` into the
|
|
881
876
|
container.
|
|
882
|
-
log_dir: Directory to write per-component ``<namespace>.<name>.
|
|
883
|
-
files under.
|
|
877
|
+
log_dir: Directory to write per-component ``<namespace>.<name>.jsonl``
|
|
878
|
+
files (``.log`` in the ``text`` format) under.
|
|
884
879
|
``None`` (the default) disables file logging entirely — see
|
|
885
880
|
``opendataframework.logger.LogManager``. ``Project.from_config``/``from_dict``
|
|
886
881
|
opt in to a real directory.
|
|
882
|
+
log_config: Log format and level thresholds, shaped like
|
|
883
|
+
``[project.logging]`` — see ``opendataframework.logger.LogManager``.
|
|
884
|
+
``None`` writes JSON Lines and keeps ``INFO`` and up. Names of
|
|
885
|
+
namespaces and components in it are checked by ``open()``.
|
|
887
886
|
|
|
888
887
|
Attributes:
|
|
889
888
|
instances: Class-to-instance mapping, populated after ``open()`` returns.
|
|
@@ -927,16 +926,19 @@ class Context:
|
|
|
927
926
|
namespaces: Iterable[type[Namespace]] | None = None,
|
|
928
927
|
config: dict | None = None,
|
|
929
928
|
log_dir: str | Path | None = None,
|
|
929
|
+
log_config: Mapping[str, object] | None = None,
|
|
930
930
|
) -> None:
|
|
931
931
|
"""See class docstring for ``Args``."""
|
|
932
932
|
extra = {Config: Config(config)} if config is not None else {}
|
|
933
|
-
self._log_manager = LogManager(log_dir)
|
|
933
|
+
self._log_manager = LogManager(log_dir, log_config)
|
|
934
934
|
self._events = Events(self._log_manager)
|
|
935
|
+
self._notifications = Notifications(self._log_manager, key_of=self.key)
|
|
935
936
|
self._resolver = Resolver(
|
|
936
937
|
namespaces if namespaces is not None else _DEFAULT_NAMESPACES,
|
|
937
938
|
extra_instances=extra,
|
|
938
939
|
log_manager=self._log_manager,
|
|
939
940
|
events=self._events,
|
|
941
|
+
notifications=self._notifications,
|
|
940
942
|
)
|
|
941
943
|
self._lifecycle: Lifecycle | None = None
|
|
942
944
|
self._stream_manager = StreamManager(log_manager=self._log_manager, events=self._events)
|
|
@@ -968,6 +970,22 @@ class Context:
|
|
|
968
970
|
"""
|
|
969
971
|
return self._events
|
|
970
972
|
|
|
973
|
+
@property
|
|
974
|
+
def notifications(self) -> Notifications:
|
|
975
|
+
"""The hub every notification published in this ``Context`` goes through.
|
|
976
|
+
|
|
977
|
+
Unbound, unlike the view a ``notifications: Notifications`` parameter
|
|
978
|
+
receives, so ``publish()`` here needs ``key=``. Usable before
|
|
979
|
+
``open()``, so a subscriber added first also sees notifications
|
|
980
|
+
published during startup. See ``opendataframework.notifications``.
|
|
981
|
+
|
|
982
|
+
Example:
|
|
983
|
+
>>> unsubscribe = context.notifications.subscribe(print, topic="orders.*")
|
|
984
|
+
>>> context.execute("task.take-order", customer="Le Petit", item="baguette")
|
|
985
|
+
Notification(key='task.take-order', topic='orders.received', ...)
|
|
986
|
+
"""
|
|
987
|
+
return self._notifications
|
|
988
|
+
|
|
971
989
|
@property
|
|
972
990
|
def functions(self) -> dict[Callable[..., object], Callable[..., object]]:
|
|
973
991
|
"""Registered ``Function`` → bound callable mapping, populated by ``open()``.
|
|
@@ -1041,15 +1059,20 @@ class Context:
|
|
|
1041
1059
|
``Lifecycle`` from the result and calls ``Lifecycle.on_start()``.
|
|
1042
1060
|
|
|
1043
1061
|
Raises:
|
|
1044
|
-
ValueError: If a circular dependency is detected,
|
|
1062
|
+
ValueError: If a circular dependency is detected, a gathered
|
|
1045
1063
|
object would not have exactly one key (see
|
|
1046
|
-
``Resolver._index``)
|
|
1064
|
+
``Resolver._index``), or ``log_config`` names a namespace or
|
|
1065
|
+
component that does not exist (see ``LogManager.validate``).
|
|
1047
1066
|
TypeError: If a registered function, or a method the ``Context``
|
|
1048
1067
|
calls, is async — see ``_require_sync``.
|
|
1049
1068
|
"""
|
|
1050
1069
|
self._require_sync()
|
|
1051
1070
|
self._resolver.resolve()
|
|
1052
1071
|
keys = self._resolver.keys
|
|
1072
|
+
self._log_manager.validate(
|
|
1073
|
+
(kebab(namespace.__name__) for namespace in self._resolver.namespaces),
|
|
1074
|
+
keys.values(),
|
|
1075
|
+
)
|
|
1053
1076
|
self._instrument(keys)
|
|
1054
1077
|
self._functions = self._resolver.functions
|
|
1055
1078
|
self._stream_manager = StreamManager(keys, self._log_manager, self._events)
|
|
@@ -1291,6 +1314,35 @@ class Context:
|
|
|
1291
1314
|
raise KeyError(key)
|
|
1292
1315
|
return cls, instances[cls]
|
|
1293
1316
|
|
|
1317
|
+
def trace(self, call_id: str) -> dict:
|
|
1318
|
+
"""Return the tree of a protocol call and every call made inside it, from the logs.
|
|
1319
|
+
|
|
1320
|
+
Reads every ``.jsonl`` log file once and follows ``parent`` links,
|
|
1321
|
+
so it works after the fact — after the call ended, or the process
|
|
1322
|
+
restarted — as long as the records were kept. Which files it reads
|
|
1323
|
+
follows the configured log format, never what is on disk: ``.log``
|
|
1324
|
+
files left from an earlier ``text`` setup are ignored. Records dropped by a level
|
|
1325
|
+
threshold can't appear: with ``[project.logging] events =
|
|
1326
|
+
"WARNING"``, only failed calls do. See ``events.build_trace`` for
|
|
1327
|
+
the shape of the tree, and to build one from live events instead.
|
|
1328
|
+
|
|
1329
|
+
Args:
|
|
1330
|
+
call_id: The call's id — ``Event.call_id``, or a record's
|
|
1331
|
+
``call``.
|
|
1332
|
+
|
|
1333
|
+
Raises:
|
|
1334
|
+
KeyError: If no call in the ``.jsonl`` logs has the id ``call_id``.
|
|
1335
|
+
ValueError: If file logging is off, or the configured log format
|
|
1336
|
+
is ``text``.
|
|
1337
|
+
|
|
1338
|
+
Example:
|
|
1339
|
+
>>> logged = context.tail_logs("task.ship-orders")
|
|
1340
|
+
>>> failed = [r for r in logged if r.get("state") == "failed"]
|
|
1341
|
+
>>> context.trace(failed[-1]["parent"])["key"]
|
|
1342
|
+
'pipeline.nightly-orders'
|
|
1343
|
+
"""
|
|
1344
|
+
return build_trace(self._log_manager.records(), call_id)
|
|
1345
|
+
|
|
1294
1346
|
def tail_logs(self, key: str, lines: int = 300) -> list[dict]:
|
|
1295
1347
|
"""Return the last ``lines`` log entries for a resolved class or function.
|
|
1296
1348
|
|
|
@@ -1300,7 +1352,9 @@ class Context:
|
|
|
1300
1352
|
lines: Maximum number of most-recent entries to return.
|
|
1301
1353
|
|
|
1302
1354
|
Returns:
|
|
1303
|
-
|
|
1355
|
+
Log records, oldest first, each with at least ``"ts"``,
|
|
1356
|
+
``"level"`` and ``"message"``; in the default ``json`` format,
|
|
1357
|
+
the full record (``key``, ``source``, ``call``, ``fields``, ...).
|
|
1304
1358
|
Empty if file logging is disabled or nothing was logged.
|
|
1305
1359
|
|
|
1306
1360
|
Raises:
|
|
@@ -1367,19 +1421,17 @@ class Context:
|
|
|
1367
1421
|
for cls, instance in self._resolver.instances.items():
|
|
1368
1422
|
if cls in tasks | pipelines and hasattr(instance, "execute"):
|
|
1369
1423
|
protocol = PipelineProtocol if cls in pipelines else TaskProtocol
|
|
1370
|
-
instance.execute =
|
|
1424
|
+
instance.execute = _notified_call(
|
|
1371
1425
|
instance.execute,
|
|
1372
1426
|
keys[cls],
|
|
1373
|
-
self._log_manager,
|
|
1374
1427
|
self._events,
|
|
1375
1428
|
protocol,
|
|
1376
1429
|
"execute",
|
|
1377
1430
|
)
|
|
1378
1431
|
if cls in agents:
|
|
1379
|
-
instance.invoke =
|
|
1432
|
+
instance.invoke = _notified_call(
|
|
1380
1433
|
instance.invoke,
|
|
1381
1434
|
keys[cls],
|
|
1382
|
-
self._log_manager,
|
|
1383
1435
|
self._events,
|
|
1384
1436
|
AgentProtocol,
|
|
1385
1437
|
"invoke",
|