sibi-dst 2025.8.8__py3-none-any.whl → 2025.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sibi_dst/df_helper/_df_helper.py +379 -1
- sibi_dst/df_helper/backends/parquet/_parquet_options.py +2 -0
- sibi_dst/utils/base.py +567 -122
- sibi_dst/utils/boilerplate/__init__.py +9 -4
- sibi_dst/utils/boilerplate/base_attacher.py +25 -0
- sibi_dst/utils/boilerplate/{base_data_artifact.py → base_parquet_artifact.py} +1 -1
- sibi_dst/utils/boilerplate/base_parquet_reader.py +21 -0
- sibi_dst/utils/log_utils.py +108 -183
- sibi_dst/utils/progress/sse_runner.py +2 -0
- {sibi_dst-2025.8.8.dist-info → sibi_dst-2025.9.1.dist-info}/METADATA +2 -1
- {sibi_dst-2025.8.8.dist-info → sibi_dst-2025.9.1.dist-info}/RECORD +12 -10
- {sibi_dst-2025.8.8.dist-info → sibi_dst-2025.9.1.dist-info}/WHEEL +0 -0
sibi_dst/utils/base.py
CHANGED
@@ -1,20 +1,72 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
1
3
|
import abc
|
4
|
+
import asyncio
|
5
|
+
import contextlib
|
6
|
+
import json
|
2
7
|
import threading
|
3
8
|
import weakref
|
4
|
-
from typing import
|
9
|
+
from typing import Any, Awaitable, Callable, Dict, Optional, Self, final
|
5
10
|
|
6
11
|
import fsspec
|
7
|
-
|
8
12
|
from sibi_dst.utils import Logger
|
9
13
|
|
10
14
|
|
15
|
+
# --------- Minimal built-in SSE sink (used when auto_sse=True) ----------
|
16
|
+
class _QueueSSE:
|
17
|
+
"""Async queue–backed SSE sink: async send/put, async iterator, graceful close."""
|
18
|
+
__slots__ = ("q", "_closed")
|
19
|
+
|
20
|
+
def __init__(self) -> None:
|
21
|
+
self.q: asyncio.Queue = asyncio.Queue()
|
22
|
+
self._closed = False
|
23
|
+
|
24
|
+
async def send(self, event: str, data: Dict[str, Any]) -> None:
|
25
|
+
await self.q.put({"event": event, "data": json.dumps(data)})
|
26
|
+
|
27
|
+
async def put(self, item: Dict[str, Any]) -> None:
|
28
|
+
await self.q.put(item)
|
29
|
+
|
30
|
+
async def aclose(self) -> None:
|
31
|
+
if not self._closed:
|
32
|
+
self._closed = True
|
33
|
+
await self.q.put({"event": "__close__", "data": ""})
|
34
|
+
|
35
|
+
def close(self) -> None:
|
36
|
+
pass
|
37
|
+
|
38
|
+
async def __aiter__(self):
|
39
|
+
while True:
|
40
|
+
item = await self.q.get()
|
41
|
+
if item.get("event") == "__close__":
|
42
|
+
break
|
43
|
+
yield item
|
44
|
+
|
45
|
+
|
46
|
+
# ------------------------------ Base class ------------------------------
|
11
47
|
class ManagedResource(abc.ABC):
|
12
48
|
"""
|
13
|
-
|
14
|
-
|
15
|
-
|
49
|
+
Owns a logger and optional fsspec filesystem. Can emit SSE events via:
|
50
|
+
- an async emitter callable: await emitter(event, data)
|
51
|
+
- a sink exposing async send(event, data) or async put(item)
|
52
|
+
If neither is provided and auto_sse=True, a queue-backed sink is created eagerly.
|
53
|
+
Thread-safe lifecycle, CM support, GC finalizer.
|
16
54
|
"""
|
17
55
|
|
56
|
+
__slots__ = (
|
57
|
+
# config
|
58
|
+
"verbose", "debug", "_log_cleanup_errors",
|
59
|
+
# logger
|
60
|
+
"logger", "_owns_logger",
|
61
|
+
# fs
|
62
|
+
"fs", "_fs_factory", "_owns_fs",
|
63
|
+
# sse
|
64
|
+
"_sse", "_sse_factory", "_owns_sse",
|
65
|
+
"_emitter", "_auto_sse",
|
66
|
+
# lifecycle
|
67
|
+
"_is_closed", "_closing", "_close_lock", "_finalizer"
|
68
|
+
)
|
69
|
+
|
18
70
|
def __init__(
|
19
71
|
self,
|
20
72
|
*,
|
@@ -22,25 +74,28 @@ class ManagedResource(abc.ABC):
|
|
22
74
|
debug: bool = False,
|
23
75
|
log_cleanup_errors: bool = True,
|
24
76
|
logger: Optional[Logger] = None,
|
77
|
+
# filesystem
|
25
78
|
fs: Optional[fsspec.AbstractFileSystem] = None,
|
26
79
|
fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None,
|
80
|
+
# SSE
|
81
|
+
emitter: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
|
82
|
+
emitter_factory: Optional[Callable[[], Callable[[str, Dict[str, Any]], Awaitable[None]]]] = None,
|
83
|
+
sse: Optional[object] = None,
|
84
|
+
sse_factory: Optional[Callable[[], object]] = None,
|
85
|
+
auto_sse: bool = False, # eager auto-create if no emitter/sink is supplied
|
27
86
|
**_: object,
|
28
87
|
) -> None:
|
29
|
-
#
|
30
|
-
self.logger: Logger
|
31
|
-
self.fs: Optional[fsspec.AbstractFileSystem] = None
|
32
|
-
self._fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None
|
33
|
-
self._owns_logger: bool = False
|
34
|
-
self._owns_fs: bool = False
|
35
|
-
self._is_closed: bool = False
|
36
|
-
self._closing: bool = False
|
37
|
-
self._close_lock = threading.RLock()
|
38
|
-
|
88
|
+
# flags
|
39
89
|
self.verbose = verbose
|
40
90
|
self.debug = debug
|
41
91
|
self._log_cleanup_errors = log_cleanup_errors
|
42
92
|
|
43
|
-
#
|
93
|
+
# lifecycle
|
94
|
+
self._is_closed = False
|
95
|
+
self._closing = False
|
96
|
+
self._close_lock = threading.RLock()
|
97
|
+
|
98
|
+
# logger
|
44
99
|
if logger is None:
|
45
100
|
self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
|
46
101
|
self._owns_logger = True
|
@@ -48,205 +103,595 @@ class ManagedResource(abc.ABC):
|
|
48
103
|
self.logger.set_level(level)
|
49
104
|
else:
|
50
105
|
self.logger = logger
|
51
|
-
self._owns_logger = False
|
106
|
+
self._owns_logger = False
|
52
107
|
|
53
|
-
#
|
108
|
+
# fs
|
109
|
+
self.fs: Optional[fsspec.AbstractFileSystem] = None
|
110
|
+
self._fs_factory = None
|
111
|
+
self._owns_fs = False
|
54
112
|
if fs is not None:
|
113
|
+
if not isinstance(fs, fsspec.AbstractFileSystem):
|
114
|
+
raise TypeError(f"fs must be an fsspec.AbstractFileSystem, got {type(fs)!r}")
|
55
115
|
self.fs = fs
|
56
|
-
self._owns_fs = False
|
57
|
-
self._fs_factory = None
|
58
116
|
elif fs_factory is not None:
|
59
|
-
|
117
|
+
if not callable(fs_factory):
|
118
|
+
raise TypeError("fs_factory must be callable")
|
60
119
|
self._fs_factory = fs_factory
|
61
|
-
self._owns_fs = True
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
120
|
+
self._owns_fs = True
|
121
|
+
|
122
|
+
# sse / emitter
|
123
|
+
self._sse: Optional[object] = None
|
124
|
+
self._sse_factory: Optional[Callable[[], object]] = None
|
125
|
+
self._owns_sse = False
|
126
|
+
self._auto_sse = auto_sse
|
67
127
|
|
68
|
-
|
69
|
-
|
70
|
-
|
128
|
+
self._emitter: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None
|
129
|
+
if emitter is not None:
|
130
|
+
self._emitter = emitter
|
131
|
+
elif emitter_factory is not None:
|
132
|
+
self._emitter = emitter_factory()
|
133
|
+
|
134
|
+
if sse is not None:
|
135
|
+
self._sse = sse
|
136
|
+
self._emitter = self._emitter or self._build_emitter(sse)
|
137
|
+
elif sse_factory is not None:
|
138
|
+
if not callable(sse_factory):
|
139
|
+
raise TypeError("sse_factory must be callable")
|
140
|
+
self._sse_factory = sse_factory
|
141
|
+
self._owns_sse = True
|
142
|
+
|
143
|
+
# EAGER auto-SSE: create sink+emitter now if none supplied
|
144
|
+
if self._auto_sse and self._sse is None and self._emitter is None and self._sse_factory is None:
|
145
|
+
self._create_auto_sse()
|
146
|
+
|
147
|
+
# GC finalizer
|
148
|
+
self._finalizer = weakref.finalize(self, self._finalize_static, weakref.ref(self))
|
71
149
|
|
72
150
|
if self.debug:
|
73
|
-
|
74
|
-
self.logger.debug("
|
75
|
-
except Exception:
|
76
|
-
pass
|
151
|
+
with contextlib.suppress(Exception):
|
152
|
+
self.logger.debug("Initialized %s %s", self.__class__.__name__, repr(self))
|
77
153
|
|
78
154
|
# ---------- Introspection ----------
|
79
155
|
@property
|
80
|
-
def
|
156
|
+
def closed(self) -> bool:
|
81
157
|
return self._is_closed
|
82
158
|
|
83
159
|
@property
|
84
|
-
def
|
85
|
-
return self.
|
160
|
+
def has_fs(self) -> bool:
|
161
|
+
return self.fs is not None or self._fs_factory is not None
|
162
|
+
|
163
|
+
@property
|
164
|
+
def has_sse(self) -> bool:
|
165
|
+
return (self._emitter is not None) or (self._sse is not None)
|
86
166
|
|
87
167
|
def __repr__(self) -> str:
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
168
|
+
def _status(current: bool, factory: bool, owned: bool) -> str:
|
169
|
+
if current:
|
170
|
+
return "own" if owned else "external"
|
171
|
+
if factory:
|
172
|
+
return "own(lazy)"
|
173
|
+
return "none"
|
174
|
+
|
175
|
+
fs_status = _status(self.fs is not None, self._fs_factory is not None, self._owns_fs)
|
176
|
+
sse_status = _status(self._sse is not None or self._emitter is not None,
|
177
|
+
self._sse_factory is not None or self._auto_sse, self._owns_sse)
|
178
|
+
return (f"<{self.__class__.__name__} debug={self.debug} verbose={self.verbose} "
|
179
|
+
f"log_cleanup_errors={self._log_cleanup_errors} fs={fs_status} sse={sse_status}>")
|
99
180
|
|
100
181
|
# ---------- Subclass hooks ----------
|
101
182
|
def _cleanup(self) -> None:
|
102
|
-
"""Sync cleanup for resources created BY THE SUBCLASS."""
|
103
183
|
return
|
104
184
|
|
105
185
|
async def _acleanup(self) -> None:
|
106
|
-
"""Async cleanup for resources created BY THE SUBCLASS."""
|
107
186
|
return
|
108
187
|
|
109
|
-
# ----------
|
188
|
+
# ---------- Guards ----------
|
189
|
+
def _assert_open(self) -> None:
|
190
|
+
if self._is_closed or self._closing:
|
191
|
+
raise RuntimeError(f"{self.__class__.__name__} is closed")
|
192
|
+
|
193
|
+
# ---------- FS ----------
|
194
|
+
def set_fs_factory(self, factory: Optional[Callable[[], fsspec.AbstractFileSystem]]) -> None:
|
195
|
+
with self._close_lock:
|
196
|
+
self._assert_open()
|
197
|
+
if self.fs is not None:
|
198
|
+
return
|
199
|
+
if factory is not None and not callable(factory):
|
200
|
+
raise TypeError("fs_factory must be callable")
|
201
|
+
self._fs_factory = factory
|
202
|
+
self._owns_fs = factory is not None
|
203
|
+
|
110
204
|
def _ensure_fs(self) -> Optional[fsspec.AbstractFileSystem]:
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
205
|
+
with self._close_lock:
|
206
|
+
self._assert_open()
|
207
|
+
if self.fs is not None:
|
208
|
+
return self.fs
|
209
|
+
if self._fs_factory is None:
|
210
|
+
return None
|
211
|
+
fs_new = self._fs_factory()
|
212
|
+
if not isinstance(fs_new, fsspec.AbstractFileSystem):
|
213
|
+
raise TypeError(f"fs_factory() must return fsspec.AbstractFileSystem, got {type(fs_new)!r}")
|
214
|
+
self.fs = fs_new
|
215
|
+
return self.fs
|
119
216
|
|
120
217
|
def require_fs(self) -> fsspec.AbstractFileSystem:
|
121
|
-
"""Return a filesystem or raise if not configured/creatable."""
|
122
218
|
fs = self._ensure_fs()
|
123
219
|
if fs is None:
|
124
|
-
raise RuntimeError(
|
125
|
-
f"{self.__class__.__name__}: filesystem is required but not configured"
|
126
|
-
)
|
220
|
+
raise RuntimeError(f"{self.__class__.__name__}: filesystem is required but not configured")
|
127
221
|
return fs
|
128
222
|
|
129
|
-
# ----------
|
223
|
+
# ---------- SSE ----------
|
224
|
+
def _create_auto_sse(self) -> None:
|
225
|
+
# internal helper: create queue sink + emitter, mark as owned
|
226
|
+
sink = _QueueSSE()
|
227
|
+
self._sse = sink
|
228
|
+
self._owns_sse = True
|
229
|
+
self._emitter = self._build_emitter(sink)
|
230
|
+
|
231
|
+
def set_sse_factory(self, factory: Optional[Callable[[], object]]) -> None:
|
232
|
+
with self._close_lock:
|
233
|
+
self._assert_open()
|
234
|
+
if self._sse is not None or self._emitter is not None:
|
235
|
+
return
|
236
|
+
if factory is not None and not callable(factory):
|
237
|
+
raise TypeError("sse_factory must be callable")
|
238
|
+
self._sse_factory = factory
|
239
|
+
self._owns_sse = factory is not None
|
240
|
+
|
241
|
+
def _ensure_sse(self) -> Optional[object]:
|
242
|
+
with self._close_lock:
|
243
|
+
if self._sse is not None:
|
244
|
+
return self._sse
|
245
|
+
self._assert_open()
|
246
|
+
if self._sse_factory is not None:
|
247
|
+
sink = self._sse_factory()
|
248
|
+
self._sse = sink
|
249
|
+
self._owns_sse = True
|
250
|
+
if self._emitter is None:
|
251
|
+
self._emitter = self._build_emitter(sink)
|
252
|
+
return self._sse
|
253
|
+
if self._auto_sse and self._emitter is None:
|
254
|
+
self._create_auto_sse()
|
255
|
+
return self._sse
|
256
|
+
return None
|
257
|
+
|
258
|
+
def get_sse(self) -> Optional[object]:
|
259
|
+
"""Public getter; creates the sink if auto_sse/factory available."""
|
260
|
+
return self._ensure_sse()
|
261
|
+
|
262
|
+
def _build_emitter(self, sink: object) -> Callable[[str, Dict[str, Any]], Awaitable[None]]:
|
263
|
+
send = getattr(sink, "send", None)
|
264
|
+
put = getattr(sink, "put", None)
|
265
|
+
|
266
|
+
if callable(send) and asyncio.iscoroutinefunction(send):
|
267
|
+
async def _emit(event: str, payload: Dict[str, Any]) -> None:
|
268
|
+
await send(event, payload)
|
269
|
+
return _emit
|
270
|
+
|
271
|
+
if callable(put) and asyncio.iscoroutinefunction(put):
|
272
|
+
async def _emit(event: str, payload: Dict[str, Any]) -> None:
|
273
|
+
await put({"event": event, "data": json.dumps(payload)})
|
274
|
+
return _emit
|
275
|
+
|
276
|
+
if callable(send) and not asyncio.iscoroutinefunction(send):
|
277
|
+
async def _emit(event: str, payload: Dict[str, Any]) -> None:
|
278
|
+
await asyncio.to_thread(send, event, payload)
|
279
|
+
return _emit
|
280
|
+
|
281
|
+
if callable(put) and not asyncio.iscoroutinefunction(put):
|
282
|
+
async def _emit(event: str, payload: Dict[str, Any]) -> None:
|
283
|
+
await asyncio.to_thread(put, {"event": event, "data": json.dumps(payload)})
|
284
|
+
return _emit
|
285
|
+
|
286
|
+
raise TypeError(f"{self.__class__.__name__}: SSE sink must expose send(event, data) or put(item)")
|
287
|
+
|
288
|
+
async def emit(self, event: str, **data: Any) -> None:
|
289
|
+
"""No-op during closing/closed or if no emitter is configured."""
|
290
|
+
if self._is_closed or self._closing:
|
291
|
+
return
|
292
|
+
if self._emitter is None:
|
293
|
+
self._ensure_sse()
|
294
|
+
emitter = self._emitter
|
295
|
+
if emitter is None:
|
296
|
+
return
|
297
|
+
try:
|
298
|
+
await emitter(event, data)
|
299
|
+
except Exception:
|
300
|
+
if self._log_cleanup_errors:
|
301
|
+
with contextlib.suppress(Exception):
|
302
|
+
self.logger.error("Error emitting SSE event %r", event, exc_info=self.debug)
|
303
|
+
|
304
|
+
# ---------- Shutdown helpers ----------
|
130
305
|
def _release_owned_fs(self) -> None:
|
131
|
-
if self._owns_fs:
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
306
|
+
if self._owns_fs and self.fs is not None:
|
307
|
+
close = getattr(self.fs, "close", None)
|
308
|
+
with contextlib.suppress(Exception):
|
309
|
+
if callable(close):
|
310
|
+
close()
|
311
|
+
self.fs = None
|
312
|
+
|
313
|
+
async def _aclose_obj(self, obj: object, timeout: float = 1.0) -> None:
|
314
|
+
aclose = getattr(obj, "aclose", None)
|
315
|
+
if callable(aclose):
|
316
|
+
with contextlib.suppress(Exception):
|
317
|
+
await asyncio.wait_for(aclose(), timeout=timeout)
|
318
|
+
close = getattr(obj, "close", None)
|
319
|
+
if callable(close):
|
320
|
+
with contextlib.suppress(Exception):
|
321
|
+
close()
|
141
322
|
|
142
323
|
def _shutdown_logger(self) -> None:
|
143
324
|
if self._owns_logger:
|
144
|
-
|
145
|
-
self.logger
|
146
|
-
|
147
|
-
|
325
|
+
with contextlib.suppress(Exception):
|
326
|
+
shutdown = getattr(self.logger, "shutdown", None)
|
327
|
+
if callable(shutdown):
|
328
|
+
shutdown()
|
329
|
+
else:
|
330
|
+
close_handlers = getattr(self.logger, "close_handlers", None)
|
331
|
+
if callable(close_handlers):
|
332
|
+
close_handlers()
|
333
|
+
|
334
|
+
def _shutdown_owned_resources_sync(self) -> None:
|
335
|
+
self._release_owned_fs()
|
336
|
+
if self._owns_sse and self._sse is not None:
|
337
|
+
with contextlib.suppress(Exception):
|
338
|
+
close = getattr(self._sse, "close", None)
|
339
|
+
if callable(close):
|
340
|
+
close()
|
341
|
+
self._sse = None
|
342
|
+
self._emitter = None
|
343
|
+
self._shutdown_logger()
|
148
344
|
|
149
|
-
def
|
345
|
+
async def _shutdown_owned_resources_async(self) -> None:
|
150
346
|
self._release_owned_fs()
|
347
|
+
if self._owns_sse and self._sse is not None:
|
348
|
+
await self._aclose_obj(self._sse)
|
349
|
+
self._sse = None
|
350
|
+
self._emitter = None
|
151
351
|
self._shutdown_logger()
|
152
352
|
|
153
353
|
# ---------- Public lifecycle (sync) ----------
|
154
|
-
|
354
|
+
@final
|
355
|
+
def close(self, *, suppress_errors: bool = False) -> None:
|
155
356
|
with self._close_lock:
|
156
357
|
if self._is_closed or self._closing:
|
157
358
|
return
|
158
359
|
self._closing = True
|
159
|
-
|
160
360
|
try:
|
161
361
|
self._cleanup()
|
162
362
|
except Exception:
|
163
|
-
# Only include traceback when debug=True
|
164
363
|
if self._log_cleanup_errors:
|
165
|
-
|
166
|
-
self.logger.error(
|
167
|
-
|
168
|
-
|
169
|
-
)
|
170
|
-
except Exception:
|
171
|
-
pass
|
172
|
-
raise
|
364
|
+
with contextlib.suppress(Exception):
|
365
|
+
self.logger.error("Error during %s._cleanup()", self.__class__.__name__, exc_info=self.debug)
|
366
|
+
if not suppress_errors:
|
367
|
+
raise
|
173
368
|
finally:
|
174
369
|
with self._close_lock:
|
175
370
|
self._is_closed = True
|
176
371
|
self._closing = False
|
177
|
-
self.
|
372
|
+
self._shutdown_owned_resources_sync()
|
178
373
|
if self.debug:
|
179
|
-
|
374
|
+
with contextlib.suppress(Exception):
|
180
375
|
self.logger.debug("Component %s closed.", self.__class__.__name__)
|
181
|
-
except Exception:
|
182
|
-
pass
|
183
376
|
|
184
377
|
# ---------- Public lifecycle (async) ----------
|
185
|
-
|
378
|
+
@final
|
379
|
+
async def aclose(
|
380
|
+
self,
|
381
|
+
*,
|
382
|
+
suppress_errors: bool = False,
|
383
|
+
run_sync_cleanup_if_missing: bool = False,
|
384
|
+
) -> None:
|
186
385
|
with self._close_lock:
|
187
386
|
if self._is_closed or self._closing:
|
188
387
|
return
|
189
388
|
self._closing = True
|
190
|
-
|
191
389
|
try:
|
192
|
-
|
390
|
+
if run_sync_cleanup_if_missing and (type(self)._acleanup is ManagedResource._acleanup):
|
391
|
+
await asyncio.to_thread(self._cleanup)
|
392
|
+
else:
|
393
|
+
await self._acleanup()
|
193
394
|
except Exception:
|
194
|
-
# Only include traceback when debug=True
|
195
395
|
if self._log_cleanup_errors:
|
196
|
-
|
197
|
-
self.logger.error(
|
198
|
-
|
199
|
-
|
200
|
-
)
|
201
|
-
except Exception:
|
202
|
-
pass
|
203
|
-
raise
|
396
|
+
with contextlib.suppress(Exception):
|
397
|
+
self.logger.error("Error during %s._acleanup()", self.__class__.__name__, exc_info=self.debug)
|
398
|
+
if not suppress_errors:
|
399
|
+
raise
|
204
400
|
finally:
|
205
401
|
with self._close_lock:
|
206
402
|
self._is_closed = True
|
207
403
|
self._closing = False
|
208
|
-
self.
|
404
|
+
await self._shutdown_owned_resources_async()
|
209
405
|
if self.debug:
|
210
|
-
|
406
|
+
with contextlib.suppress(Exception):
|
211
407
|
self.logger.debug("Async component %s closed.", self.__class__.__name__)
|
212
|
-
except Exception:
|
213
|
-
pass
|
214
408
|
|
215
409
|
# ---------- Context managers ----------
|
410
|
+
@final
|
216
411
|
def __enter__(self) -> Self:
|
217
412
|
return self
|
218
413
|
|
414
|
+
@final
|
219
415
|
def __exit__(self, exc_type, exc, tb) -> bool:
|
220
416
|
self.close()
|
221
|
-
return False
|
417
|
+
return False
|
222
418
|
|
419
|
+
@final
|
223
420
|
async def __aenter__(self) -> Self:
|
224
421
|
return self
|
225
422
|
|
423
|
+
@final
|
226
424
|
async def __aexit__(self, exc_type, exc, tb) -> bool:
|
227
425
|
await self.aclose()
|
228
426
|
return False
|
229
427
|
|
230
|
-
# ---------- Finalizer (
|
428
|
+
# ---------- Finalizer (silent) ----------
|
231
429
|
@staticmethod
|
232
430
|
def _finalize_static(ref: "weakref.ReferenceType[ManagedResource]") -> None:
|
233
431
|
obj = ref()
|
234
432
|
if obj is None:
|
235
433
|
return
|
236
|
-
# No logging here; interpreter may be tearing down.
|
237
|
-
# Best-effort silent cleanup; avoid locks and context managers.
|
238
434
|
try:
|
239
435
|
if not obj._is_closed:
|
240
|
-
|
436
|
+
with contextlib.suppress(Exception):
|
241
437
|
obj._cleanup()
|
242
|
-
except Exception:
|
243
|
-
pass
|
244
438
|
obj._is_closed = True
|
245
|
-
|
246
|
-
obj.
|
247
|
-
except Exception:
|
248
|
-
pass
|
439
|
+
with contextlib.suppress(Exception):
|
440
|
+
obj._shutdown_owned_resources_sync()
|
249
441
|
except Exception:
|
250
|
-
# do not show anything at garbage collection time
|
251
442
|
pass
|
252
443
|
|
444
|
+
## Before SSE handling
|
445
|
+
|
446
|
+
# import abc
|
447
|
+
# import threading
|
448
|
+
# import weakref
|
449
|
+
# from typing import Self, Optional, Callable
|
450
|
+
#
|
451
|
+
# import fsspec
|
452
|
+
#
|
453
|
+
# from sibi_dst.utils import Logger
|
454
|
+
#
|
455
|
+
#
|
456
|
+
# class ManagedResource(abc.ABC):
|
457
|
+
# """
|
458
|
+
# Boilerplate ABC for components that manage a logger and an optional fsspec filesystem,
|
459
|
+
# with sync/async lifecycle helpers, lazy FS creation via an optional factory, and
|
460
|
+
# configurable cleanup-error logging.
|
461
|
+
# """
|
462
|
+
#
|
463
|
+
# def __init__(
|
464
|
+
# self,
|
465
|
+
# *,
|
466
|
+
# verbose: bool = False,
|
467
|
+
# debug: bool = False,
|
468
|
+
# log_cleanup_errors: bool = True,
|
469
|
+
# logger: Optional[Logger] = None,
|
470
|
+
# fs: Optional[fsspec.AbstractFileSystem] = None,
|
471
|
+
# fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None,
|
472
|
+
# **_: object,
|
473
|
+
# ) -> None:
|
474
|
+
# # ---- Declared upfront for type checkers
|
475
|
+
# self.logger: Logger
|
476
|
+
# self.fs: Optional[fsspec.AbstractFileSystem] = None
|
477
|
+
# self._fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None
|
478
|
+
# self._owns_logger: bool = False
|
479
|
+
# self._owns_fs: bool = False
|
480
|
+
# self._is_closed: bool = False
|
481
|
+
# self._closing: bool = False
|
482
|
+
# self._close_lock = threading.RLock()
|
483
|
+
#
|
484
|
+
# self.verbose = verbose
|
485
|
+
# self.debug = debug
|
486
|
+
# self._log_cleanup_errors = log_cleanup_errors
|
487
|
+
#
|
488
|
+
# # ---- Logger ownership
|
489
|
+
# if logger is None:
|
490
|
+
# self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
|
491
|
+
# self._owns_logger = True
|
492
|
+
# level = Logger.DEBUG if self.debug else (Logger.INFO if self.verbose else Logger.WARNING)
|
493
|
+
# self.logger.set_level(level)
|
494
|
+
# else:
|
495
|
+
# self.logger = logger
|
496
|
+
# self._owns_logger = False # do not mutate external logger
|
497
|
+
#
|
498
|
+
# # ---- FS ownership & lazy creation
|
499
|
+
# if fs is not None:
|
500
|
+
# self.fs = fs
|
501
|
+
# self._owns_fs = False
|
502
|
+
# self._fs_factory = None
|
503
|
+
# elif fs_factory is not None:
|
504
|
+
# # Lazy: don't create until first use
|
505
|
+
# self._fs_factory = fs_factory
|
506
|
+
# self._owns_fs = True # we will own it *if* created
|
507
|
+
# self.fs = None
|
508
|
+
# else:
|
509
|
+
# self.fs = None
|
510
|
+
# self._owns_fs = False
|
511
|
+
# self._fs_factory = None
|
512
|
+
#
|
513
|
+
# # Register a GC-time finalizer that does not capture self
|
514
|
+
# self_ref = weakref.ref(self)
|
515
|
+
# self._finalizer = weakref.finalize(self, self._finalize_static, self_ref)
|
516
|
+
#
|
517
|
+
# if self.debug:
|
518
|
+
# try:
|
519
|
+
# self.logger.debug("Component %s initialized. %s", self.__class__.__name__, repr(self))
|
520
|
+
# except Exception:
|
521
|
+
# pass
|
522
|
+
#
|
523
|
+
# # ---------- Introspection ----------
|
524
|
+
# @property
|
525
|
+
# def is_closed(self) -> bool:
|
526
|
+
# return self._is_closed
|
527
|
+
#
|
528
|
+
# @property
|
529
|
+
# def closed(self) -> bool: # alias
|
530
|
+
# return self._is_closed
|
531
|
+
#
|
532
|
+
# def __repr__(self) -> str:
|
533
|
+
# class_name = self.__class__.__name__
|
534
|
+
# logger_status = "own" if self._owns_logger else "external"
|
535
|
+
# if self.fs is None and self._fs_factory is not None:
|
536
|
+
# fs_status = "own(lazy)"
|
537
|
+
# elif self.fs is None:
|
538
|
+
# fs_status = "none"
|
539
|
+
# else:
|
540
|
+
# fs_status = "own" if self._owns_fs else "external"
|
541
|
+
# return (f"<{class_name} debug={self.debug} verbose={self.verbose} "
|
542
|
+
# f"log_cleanup_errors={self._log_cleanup_errors} "
|
543
|
+
# f"logger={logger_status} fs={fs_status}>")
|
544
|
+
#
|
545
|
+
# # ---------- Subclass hooks ----------
|
546
|
+
# def _cleanup(self) -> None:
|
547
|
+
# """Sync cleanup for resources created BY THE SUBCLASS."""
|
548
|
+
# return
|
549
|
+
#
|
550
|
+
# async def _acleanup(self) -> None:
|
551
|
+
# """Async cleanup for resources created BY THE SUBCLASS."""
|
552
|
+
# return
|
553
|
+
#
|
554
|
+
# # ---------- FS helpers ----------
|
555
|
+
# def _ensure_fs(self) -> Optional[fsspec.AbstractFileSystem]:
|
556
|
+
# """Create the FS lazily if a factory was provided. Return fs (or None)."""
|
557
|
+
# if self.fs is None and self._fs_factory is not None:
|
558
|
+
# created = self._fs_factory()
|
559
|
+
# if not isinstance(created, fsspec.AbstractFileSystem):
|
560
|
+
# raise TypeError(f"fs_factory() must return fsspec.AbstractFileSystem, got {type(created)!r}")
|
561
|
+
# self.fs = created
|
562
|
+
# # _owns_fs already True when factory is present
|
563
|
+
# return self.fs
|
564
|
+
#
|
565
|
+
# def require_fs(self) -> fsspec.AbstractFileSystem:
|
566
|
+
# """Return a filesystem or raise if not configured/creatable."""
|
567
|
+
# fs = self._ensure_fs()
|
568
|
+
# if fs is None:
|
569
|
+
# raise RuntimeError(
|
570
|
+
# f"{self.__class__.__name__}: filesystem is required but not configured"
|
571
|
+
# )
|
572
|
+
# return fs
|
573
|
+
#
|
574
|
+
# # ---------- Shared shutdown helpers (no logging; safe for late shutdown) ----------
|
575
|
+
# def _release_owned_fs(self) -> None:
|
576
|
+
# if self._owns_fs:
|
577
|
+
# # ensure creation state is respected even if never used
|
578
|
+
# _ = self.fs or None # no-op; if never created, nothing to close
|
579
|
+
# if self.fs is not None:
|
580
|
+
# close = getattr(self.fs, "close", None)
|
581
|
+
# try:
|
582
|
+
# if callable(close):
|
583
|
+
# close()
|
584
|
+
# finally:
|
585
|
+
# self.fs = None
|
586
|
+
#
|
587
|
+
# def _shutdown_logger(self) -> None:
|
588
|
+
# if self._owns_logger:
|
589
|
+
# try:
|
590
|
+
# self.logger.shutdown()
|
591
|
+
# except Exception:
|
592
|
+
# pass
|
593
|
+
#
|
594
|
+
# def _shutdown_owned_resources(self) -> None:
|
595
|
+
# self._release_owned_fs()
|
596
|
+
# self._shutdown_logger()
|
597
|
+
#
|
598
|
+
# # ---------- Public lifecycle (sync) ----------
|
599
|
+
# def close(self) -> None:
|
600
|
+
# with self._close_lock:
|
601
|
+
# if self._is_closed or self._closing:
|
602
|
+
# return
|
603
|
+
# self._closing = True
|
604
|
+
#
|
605
|
+
# try:
|
606
|
+
# self._cleanup()
|
607
|
+
# except Exception:
|
608
|
+
# # Only include traceback when debug=True
|
609
|
+
# if self._log_cleanup_errors:
|
610
|
+
# try:
|
611
|
+
# self.logger.error(
|
612
|
+
# "Error during %s._cleanup()", self.__class__.__name__,
|
613
|
+
# exc_info=self.debug
|
614
|
+
# )
|
615
|
+
# except Exception:
|
616
|
+
# pass
|
617
|
+
# raise
|
618
|
+
# finally:
|
619
|
+
# with self._close_lock:
|
620
|
+
# self._is_closed = True
|
621
|
+
# self._closing = False
|
622
|
+
# self._shutdown_owned_resources()
|
623
|
+
# if self.debug:
|
624
|
+
# try:
|
625
|
+
# self.logger.debug("Component %s closed.", self.__class__.__name__)
|
626
|
+
# except Exception:
|
627
|
+
# pass
|
628
|
+
#
|
629
|
+
# # ---------- Public lifecycle (async) ----------
|
630
|
+
# async def aclose(self) -> None:
|
631
|
+
# with self._close_lock:
|
632
|
+
# if self._is_closed or self._closing:
|
633
|
+
# return
|
634
|
+
# self._closing = True
|
635
|
+
#
|
636
|
+
# try:
|
637
|
+
# await self._acleanup()
|
638
|
+
# except Exception:
|
639
|
+
# # Only include traceback when debug=True
|
640
|
+
# if self._log_cleanup_errors:
|
641
|
+
# try:
|
642
|
+
# self.logger.error(
|
643
|
+
# "Error during %s._acleanup()", self.__class__.__name__,
|
644
|
+
# exc_info=self.debug
|
645
|
+
# )
|
646
|
+
# except Exception:
|
647
|
+
# pass
|
648
|
+
# raise
|
649
|
+
# finally:
|
650
|
+
# with self._close_lock:
|
651
|
+
# self._is_closed = True
|
652
|
+
# self._closing = False
|
653
|
+
# self._shutdown_owned_resources()
|
654
|
+
# if self.debug:
|
655
|
+
# try:
|
656
|
+
# self.logger.debug("Async component %s closed.", self.__class__.__name__)
|
657
|
+
# except Exception:
|
658
|
+
# pass
|
659
|
+
#
|
660
|
+
# # ---------- Context managers ----------
|
661
|
+
# def __enter__(self) -> Self:
|
662
|
+
# return self
|
663
|
+
#
|
664
|
+
# def __exit__(self, exc_type, exc, tb) -> bool:
|
665
|
+
# self.close()
|
666
|
+
# return False # propagate exceptions
|
667
|
+
#
|
668
|
+
# async def __aenter__(self) -> Self:
|
669
|
+
# return self
|
670
|
+
#
|
671
|
+
# async def __aexit__(self, exc_type, exc, tb) -> bool:
|
672
|
+
# await self.aclose()
|
673
|
+
# return False
|
674
|
+
#
|
675
|
+
# # ---------- Finalizer ( at Garbage Collection-time absolutely silent) ----------
|
676
|
+
# @staticmethod
|
677
|
+
# def _finalize_static(ref: "weakref.ReferenceType[ManagedResource]") -> None:
|
678
|
+
# obj = ref()
|
679
|
+
# if obj is None:
|
680
|
+
# return
|
681
|
+
# # No logging here; interpreter may be tearing down.
|
682
|
+
# # Best-effort silent cleanup; avoid locks and context managers.
|
683
|
+
# try:
|
684
|
+
# if not obj._is_closed:
|
685
|
+
# try:
|
686
|
+
# obj._cleanup()
|
687
|
+
# except Exception:
|
688
|
+
# pass
|
689
|
+
# obj._is_closed = True
|
690
|
+
# try:
|
691
|
+
# obj._shutdown_owned_resources()
|
692
|
+
# except Exception:
|
693
|
+
# pass
|
694
|
+
# except Exception:
|
695
|
+
# # do not show anything at garbage collection time
|
696
|
+
# pass
|
697
|
+
#
|