glow 0.15.3__tar.gz → 0.15.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {glow-0.15.3 → glow-0.15.4}/PKG-INFO +1 -1
  2. {glow-0.15.3 → glow-0.15.4}/pyproject.toml +1 -1
  3. {glow-0.15.3 → glow-0.15.4}/src/glow/_async.py +21 -14
  4. {glow-0.15.3 → glow-0.15.4}/src/glow/_cache.py +67 -52
  5. {glow-0.15.3 → glow-0.15.4}/src/glow/_concurrency.py +20 -13
  6. glow-0.15.4/src/glow/_dev.py +18 -0
  7. {glow-0.15.3 → glow-0.15.4}/src/glow/_parallel.py +50 -48
  8. {glow-0.15.3 → glow-0.15.4}/src/glow/cli.py +82 -23
  9. {glow-0.15.3 → glow-0.15.4}/test/test_cli.py +25 -19
  10. glow-0.15.3/src/glow/_dev.py +0 -16
  11. {glow-0.15.3 → glow-0.15.4}/.gitignore +0 -0
  12. {glow-0.15.3 → glow-0.15.4}/LICENSE +0 -0
  13. {glow-0.15.3 → glow-0.15.4}/README.md +0 -0
  14. {glow-0.15.3 → glow-0.15.4}/src/glow/__init__.py +0 -0
  15. {glow-0.15.3 → glow-0.15.4}/src/glow/_array.py +0 -0
  16. {glow-0.15.3 → glow-0.15.4}/src/glow/_async.pyi +0 -0
  17. {glow-0.15.3 → glow-0.15.4}/src/glow/_cache.pyi +0 -0
  18. {glow-0.15.3 → glow-0.15.4}/src/glow/_concurrency.pyi +0 -0
  19. {glow-0.15.3 → glow-0.15.4}/src/glow/_coro.py +0 -0
  20. {glow-0.15.3 → glow-0.15.4}/src/glow/_debug.py +0 -0
  21. {glow-0.15.3 → glow-0.15.4}/src/glow/_ic.py +0 -0
  22. {glow-0.15.3 → glow-0.15.4}/src/glow/_import_hook.py +0 -0
  23. {glow-0.15.3 → glow-0.15.4}/src/glow/_imutil.py +0 -0
  24. {glow-0.15.3 → glow-0.15.4}/src/glow/_keys.py +0 -0
  25. {glow-0.15.3 → glow-0.15.4}/src/glow/_logging.py +0 -0
  26. {glow-0.15.3 → glow-0.15.4}/src/glow/_more.py +0 -0
  27. {glow-0.15.3 → glow-0.15.4}/src/glow/_parallel.pyi +0 -0
  28. {glow-0.15.3 → glow-0.15.4}/src/glow/_patch_len.py +0 -0
  29. {glow-0.15.3 → glow-0.15.4}/src/glow/_patch_print.py +0 -0
  30. {glow-0.15.3 → glow-0.15.4}/src/glow/_patch_scipy.py +0 -0
  31. {glow-0.15.3 → glow-0.15.4}/src/glow/_profile.py +0 -0
  32. {glow-0.15.3 → glow-0.15.4}/src/glow/_profile.pyi +0 -0
  33. {glow-0.15.3 → glow-0.15.4}/src/glow/_reduction.py +0 -0
  34. {glow-0.15.3 → glow-0.15.4}/src/glow/_repr.py +0 -0
  35. {glow-0.15.3 → glow-0.15.4}/src/glow/_reusable.py +0 -0
  36. {glow-0.15.3 → glow-0.15.4}/src/glow/_sizeof.py +0 -0
  37. {glow-0.15.3 → glow-0.15.4}/src/glow/_streams.py +0 -0
  38. {glow-0.15.3 → glow-0.15.4}/src/glow/_thread_quota.py +0 -0
  39. {glow-0.15.3 → glow-0.15.4}/src/glow/_types.py +0 -0
  40. {glow-0.15.3 → glow-0.15.4}/src/glow/_uuid.py +0 -0
  41. {glow-0.15.3 → glow-0.15.4}/src/glow/_wrap.py +0 -0
  42. {glow-0.15.3 → glow-0.15.4}/src/glow/api/__init__.py +0 -0
  43. {glow-0.15.3 → glow-0.15.4}/src/glow/api/config.py +0 -0
  44. {glow-0.15.3 → glow-0.15.4}/src/glow/api/exporting.py +0 -0
  45. {glow-0.15.3 → glow-0.15.4}/src/glow/cli.pyi +0 -0
  46. {glow-0.15.3 → glow-0.15.4}/src/glow/io/__init__.py +0 -0
  47. {glow-0.15.3 → glow-0.15.4}/src/glow/io/_sound.py +0 -0
  48. {glow-0.15.3 → glow-0.15.4}/src/glow/io/_svg.py +0 -0
  49. {glow-0.15.3 → glow-0.15.4}/src/glow/py.typed +0 -0
  50. {glow-0.15.3 → glow-0.15.4}/test/__init__.py +0 -0
  51. {glow-0.15.3 → glow-0.15.4}/test/test_api.py +0 -0
  52. {glow-0.15.3 → glow-0.15.4}/test/test_batch.py +0 -0
  53. {glow-0.15.3 → glow-0.15.4}/test/test_buffered.py +0 -0
  54. {glow-0.15.3 → glow-0.15.4}/test/test_iter.py +0 -0
  55. {glow-0.15.3 → glow-0.15.4}/test/test_shm.py +0 -0
  56. {glow-0.15.3 → glow-0.15.4}/test/test_thread_pool.py +0 -0
  57. {glow-0.15.3 → glow-0.15.4}/test/test_timed.py +0 -0
  58. {glow-0.15.3 → glow-0.15.4}/test/test_timer.py +0 -0
  59. {glow-0.15.3 → glow-0.15.4}/test/test_uuid.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: glow
3
- Version: 0.15.3
3
+ Version: 0.15.4
4
4
  Summary: Functional Python tools
5
5
  Project-URL: homepage, https://github.com/arquolo/glow
6
6
  Author-email: Paul Maevskikh <arquolo@gmail.com>
@@ -7,7 +7,7 @@ only-packages = true
7
7
 
8
8
  [project]
9
9
  name = "glow"
10
- version = "0.15.3"
10
+ version = "0.15.4"
11
11
  description = "Functional Python tools"
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.12"
@@ -16,7 +16,7 @@ from contextlib import suppress
16
16
  from functools import partial
17
17
  from typing import TypeGuard, cast, overload
18
18
 
19
- from ._dev import declutter_tb
19
+ from ._dev import hide_frame
20
20
  from ._types import (
21
21
  ABatchDecorator,
22
22
  ABatchFn,
@@ -24,6 +24,7 @@ from ._types import (
24
24
  AnyIterable,
25
25
  AnyIterator,
26
26
  Coro,
27
+ Some,
27
28
  )
28
29
 
29
30
  type _Job[T, R] = tuple[T, AnyFuture[R]]
@@ -309,7 +310,8 @@ def astreaming[T, R](
309
310
  async with lock:
310
311
  await _adispatch(fn, *batch)
311
312
 
312
- return await asyncio.gather(*fs)
313
+ with hide_frame:
314
+ return await asyncio.gather(*fs)
313
315
 
314
316
  return wrapper
315
317
 
@@ -317,20 +319,25 @@ def astreaming[T, R](
317
319
  async def _adispatch[T, R](fn: ABatchFn[T, R], *xs: _Job[T, R]) -> None:
318
320
  if not xs:
319
321
  return
320
- obj: list[R] | BaseException
322
+ obj: Some[Sequence[R]] | BaseException
321
323
  try:
322
- obj = list(await fn([x for x, _ in xs]))
323
- if len(obj) != len(xs):
324
- obj = RuntimeError(
325
- f'Call with {len(xs)} arguments '
326
- f'incorrectly returned {len(obj)} results'
327
- )
324
+ with hide_frame:
325
+ obj = Some(await fn([x for x, _ in xs]))
326
+ if not isinstance(obj.x, Sequence):
327
+ obj = TypeError(
328
+ f'Call returned non-sequence. Got {type(obj.x).__name__}'
329
+ )
330
+ elif len(obj.x) != len(xs):
331
+ obj = RuntimeError(
332
+ f'Call with {len(xs)} arguments '
333
+ f'incorrectly returned {len(obj.x)} results'
334
+ )
328
335
  except BaseException as exc: # noqa: BLE001
329
- obj = declutter_tb(exc, fn.__code__)
336
+ obj = exc
330
337
 
331
- if isinstance(obj, BaseException):
338
+ if isinstance(obj, Some):
339
+ for (_, f), res in zip(xs, obj.x):
340
+ f.set_result(res)
341
+ else:
332
342
  for _, f in xs:
333
343
  f.set_exception(obj)
334
- else:
335
- for (_, f), res in zip(xs, obj):
336
- f.set_result(res)
@@ -12,19 +12,27 @@ from collections.abc import (
12
12
  Iterator,
13
13
  KeysView,
14
14
  MutableMapping,
15
+ Sequence,
15
16
  )
16
17
  from dataclasses import dataclass, field
17
18
  from inspect import iscoroutinefunction
18
19
  from threading import RLock
19
- from types import CodeType
20
20
  from typing import Final, Protocol, SupportsInt, cast
21
21
  from weakref import WeakValueDictionary
22
22
 
23
- from ._dev import declutter_tb
23
+ from ._dev import hide_frame
24
24
  from ._keys import make_key
25
25
  from ._repr import si_bin
26
26
  from ._sizeof import sizeof
27
- from ._types import ABatchFn, AnyFuture, BatchFn, CachePolicy, Decorator, KeyFn
27
+ from ._types import (
28
+ ABatchFn,
29
+ AnyFuture,
30
+ BatchFn,
31
+ CachePolicy,
32
+ Decorator,
33
+ KeyFn,
34
+ Some,
35
+ )
28
36
 
29
37
 
30
38
  class _Empty(enum.Enum):
@@ -228,7 +236,6 @@ class _StrongCache[R](_WeakCache[R]):
228
236
  @dataclass(frozen=True, slots=True)
229
237
  class _CacheState[R]:
230
238
  cache: _AbstractCache[R]
231
- code: CodeType # for short tracebacks
232
239
  key_fn: KeyFn
233
240
  futures: WeakValueDictionary[Hashable, AnyFuture[R]] = field(
234
241
  default_factory=WeakValueDictionary
@@ -261,19 +268,20 @@ def _sync_memoize[**P, R](
261
268
  if not is_owner:
262
269
  return f.result()
263
270
 
264
- try:
265
- ret = fn(*args, **kwargs)
266
- except BaseException as exc:
267
- f.set_exception(exc)
268
- with lock:
269
- cs.futures.pop(key)
270
- raise
271
- else:
272
- f.set_result(ret)
273
- with lock:
274
- cs.cache[key] = ret
275
- cs.futures.pop(key)
276
- return ret
271
+ with hide_frame:
272
+ try:
273
+ ret = fn(*args, **kwargs)
274
+ except BaseException as exc:
275
+ f.set_exception(exc)
276
+ with lock:
277
+ cs.futures.pop(key)
278
+ raise
279
+ else:
280
+ f.set_result(ret)
281
+ with lock:
282
+ cs.cache[key] = ret
283
+ cs.futures.pop(key)
284
+ return ret
277
285
 
278
286
  return wrapper
279
287
 
@@ -296,17 +304,18 @@ def _async_memoize[**P, R](
296
304
 
297
305
  # NOTE: fn() is not within threading.Lock, thus it's not thread safe
298
306
  # NOTE: but it's async-safe because this `await` is only one here.
299
- try:
300
- ret = await fn(*args, **kwargs)
301
- except BaseException as exc:
302
- f.set_exception(exc)
303
- cs.futures.pop(key)
304
- raise
305
- else:
306
- f.set_result(ret)
307
- cs.cache[key] = ret
308
- cs.futures.pop(key)
309
- return ret
307
+ with hide_frame:
308
+ try:
309
+ ret = await fn(*args, **kwargs)
310
+ except BaseException as exc:
311
+ f.set_exception(exc)
312
+ cs.futures.pop(key)
313
+ raise
314
+ else:
315
+ f.set_result(ret)
316
+ cs.cache[key] = ret
317
+ cs.futures.pop(key)
318
+ return ret
310
319
 
311
320
  return wrapper
312
321
 
@@ -352,12 +361,12 @@ class _BatchedQuery[T, R]:
352
361
  return bool(self._jobs)
353
362
 
354
363
  @property
355
- def result(self) -> list[R] | BaseException:
364
+ def result(self) -> Some[Sequence[R]] | BaseException:
356
365
  match list(self._errors):
357
366
  case []:
358
367
  if self._default_tp:
359
368
  return self._default_tp()
360
- return [self._done[k] for k in self._keys]
369
+ return Some([self._done[k] for k in self._keys])
361
370
  case [e]:
362
371
  return e
363
372
  case excs:
@@ -367,20 +376,25 @@ class _BatchedQuery[T, R]:
367
376
  return BaseExceptionGroup(msg, excs)
368
377
 
369
378
  @result.setter
370
- def result(self, obj: list[R] | BaseException) -> None:
379
+ def result(self, obj: Some[Sequence[R]] | BaseException) -> None:
371
380
  done_jobs = [(k, f) for k, a, f in self._jobs if a]
372
381
 
373
- if not isinstance(obj, BaseException):
374
- if len(obj) == len(done_jobs):
375
- for (k, f), value in zip(done_jobs, obj):
376
- f.set_result(value)
377
- self._stash.append((k, value))
378
- return
379
-
380
- obj = RuntimeError(
381
- f'Call with {len(done_jobs)} arguments '
382
- f'incorrectly returned {len(obj)} results'
383
- )
382
+ if isinstance(obj, Some):
383
+ if isinstance(obj.x, Sequence):
384
+ if len(obj.x) == len(done_jobs):
385
+ for (k, f), value in zip(done_jobs, obj.x):
386
+ f.set_result(value)
387
+ self._stash.append((k, value))
388
+ return
389
+
390
+ obj = RuntimeError(
391
+ f'Call with {len(done_jobs)} arguments '
392
+ f'incorrectly returned {len(obj.x)} results'
393
+ )
394
+ else:
395
+ obj = TypeError(
396
+ f'Call returned non-sequence. Got {type(obj.x).__name__}'
397
+ )
384
398
 
385
399
  for _, f in done_jobs:
386
400
  f.set_exception(obj)
@@ -409,9 +423,6 @@ class _BatchedQuery[T, R]:
409
423
  self._stash.append((k, f.result()))
410
424
 
411
425
  def sync(self) -> None:
412
- for e in self._errors:
413
- declutter_tb(e, self._cs.code)
414
-
415
426
  for k, r in self._stash:
416
427
  self._done[k] = self._cs.cache[k] = r
417
428
 
@@ -433,7 +444,8 @@ def _sync_memoize_batched[T, R](
433
444
  # Run tasks we are first to schedule
434
445
  if args := q.args:
435
446
  try:
436
- q.result = list(fn(args))
447
+ with hide_frame:
448
+ q.result = Some(fn(args))
437
449
  except BaseException as exc: # noqa: BLE001
438
450
  q.result = exc
439
451
 
@@ -446,9 +458,10 @@ def _sync_memoize_batched[T, R](
446
458
  with lock:
447
459
  q.sync()
448
460
 
449
- if isinstance(ret := q.result, BaseException):
461
+ if isinstance(ret := q.result, Some):
462
+ return list(ret.x)
463
+ with hide_frame:
450
464
  raise ret
451
- return ret
452
465
 
453
466
  return wrapper
454
467
 
@@ -463,7 +476,8 @@ def _async_memoize_batched[T, R](
463
476
  # Run tasks we are first to schedule
464
477
  if args := q.args:
465
478
  try:
466
- q.result = list(await fn(args))
479
+ with hide_frame:
480
+ q.result = Some(await fn(args))
467
481
  except BaseException as exc: # noqa: BLE001
468
482
  q.result = exc # Raise later in `q.exception()`
469
483
 
@@ -474,9 +488,10 @@ def _async_memoize_batched[T, R](
474
488
  finally:
475
489
  q.sync()
476
490
 
477
- if isinstance(ret := q.result, BaseException):
491
+ if isinstance(ret := q.result, Some):
492
+ return list(ret.x)
493
+ with hide_frame:
478
494
  raise ret
479
- return ret
480
495
 
481
496
  return wrapper
482
497
 
@@ -491,7 +506,7 @@ def _memoize[**P, R](
491
506
  key_fn: KeyFn,
492
507
  batched: bool,
493
508
  ) -> Callable[P, R]:
494
- cs = _CacheState(cache, fn.__code__, key_fn)
509
+ cs = _CacheState(cache, key_fn)
495
510
 
496
511
  if batched and iscoroutinefunction(fn):
497
512
  w = cast(
@@ -18,7 +18,8 @@ from typing import Never, cast
18
18
  from warnings import warn
19
19
 
20
20
  from ._cache import memoize
21
- from ._types import AnyFuture, BatchDecorator, BatchFn
21
+ from ._dev import hide_frame
22
+ from ._types import AnyFuture, BatchDecorator, BatchFn, Some
22
23
 
23
24
  _PATIENCE = 0.01
24
25
 
@@ -120,23 +121,28 @@ def _batch_invoke[T, R](
120
121
  if not batch:
121
122
  return
122
123
 
123
- obj: list[R] | BaseException
124
+ obj: Some[Sequence[R]] | BaseException
124
125
  try:
125
- obj = [*func([x for x, _ in batch])]
126
- if len(obj) != len(batch):
127
- obj = RuntimeError(
128
- f'Call with {len(batch)} arguments '
129
- f'incorrectly returned {len(obj)} results'
130
- )
126
+ with hide_frame:
127
+ obj = Some(func([x for x, _ in batch]))
128
+ if not isinstance(obj.x, Sequence):
129
+ obj = TypeError(
130
+ f'Call returned non-sequence. Got {type(obj.x).__name__}'
131
+ )
132
+ elif len(obj.x) != len(batch):
133
+ obj = RuntimeError(
134
+ f'Call with {len(batch)} arguments '
135
+ f'incorrectly returned {len(obj.x)} results'
136
+ )
131
137
  except BaseException as exc: # noqa: BLE001
132
138
  obj = exc
133
139
 
134
- if isinstance(obj, BaseException):
140
+ if isinstance(obj, Some):
141
+ for (_, f), r in zip(batch, obj.x):
142
+ f.set_result(r)
143
+ else:
135
144
  for _, f in batch:
136
145
  f.set_exception(obj)
137
- else:
138
- for (_, f), r in zip(batch, obj):
139
- f.set_result(r)
140
146
 
141
147
 
142
148
  def _start_fetch_compute[T, R](
@@ -227,7 +233,8 @@ def streaming[T, R](
227
233
 
228
234
  # Cannot time out - all are done
229
235
  if isinstance(obj := _gather(fs), BaseException):
230
- raise obj
236
+ with hide_frame:
237
+ raise obj
231
238
  return obj
232
239
 
233
240
  # TODO: if func is instance method - recreate wrapper per instance
@@ -0,0 +1,18 @@
1
+ __all__ = ['hide_frame']
2
+
3
+
4
+ class _HideFrame:
5
+ """Context manager to hide current frame in traceback"""
6
+
7
+ def __enter__(self):
8
+ return self
9
+
10
+ def __exit__(self, tp, val, tb):
11
+ if tp is None:
12
+ return True
13
+ if tb := val.__traceback__:
14
+ val.__traceback__ = tb.tb_next
15
+ return False
16
+
17
+
18
+ hide_frame = _HideFrame()
@@ -16,7 +16,6 @@ import warnings
16
16
  import weakref
17
17
  from collections.abc import Callable, Iterable, Iterator, Mapping, Sized
18
18
  from concurrent.futures import Executor, Future
19
- from concurrent.futures import TimeoutError as _TimeoutError
20
19
  from contextlib import ExitStack, contextmanager
21
20
  from cProfile import Profile
22
21
  from functools import partial
@@ -27,8 +26,7 @@ from operator import methodcaller
27
26
  from pstats import Stats
28
27
  from queue import Empty, SimpleQueue
29
28
  from threading import Lock
30
- from time import perf_counter, sleep
31
- from types import CodeType
29
+ from time import monotonic, sleep
32
30
  from typing import Final, Protocol, Self, cast
33
31
 
34
32
  import loky
@@ -38,7 +36,7 @@ try:
38
36
  except ImportError:
39
37
  psutil = None
40
38
 
41
- from ._dev import declutter_tb
39
+ from ._dev import hide_frame
42
40
  from ._more import chunked, ilen
43
41
  from ._reduction import move_to_shmem, reducers
44
42
  from ._thread_quota import ThreadQuota
@@ -133,18 +131,15 @@ def _retry_call[T](fn: Callable[..., T], *exc: type[BaseException]) -> T:
133
131
  if sys.platform == 'win32':
134
132
 
135
133
  def _exception[T](f: Future[T], /) -> BaseException | None:
136
- return _retry_call(f.exception, _TimeoutError)
134
+ return _retry_call(f.exception, TimeoutError)
137
135
 
138
136
  else:
139
137
  _exception = Future.exception
140
138
 
141
139
 
142
- def _result[T](
143
- f: Future[T], code: CodeType, cancel: bool = True
144
- ) -> Some[T] | BaseException:
140
+ def _result[T](f: Future[T], cancel: bool = True) -> Some[T] | BaseException:
145
141
  try:
146
- exc = _exception(f)
147
- return declutter_tb(exc, code) if exc else Some(f.result())
142
+ return exc if (exc := _exception(f)) else Some(f.result())
148
143
  finally:
149
144
  if cancel:
150
145
  f.cancel()
@@ -280,9 +275,10 @@ class buffered[T](Iterator[T]): # noqa: N801
280
275
 
281
276
  self.close()
282
277
  # Reraise exception from source iterable if any
283
- obj = _result(self._consume, _consume.__code__, cancel=False)
278
+ obj = _result(self._consume, cancel=False)
284
279
  if not isinstance(obj, Some):
285
- raise obj
280
+ with hide_frame:
281
+ raise obj
286
282
 
287
283
  raise StopIteration
288
284
 
@@ -298,43 +294,49 @@ class _AutoSize:
298
294
 
299
295
  def __init__(self) -> None:
300
296
  self.lock = Lock()
297
+ assert self.MIN_DURATION * 2 < self.MAX_DURATION, 'Range is too tight'
301
298
 
302
299
  def suggest(self) -> int:
303
300
  with self.lock:
304
- if 0 < self.duration < self.MIN_DURATION:
305
- self.size *= 2
306
- self.duration = 0.0
307
- _LOGGER.debug('Doubling batch size to %d', self.size)
308
-
309
- elif self.duration > self.MAX_DURATION:
310
- size = int(2 * self.size * self.MIN_DURATION / self.duration)
311
- size = max(size, 1)
312
- if self.size != size:
313
- self.duration = 0.0
314
- self.size = size
315
- _LOGGER.debug('Reducing batch size to %d', self.size)
316
-
317
301
  return self.size
318
302
 
319
- def update(self, start_time: float, fut: Future[Sized]) -> None:
303
+ def update(self, n: int, start_time: float, fut: Future[Sized]) -> None:
320
304
  # Compute as soon as future became done, discard later if not needed
321
- duration = perf_counter() - start_time
305
+ duration = monotonic() - start_time
322
306
 
323
- try:
324
- # Cannot time out, as future is always completed at this point.
325
- # Though it's unknown whether it succeeded or not, so use EAFP
326
- r = fut.result()
327
- except BaseException: # noqa: BLE001
307
+ if fut.cancelled(): # Job never run, zero load
328
308
  return
329
309
 
330
310
  with self.lock:
331
- if len(r) != self.size:
311
+ if n != self.size: # Ran with old size
332
312
  return
313
+
314
+ # Do EMA smoothing
333
315
  self.duration = (
334
316
  (0.8 * self.duration + 0.2 * duration)
335
317
  if self.duration > 0
336
318
  else duration
337
319
  )
320
+ if self.duration <= 0: # Smh not initialized yet
321
+ return # Or duration is less then `monotonic()` precision
322
+
323
+ if self.duration < self.MIN_DURATION: # Too high IPC overhead
324
+ size = self.size * 2
325
+ _LOGGER.debug('Doubling batch size to %d', size)
326
+
327
+ elif (
328
+ self.duration <= self.MAX_DURATION # Range is optimal
329
+ or self.size == 1 # Cannot reduce already minimal batch
330
+ ):
331
+ return
332
+
333
+ else: # Too high latency
334
+ size = int(2 * self.size * self.MIN_DURATION / self.duration)
335
+ size = max(size, 1)
336
+ _LOGGER.debug('Reducing batch size to %d', size)
337
+
338
+ self.size = size
339
+ self.duration = 0.0
338
340
 
339
341
 
340
342
  # ---------------------- map iterable through function ----------------------
@@ -355,8 +357,9 @@ def _schedule_auto[F: Future](
355
357
  size = _AutoSize()
356
358
  while tuples := [*islice(args_zip, size.suggest() * max_workers)]:
357
359
  chunksize = len(tuples) // max_workers or 1
358
- for f in starmap(make_future, chunked(tuples, chunksize)):
359
- f.add_done_callback(partial(size.update, perf_counter()))
360
+ for args in chunked(tuples, chunksize):
361
+ f = make_future(*args)
362
+ f.add_done_callback(partial(size.update, len(args), monotonic()))
360
363
  yield f
361
364
 
362
365
 
@@ -367,7 +370,7 @@ def _schedule_auto_v2[F: Future](
367
370
  size = _AutoSize()
368
371
  while args := [*islice(args_zip, size.suggest())]:
369
372
  f = make_future(*args)
370
- f.add_done_callback(partial(size.update, perf_counter()))
373
+ f.add_done_callback(partial(size.update, len(args), monotonic()))
371
374
  yield f
372
375
 
373
376
 
@@ -376,19 +379,20 @@ def _get_unwrap_iter[T](
376
379
  qsize: int,
377
380
  get_done_f: Callable[[], Future[T]],
378
381
  fs_scheduler: Iterator,
379
- code: CodeType,
380
382
  ) -> Iterator[T]:
381
383
  with s:
382
384
  if not qsize: # No tasks to do
383
385
  return
384
386
 
385
387
  # Unwrap 1st / schedule `N-qsize` / unwrap `qsize-1`
386
- for _ in chain([None], fs_scheduler, range(qsize - 1)):
387
- # Retrieve done task, exactly `N` calls
388
- obj = _result(get_done_f(), code)
389
- if not isinstance(obj, Some):
390
- raise obj
391
- yield obj.x
388
+ with hide_frame:
389
+ for _ in chain([None], fs_scheduler, range(qsize - 1)):
390
+ # Retrieve done task, exactly `N` calls
391
+ obj = _result(get_done_f())
392
+ if not isinstance(obj, Some):
393
+ with hide_frame:
394
+ raise obj
395
+ yield obj.x
392
396
 
393
397
 
394
398
  def _unwrap[T](
@@ -397,7 +401,6 @@ def _unwrap[T](
397
401
  *,
398
402
  qsize: int | None,
399
403
  order: bool,
400
- code: CodeType,
401
404
  ) -> Iterator[T]:
402
405
  q = SimpleQueue[Future[T]]()
403
406
 
@@ -421,7 +424,7 @@ def _unwrap[T](
421
424
  raise
422
425
 
423
426
  else:
424
- return _get_unwrap_iter(s, qsize, _q_get_fn(q), fs_scheduler, code)
427
+ return _get_unwrap_iter(s, qsize, _q_get_fn(q), fs_scheduler)
425
428
 
426
429
 
427
430
  def _batch_invoke[*Ts, R](
@@ -489,7 +492,6 @@ def starmap_n[T](
489
492
  s = ExitStack()
490
493
  submit = s.enter_context(get_executor(max_workers, mp=mp)).submit
491
494
 
492
- code = func.__code__
493
495
  if mp:
494
496
  func = move_to_shmem(func)
495
497
  else:
@@ -498,7 +500,7 @@ def starmap_n[T](
498
500
  if chunksize == 1:
499
501
  submit_one = cast('Callable[..., Future[T]]', partial(submit, func))
500
502
  f1s = starmap(submit_one, it)
501
- return _unwrap(s, f1s, qsize=prefetch, order=order, code=code)
503
+ return _unwrap(s, f1s, qsize=prefetch, order=order)
502
504
 
503
505
  submit_many = cast(
504
506
  'Callable[..., Future[list[T]]]', partial(submit, _batch_invoke, func)
@@ -513,7 +515,7 @@ def starmap_n[T](
513
515
  # Dynamic chunksize scaling
514
516
  fs = _schedule_auto_v2(submit_many, it)
515
517
 
516
- chunks = _unwrap(s, fs, qsize=prefetch, order=order, code=code)
518
+ chunks = _unwrap(s, fs, qsize=prefetch, order=order)
517
519
  return chain.from_iterable(chunks)
518
520
 
519
521
 
@@ -43,16 +43,25 @@ Reasons not to use alternatives:
43
43
  __all__ = ['arg', 'parse_args']
44
44
 
45
45
  import argparse
46
+ import importlib
47
+ import sys
46
48
  import types
47
49
  from argparse import ArgumentParser, BooleanOptionalAction, _ArgumentGroup
48
- from collections.abc import Callable, Collection, Iterator, Sequence
50
+ from collections.abc import Callable, Iterable, Iterator, Sequence
49
51
  from dataclasses import MISSING, Field, field, fields, is_dataclass
50
52
  from inspect import getmodule, signature, stack
51
- from typing import Any, Literal, Union, get_args, get_origin, get_type_hints
53
+ from typing import (
54
+ Any,
55
+ Literal,
56
+ Required,
57
+ TypedDict,
58
+ Union,
59
+ get_args,
60
+ get_origin,
61
+ get_type_hints,
62
+ )
52
63
 
53
64
  type _Node = str | tuple[str, type, list['_Node']]
54
- _NoneType = type(None)
55
- _UNION_TYPES: list = [Union, types.UnionType]
56
65
 
57
66
 
58
67
  def arg(
@@ -84,41 +93,46 @@ def arg(
84
93
  )
85
94
 
86
95
 
87
- def _unwrap_type(tp: type) -> tuple[type, dict[str, Any]]:
88
- if tp is list:
89
- msg = 'Type list should be parametrized'
90
- raise ValueError(msg)
96
+ class _Opts(TypedDict, total=False):
97
+ type: Required[Callable]
98
+ nargs: str
99
+ choices: Iterable
100
+
91
101
 
102
+ def _unwrap_type(tp: type) -> tuple[type, _Opts]:
92
103
  origin = get_origin(tp)
93
- args = [*get_args(tp)]
104
+ args = get_args(tp)
94
105
  if not origin or not args: # Not a generic type
95
106
  return tp, {'type': tp}
96
107
 
97
108
  if origin is list: # `List[T]`
98
109
  cls, opts = _unwrap_type(args[0])
99
- return cls, opts | {'nargs': argparse.ZERO_OR_MORE}
100
-
101
- # `Optional[T]` or `T | None`
102
- if origin in _UNION_TYPES and len(args) == 2 and _NoneType in args:
103
- args.remove(_NoneType)
104
- cls, opts = _unwrap_type(args[0])
110
+ return cls, {**opts, 'nargs': argparse.ZERO_OR_MORE}
111
+
112
+ if ( # `Optional[T]` or `T | None`
113
+ origin in {Union, types.UnionType}
114
+ and len(args) == 2
115
+ and types.NoneType in args
116
+ ):
117
+ [tp_] = set(args) - {types.NoneType}
118
+ cls, opts = _unwrap_type(tp_)
105
119
  if opts.get('nargs') == argparse.ZERO_OR_MORE:
106
120
  return cls, opts
107
- return cls, opts | {'nargs': argparse.OPTIONAL}
121
+ return cls, {**opts, 'nargs': argparse.OPTIONAL}
108
122
 
109
123
  if origin is Literal: # `Literal[x, y]`
110
124
  choices = get_args(tp)
111
125
  if len(tps := {type(c) for c in choices}) != 1:
112
126
  msg = f'Literal parameters should have the same type. Got: {tps}'
113
- raise ValueError(msg)
114
- (cls,) = tps
127
+ raise TypeError(msg)
128
+ [cls] = tps
115
129
  return cls, {'type': cls, 'choices': choices}
116
130
 
117
131
  msg = (
118
132
  'Only list, Optional and Literal are supported as generic types. '
119
133
  f'Got: {tp}'
120
134
  )
121
- raise ValueError(msg)
135
+ raise TypeError(msg)
122
136
 
123
137
 
124
138
  def _get_fields(fn: Callable) -> Iterator[Field]:
@@ -132,7 +146,7 @@ def _get_fields(fn: Callable) -> Iterator[Field]:
132
146
  raise ValueError(msg)
133
147
  if p.kind in {p.POSITIONAL_ONLY, p.VAR_POSITIONAL, p.VAR_KEYWORD}:
134
148
  msg = f'Unsupported parameter type: {p.kind}'
135
- raise ValueError(msg)
149
+ raise TypeError(msg)
136
150
 
137
151
  if isinstance(p.default, Field):
138
152
  fd = p.default
@@ -192,6 +206,13 @@ def _visit_field(
192
206
  arg_group = parser.add_argument_group(fd.name)
193
207
  return fd.name, cls, _visit_nested(arg_group, cls, seen)
194
208
 
209
+ if (vtp := opts['type']) not in {int, float, str, bool}:
210
+ msg = (
211
+ 'Only bool, int, float and str are supported as value types. '
212
+ f'Got: {vtp}'
213
+ )
214
+ raise TypeError(msg)
215
+
195
216
  snake = fd.name.replace('_', '-')
196
217
  flags = [f] if (f := fd.metadata.get('flag')) else []
197
218
 
@@ -236,7 +257,7 @@ def _visit_field(
236
257
 
237
258
 
238
259
  def _construct[T](
239
- src: dict[str, Any], fn: Callable[..., T], args: Collection[_Node]
260
+ src: dict[str, Any], fn: Callable[..., T], args: Iterable[_Node]
240
261
  ) -> T:
241
262
  kwargs = {}
242
263
  for a in args:
@@ -249,17 +270,55 @@ def _construct[T](
249
270
 
250
271
  def parse_args[T](
251
272
  fn: Callable[..., T],
273
+ /,
252
274
  args: Sequence[str] | None = None,
253
275
  prog: str | None = None,
254
276
  ) -> tuple[T, ArgumentParser]:
255
277
  """Create parser from type hints of callable, parse args and do call."""
256
278
  # TODO: Rename to `run`
279
+ if not callable(fn):
280
+ raise TypeError(f'Expectet callable. Got: {type(fn).__qualname__}')
281
+
257
282
  parser = ArgumentParser(prog)
258
283
  nodes = _visit_nested(parser, fn, {})
259
284
 
260
- if args is not None: # fool's protection
261
- args = line.split(' ') if (line := ' '.join(args).strip()) else []
285
+ assert args is None or (
286
+ isinstance(args, Sequence) and not isinstance(args, str)
287
+ )
262
288
 
263
289
  namespace = parser.parse_args(args)
264
290
  obj = _construct(vars(namespace), fn, nodes)
265
291
  return obj, parser
292
+
293
+
294
+ def _import_from_string(qualname: str):
295
+ modname, _, attrname = qualname.partition(":")
296
+ if not modname or not attrname:
297
+ msg = (
298
+ f'Import string "{qualname}" must be '
299
+ 'in format "<module>:<attribute>".'
300
+ )
301
+ raise ImportError(msg)
302
+
303
+ try:
304
+ mod = importlib.import_module(modname)
305
+ except ModuleNotFoundError as exc:
306
+ if exc.name != modname:
307
+ raise
308
+ msg = f'Could not import module "{modname}".'
309
+ raise ImportError(msg) from None
310
+
311
+ obj: Any = mod
312
+ try:
313
+ for a in attrname.split('.'):
314
+ obj = getattr(obj, a)
315
+ except AttributeError:
316
+ msg = f'Attribute "{attrname}" not found in module "{modname}".'
317
+ raise AttributeError(msg)
318
+ return obj
319
+
320
+
321
+ if __name__ == '__main__':
322
+ qualname, *argv = sys.argv
323
+ obj = _import_from_string(qualname)
324
+ parse_args(obj, argv)
@@ -8,18 +8,23 @@ from glow.cli import parse_args
8
8
 
9
9
 
10
10
  @dataclass
11
- class Arg:
11
+ class Positional:
12
12
  arg: str
13
13
 
14
14
 
15
+ @dataclass
16
+ class UntypedList: # Forbidden, as list field should always be typed
17
+ args: list
18
+
19
+
15
20
  @dataclass
16
21
  class List_: # noqa: N801
17
22
  args: list[str]
18
23
 
19
24
 
20
25
  @dataclass
21
- class UntypedList: # Forbidden, as list field should always be typed
22
- args: list
26
+ class UnsupportedTuple:
27
+ args: tuple[str, int]
23
28
 
24
29
 
25
30
  @dataclass
@@ -54,9 +59,9 @@ class Nested:
54
59
 
55
60
 
56
61
  @dataclass
57
- class NestedArg: # Forbidden, as only top level args can be positional
62
+ class NestedPositional: # Forbidden, as only top level args can be positional
58
63
  arg2: str
59
- nested: Arg
64
+ nested: Positional
60
65
 
61
66
 
62
67
  @dataclass
@@ -73,7 +78,7 @@ class NestedAliased: # Forbidden as all field names must be unique
73
78
  @pytest.mark.parametrize(
74
79
  ('argv', 'expected'),
75
80
  [
76
- (['value'], Arg('value')),
81
+ (['value'], Positional('value')),
77
82
  ([], List_([])),
78
83
  (['a'], List_(['a'])),
79
84
  (['a', 'b'], List_(['a', 'b'])),
@@ -98,12 +103,13 @@ def test_good_class(argv: list[str], expected: Any):
98
103
  @pytest.mark.parametrize(
99
104
  ('cls', 'exc_type'),
100
105
  [
101
- (Arg, SystemExit),
106
+ (Positional, SystemExit),
102
107
  (BadBoolean, ValueError),
103
- (UnsupportedSet, ValueError),
104
- (UntypedList, ValueError),
108
+ (UnsupportedTuple, TypeError),
109
+ (UnsupportedSet, TypeError),
110
+ (UntypedList, TypeError),
105
111
  (Nested, SystemExit),
106
- (NestedArg, ValueError),
112
+ (NestedPositional, ValueError),
107
113
  (NestedAliased, ValueError),
108
114
  ],
109
115
  )
@@ -116,15 +122,15 @@ def _no_op():
116
122
  return ()
117
123
 
118
124
 
119
- def _arg(a: int):
125
+ def _positional(a: int):
120
126
  return a
121
127
 
122
128
 
123
- def _kwarg(a: int = 4):
129
+ def _keyword(a: int = 4):
124
130
  return a
125
131
 
126
132
 
127
- def _kwarg_opt(a: int = None): # type: ignore[assignment] # noqa: RUF013
133
+ def _kw_nullable(a: int = None): # type: ignore[assignment] # noqa: RUF013
128
134
  return a
129
135
 
130
136
 
@@ -140,7 +146,7 @@ def _kwarg_list(a: list[int] = []): # noqa: B006
140
146
  return a
141
147
 
142
148
 
143
- def _kwarg_opt_list(a: list[int] | None = None): # type: ignore[assignment]
149
+ def _kwarg_opt_list(a: list[int] | None = None):
144
150
  return a
145
151
 
146
152
 
@@ -152,11 +158,11 @@ def _arg_kwarg(a: int, b: str = 'hello'):
152
158
  ('argv', 'func', 'expected'),
153
159
  [
154
160
  ([], _no_op, ()),
155
- (['42'], _arg, 42),
156
- ([], _kwarg, 4),
157
- (['--a', '58'], _kwarg, 58),
158
- ([], _kwarg_opt, None),
159
- (['--a', '73'], _kwarg_opt, 73),
161
+ (['42'], _positional, 42),
162
+ ([], _keyword, 4),
163
+ (['--a', '58'], _keyword, 58),
164
+ ([], _kw_nullable, None),
165
+ (['--a', '73'], _kw_nullable, 73),
160
166
  ([], _kwarg_literal, 1),
161
167
  (['--a', '2'], _kwarg_literal, 2),
162
168
  ([], _kwarg_bool, False),
@@ -1,16 +0,0 @@
1
- __all__ = ['declutter_tb']
2
-
3
- from types import CodeType
4
-
5
-
6
- def declutter_tb[E: BaseException](e: E, code: CodeType) -> E:
7
- tb = e.__traceback__
8
-
9
- # Drop outer to `code` frames
10
- while tb:
11
- if tb.tb_frame.f_code is code: # Has reached target frame
12
- e.__traceback__ = tb
13
- return e
14
-
15
- tb = tb.tb_next
16
- return e
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes