relib 1.2.4__py3-none-any.whl → 1.2.5__py3-none-any.whl

relib/__init__.py

@@ -1,4 +1,5 @@
 from .utils import (
+  noop,
   non_none,
   as_any,
   list_split,
@@ -31,8 +32,14 @@ from .utils import (
   df_from_array,
   StrFilter,
   str_filterer,
+)
+from .system import (
+  read_json,
+  write_json,
+  clear_console,
+  console_link,
   roll_tasks,
+  as_async,
 )
-from .system import read_json, write_json, clear_console, console_link
 from .hashing import hash, hash_obj
 from .measure_duration import measure_duration

relib/system.py

@@ -1,7 +1,16 @@
+import asyncio
+import contextvars
+import functools
 import json
 import os
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
-from typing import Any
+from typing import Any, Awaitable, Callable, Iterable, ParamSpec, TypeVar
+from .utils import noop
+
+P = ParamSpec("P")
+R = TypeVar("R")
+default_num_workers = min(32, (os.cpu_count() or 1) + 4)
 
 def read_json(path: Path) -> Any:
   with path.open("r") as f:
@@ -14,5 +23,35 @@ def write_json(path: Path, obj: Any) -> None:
 def clear_console() -> None:
   os.system("cls" if os.name == "nt" else "clear")
 
-def console_link(text, url) -> str:
+def console_link(text: str, url: str) -> str:
   return f"\033]8;;{url}\033\\{text}\033]8;;\033\\"
+
+async def worker[T](task: Awaitable[T], semaphore: asyncio.Semaphore, update=noop) -> T:
+  async with semaphore:
+    result = await task
+    update()
+    return result
+
+async def roll_tasks[T](tasks: Iterable[Awaitable[T]], workers: int, progress=False) -> list[T]:
+  semaphore = asyncio.Semaphore(workers)
+  if not progress:
+    return await asyncio.gather(*(worker(task, semaphore) for task in tasks))
+  from tqdm import tqdm
+  assert isinstance(tasks, (list, tuple, set, frozenset, dict))
+  with tqdm(total=len(tasks)) as pbar:
+    update = lambda: pbar.update(1)
+    return await asyncio.gather(*(worker(task, semaphore, update) for task in tasks))
+
+def as_async(num_workers=default_num_workers) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]:
+  executor = ThreadPoolExecutor(max_workers=num_workers)
+
+  def on_fn(func: Callable[P, R]) -> Callable[P, Awaitable[R]]:
+    @functools.wraps(func)
+    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
+      loop = asyncio.get_running_loop()
+      ctx = contextvars.copy_context()
+      fn_call = functools.partial(ctx.run, func, *args, **kwargs)
+      return await loop.run_in_executor(executor, fn_call)
+    return wrapper
+
+  return on_fn

relib/utils.py

@@ -1,7 +1,9 @@
-import asyncio
 import re
 from itertools import chain
-from typing import Any, Awaitable, Callable, Iterable, overload
+from typing import Any, Callable, Iterable, overload
+
+def noop() -> None:
+  pass
 
 def non_none[T](obj: T | None) -> T:
   assert obj is not None
@@ -215,11 +217,3 @@ def str_filterer(
     return any(pattern.search(string) for pattern in include_patterns)
 
   return str_filter
-
-async def worker(task, semaphore):
-  async with semaphore:
-    return await task
-
-async def roll_tasks[T](tasks: Iterable[Awaitable[T]], workers: int) -> list[T]:
-  semaphore = asyncio.Semaphore(workers)
-  return await asyncio.gather(*(worker(task, semaphore) for task in tasks))

{relib-1.2.4.dist-info → relib-1.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: relib
-Version: 1.2.4
+Version: 1.2.5
 Project-URL: Repository, https://github.com/Reddan/relib.git
 Author: Hampus Hallman
 License: Copyright 2018-2025 Hampus Hallman

relib-1.2.5.dist-info/RECORD

@@ -0,0 +1,9 @@
+relib/__init__.py,sha256=E_ye3ZsGqW3xCibphEWtXhXjIJI58zVxX53VajgvG2Q,674
+relib/hashing.py,sha256=DB_fnkj0ls01FgZbf4nPFHl4EBU8X_0OrmDvty4HlRE,6020
+relib/measure_duration.py,sha256=LCTo_D_qReNprD3fhtJ0daeWycS6xQE_cwxeg2_h0xo,456
+relib/system.py,sha256=YSovHakRl4VjERuoFgYKMZcuSpBVtuQHdjlc_p_Dfr4,1921
+relib/utils.py,sha256=4fM0zsAdfrM2w9Q1YKcTfFG5kDzoNgD2amkgfTG39OY,6933
+relib-1.2.5.dist-info/METADATA,sha256=xy2aVPv-Rhe5P93QDgxJzoJdvKvJoae5upeFwt5SgnE,1295
+relib-1.2.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+relib-1.2.5.dist-info/licenses/LICENSE,sha256=9xVsdtv_-uSyY9Xl9yujwAPm4-mjcCLeVy-ljwXEWbo,1059
+relib-1.2.5.dist-info/RECORD,,

relib-1.2.4.dist-info/RECORD

@@ -1,9 +0,0 @@
-relib/__init__.py,sha256=eEycDD38L7RxEYrMyPvcJ-vWozI_x-mUxvqfbllnAPs,641
-relib/hashing.py,sha256=DB_fnkj0ls01FgZbf4nPFHl4EBU8X_0OrmDvty4HlRE,6020
-relib/measure_duration.py,sha256=LCTo_D_qReNprD3fhtJ0daeWycS6xQE_cwxeg2_h0xo,456
-relib/system.py,sha256=H-SJccCVLNTDhWTT5jo1NFUiQJzHv2Z1xvq0OVrnJcM,431
-relib/utils.py,sha256=kJ8P8zCbp2EhFTu4lpOUe8y8VEvwA7VWUQMkPRmdEIc,7212
-relib-1.2.4.dist-info/METADATA,sha256=XwynajdyZ2ta3_1LdJqpHwkVmj4AreRiStQ-Us-0Msc,1295
-relib-1.2.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-relib-1.2.4.dist-info/licenses/LICENSE,sha256=9xVsdtv_-uSyY9Xl9yujwAPm4-mjcCLeVy-ljwXEWbo,1059
-relib-1.2.4.dist-info/RECORD,,