thds.mops 3.9.20250722200009__py3-none-any.whl → 3.9.20250722213952__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of thds.mops might be problematic. Click here for more details.
- thds/mops/impure/runner.py +1 -1
- thds/mops/k8s/__init__.py +1 -3
- thds/mops/k8s/config.py +1 -1
- thds/mops/k8s/jobs.py +0 -4
- thds/mops/k8s/{_launch.py → launch.py} +57 -56
- thds/mops/k8s/logging.py +5 -37
- thds/mops/k8s/watch.py +62 -120
- thds/mops/pure/__init__.py +1 -2
- thds/mops/pure/_magic/sauce.py +3 -11
- thds/mops/pure/_magic/shims.py +2 -2
- thds/mops/pure/core/deferred_work.py +12 -15
- thds/mops/pure/core/entry/runner_registry.py +10 -1
- thds/mops/pure/core/lock/__init__.py +0 -1
- thds/mops/pure/core/lock/_acquire.py +2 -2
- thds/mops/pure/core/lock/maintain.py +3 -22
- thds/mops/pure/core/lock/write.py +19 -19
- thds/mops/pure/core/memo/__init__.py +1 -1
- thds/mops/pure/core/memo/results.py +4 -5
- thds/mops/pure/core/use_runner.py +7 -21
- thds/mops/pure/pickling/mprunner.py +14 -21
- thds/mops/pure/pickling/pickles.py +8 -19
- thds/mops/pure/pickling/remote.py +1 -3
- thds/mops/pure/runner/local.py +87 -58
- thds/mops/pure/runner/shim_builder.py +7 -7
- thds/mops/pure/runner/simple_shims.py +0 -7
- thds/mops/pure/runner/types.py +4 -15
- thds/mops/pure/tools/summarize/run_summary.py +8 -9
- {thds_mops-3.9.20250722200009.dist-info → thds_mops-3.9.20250722213952.dist-info}/METADATA +1 -1
- {thds_mops-3.9.20250722200009.dist-info → thds_mops-3.9.20250722213952.dist-info}/RECORD +32 -37
- thds/mops/k8s/batching.py +0 -198
- thds/mops/k8s/counts.py +0 -28
- thds/mops/k8s/job_future.py +0 -109
- thds/mops/k8s/uncertain_future.py +0 -160
- thds/mops/pure/runner/get_results.py +0 -106
- {thds_mops-3.9.20250722200009.dist-info → thds_mops-3.9.20250722213952.dist-info}/WHEEL +0 -0
- {thds_mops-3.9.20250722200009.dist-info → thds_mops-3.9.20250722213952.dist-info}/entry_points.txt +0 -0
- {thds_mops-3.9.20250722200009.dist-info → thds_mops-3.9.20250722213952.dist-info}/top_level.txt +0 -0
thds/mops/pure/runner/local.py
CHANGED
|
@@ -5,32 +5,28 @@ import threading
|
|
|
5
5
|
import time
|
|
6
6
|
import typing as ty
|
|
7
7
|
from datetime import datetime, timedelta, timezone
|
|
8
|
-
from functools import partial
|
|
9
8
|
from pathlib import Path
|
|
10
9
|
|
|
11
|
-
from thds.core import
|
|
10
|
+
from thds.core import config, log, scope
|
|
12
11
|
from thds.termtool.colorize import colorized, make_colorized_out
|
|
13
12
|
|
|
14
13
|
from ...config import max_concurrent_network_ops
|
|
15
14
|
from ..core import deferred_work, lock, memo, metadata, pipeline_id_mask, uris
|
|
16
|
-
from ..core.lock.maintain import MAINTAIN_LOCKS # noqa: F401
|
|
17
15
|
from ..core.partial import unwrap_partial
|
|
18
|
-
from ..core.types import Args, Kwargs, T
|
|
16
|
+
from ..core.types import Args, Kwargs, NoResultAfterShimSuccess, T
|
|
19
17
|
from ..tools.summarize import run_summary
|
|
20
18
|
from . import strings, types
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
unwrap_value_or_error,
|
|
26
|
-
)
|
|
27
|
-
|
|
28
|
-
# this semaphore (and a similar one in get_results) allow us to prioritize getting a single unit
|
|
19
|
+
|
|
20
|
+
MAINTAIN_LOCKS = config.item("thds.mops.pure.local.maintain_locks", default=True, parse=config.tobool)
|
|
21
|
+
|
|
22
|
+
# these two semaphores allow us to prioritize getting meaningful units
|
|
29
23
|
# of progress _complete_, rather than issuing many instructions to the
|
|
30
24
|
# underlying client and allowing it to randomly order the operations
|
|
31
25
|
# such that it takes longer to get a full unit of work complete.
|
|
32
26
|
_BEFORE_INVOCATION_SEMAPHORE = threading.BoundedSemaphore(int(max_concurrent_network_ops()))
|
|
33
|
-
#
|
|
27
|
+
# _OUT prioritizes uploading a single invocation and its dependencies so the Shim can start running.
|
|
28
|
+
_AFTER_INVOCATION_SEMAPHORE = threading.BoundedSemaphore(int(max_concurrent_network_ops()))
|
|
29
|
+
# _IN prioritizes retrieving the result of a Shim that has completed.
|
|
34
30
|
|
|
35
31
|
_DarkBlue = colorized(fg="white", bg="#00008b")
|
|
36
32
|
_GreenYellow = colorized(fg="black", bg="#adff2f")
|
|
@@ -48,9 +44,9 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
48
44
|
get_meta_and_result: types.GetMetaAndResult,
|
|
49
45
|
run_directory: ty.Optional[Path] = None,
|
|
50
46
|
calls_registry: ty.Mapping[ty.Callable, ty.Collection[ty.Callable]] = dict(), # noqa: B006
|
|
51
|
-
) -> ty.Callable[[bool, str, ty.Callable[..., T], Args, Kwargs],
|
|
47
|
+
) -> ty.Callable[[bool, str, ty.Callable[..., T], Args, Kwargs], T]:
|
|
52
48
|
@scope.bound
|
|
53
|
-
def
|
|
49
|
+
def create_invocation__check_result__wait_shim(
|
|
54
50
|
rerun_exceptions: bool,
|
|
55
51
|
function_memospace: str,
|
|
56
52
|
# by allowing the caller to set the function memospace, we allow 'redirects' to look up an old result by name.
|
|
@@ -58,7 +54,7 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
58
54
|
func: ty.Callable[..., T],
|
|
59
55
|
args_: Args,
|
|
60
56
|
kwargs_: Kwargs,
|
|
61
|
-
) ->
|
|
57
|
+
) -> T:
|
|
62
58
|
"""This is the generic local runner. Its core abstractions are:
|
|
63
59
|
|
|
64
60
|
- serializers of some sort (for the function and its arguments)
|
|
@@ -93,13 +89,16 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
93
89
|
)
|
|
94
90
|
|
|
95
91
|
# Define some important and reusable 'chunks of work'
|
|
96
|
-
|
|
92
|
+
|
|
93
|
+
class ResultAndInvocationType(ty.NamedTuple):
|
|
94
|
+
value_or_error: ty.Union[memo.results.Success, memo.results.Error]
|
|
95
|
+
invoc_type: run_summary.InvocationType
|
|
96
|
+
|
|
97
|
+
def check_result(
|
|
97
98
|
invoc_type: run_summary.InvocationType,
|
|
98
99
|
) -> ty.Union[ResultAndInvocationType, None]:
|
|
99
100
|
result = memo.results.check_if_result_exists(
|
|
100
|
-
memo_uri,
|
|
101
|
-
check_for_exception=not rerun_exceptions,
|
|
102
|
-
before_raise=debug_required_result_failure,
|
|
101
|
+
memo_uri, rerun_excs=rerun_exceptions, before_raise=debug_required_result_failure
|
|
103
102
|
)
|
|
104
103
|
if not result:
|
|
105
104
|
return None
|
|
@@ -109,6 +108,28 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
109
108
|
)
|
|
110
109
|
return ResultAndInvocationType(result, invoc_type)
|
|
111
110
|
|
|
111
|
+
def unwrap_value_or_error(result_and_itype: ResultAndInvocationType) -> T:
|
|
112
|
+
result = result_and_itype.value_or_error
|
|
113
|
+
metadata = None
|
|
114
|
+
value_t = None
|
|
115
|
+
try:
|
|
116
|
+
if isinstance(result, memo.results.Success):
|
|
117
|
+
metadata, value_t = get_meta_and_result("value", result.value_uri)
|
|
118
|
+
return ty.cast(T, value_t)
|
|
119
|
+
else:
|
|
120
|
+
assert isinstance(result, memo.results.Error), "Must be Error or Success"
|
|
121
|
+
metadata, exc = get_meta_and_result("EXCEPTION", result.exception_uri)
|
|
122
|
+
raise exc
|
|
123
|
+
finally:
|
|
124
|
+
run_summary.log_function_execution(
|
|
125
|
+
*(run_directory, memo_uri, result_and_itype.invoc_type),
|
|
126
|
+
metadata=metadata,
|
|
127
|
+
runner_prefix=function_memospace.split(pipeline_id)[0],
|
|
128
|
+
was_error=not isinstance(result, memo.results.Success),
|
|
129
|
+
return_value=value_t,
|
|
130
|
+
args_kwargs=(args, kwargs),
|
|
131
|
+
)
|
|
132
|
+
|
|
112
133
|
def acquire_lock() -> ty.Optional[lock.LockAcquired]:
|
|
113
134
|
return lock.acquire(fs.join(memo_uri, "lock"), expire=timedelta(seconds=88))
|
|
114
135
|
|
|
@@ -133,14 +154,6 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
133
154
|
|
|
134
155
|
inspect_and_log(memo_uri)
|
|
135
156
|
|
|
136
|
-
p_unwrap_value_or_error = partial(
|
|
137
|
-
unwrap_value_or_error,
|
|
138
|
-
get_meta_and_result,
|
|
139
|
-
run_directory,
|
|
140
|
-
function_memospace.split(pipeline_id)[0], # runner_prefix
|
|
141
|
-
run_summary.extract_source_uris((args, kwargs)),
|
|
142
|
-
)
|
|
143
|
-
|
|
144
157
|
# the network ops being grouped by _BEFORE_INVOCATION include one or more
|
|
145
158
|
# download attempts (consider possible Paths) plus
|
|
146
159
|
# one or more uploads (embedded Paths & Sources/refs, and then invocation).
|
|
@@ -149,9 +162,9 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
149
162
|
|
|
150
163
|
# it's possible that our result may already exist from a previous run of this pipeline id.
|
|
151
164
|
# we can short-circuit the entire process by looking for that result and returning it immediately.
|
|
152
|
-
result =
|
|
165
|
+
result = check_result("memoized")
|
|
153
166
|
if result:
|
|
154
|
-
return
|
|
167
|
+
return unwrap_value_or_error(result)
|
|
155
168
|
|
|
156
169
|
lock_owned = acquire_lock()
|
|
157
170
|
# if no result exists, the vastly most common outcome here will be acquiring
|
|
@@ -162,6 +175,10 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
162
175
|
# LOCK LOOP: entering this loop (where we attempt to acquire the lock) is the common non-memoized case
|
|
163
176
|
while not result:
|
|
164
177
|
if lock_owned:
|
|
178
|
+
if MAINTAIN_LOCKS():
|
|
179
|
+
release_lock = lock.launch_daemon_lock_maintainer(lock_owned)
|
|
180
|
+
else:
|
|
181
|
+
release_lock = lock_owned.release
|
|
165
182
|
break # we own the invocation - invoke the shim ourselves (below)
|
|
166
183
|
|
|
167
184
|
# getting to this point ONLY happens if we failed to acquire the lock, which
|
|
@@ -173,45 +190,57 @@ def invoke_via_shim_or_return_memoized( # noqa: C901
|
|
|
173
190
|
time.sleep(22)
|
|
174
191
|
|
|
175
192
|
with _BEFORE_INVOCATION_SEMAPHORE:
|
|
176
|
-
result =
|
|
193
|
+
result = check_result("awaited")
|
|
177
194
|
if result:
|
|
178
195
|
_LogAwaitedResult(
|
|
179
196
|
f"{val_or_res} for {memo_uri} was found after waiting for the lock."
|
|
180
197
|
)
|
|
181
|
-
return
|
|
198
|
+
return unwrap_value_or_error(result)
|
|
182
199
|
|
|
183
200
|
lock_owned = acquire_lock() # still inside the semaphore, as it's a network op
|
|
184
201
|
|
|
202
|
+
assert release_lock is not None
|
|
185
203
|
assert lock_owned is not None
|
|
186
204
|
# if/when we acquire the lock, we move forever into 'run this ourselves mode'.
|
|
187
205
|
# If something about our invocation fails,
|
|
188
206
|
# we fail just as we would have previously, without any attempt to go
|
|
189
207
|
# 'back' to waiting for someone else to compute the result.
|
|
190
208
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
)
|
|
207
|
-
if hasattr(future_or_shim_result, "add_done_callback"):
|
|
208
|
-
# if the shim returns a Future, we wrap it.
|
|
209
|
-
logger.debug("Shim returned a Future; wrapping it for post-shim result retrieval.")
|
|
210
|
-
return futures.make_lazy(lock_maintaining_future)(
|
|
211
|
-
lock_owned, future_result_getter, future_or_shim_result
|
|
209
|
+
try:
|
|
210
|
+
with _BEFORE_INVOCATION_SEMAPHORE:
|
|
211
|
+
_LogNewInvocation(f"Invoking {memo_uri}")
|
|
212
|
+
upload_invocation_and_deps()
|
|
213
|
+
|
|
214
|
+
# can't hold the semaphore while we block on the shim, though.
|
|
215
|
+
shim_ex = None
|
|
216
|
+
shim = shim_builder(func, args_, kwargs_)
|
|
217
|
+
shim( # ACTUAL INVOCATION (handoff to remote shim) HAPPENS HERE
|
|
218
|
+
(
|
|
219
|
+
memo_uri,
|
|
220
|
+
*metadata.format_invocation_cli_args(
|
|
221
|
+
metadata.InvocationMetadata.new(pipeline_id, invoked_at, lock_owned.writer_id)
|
|
222
|
+
),
|
|
223
|
+
)
|
|
212
224
|
)
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
225
|
+
except Exception as ex:
|
|
226
|
+
# network or similar errors are very common and hard to completely eliminate.
|
|
227
|
+
# We know that if a result (or error) exists, then the network failure is
|
|
228
|
+
# not important, because results in blob storage are atomically populated (either fully there or not)
|
|
229
|
+
logger.exception("Error awaiting shim. Optimistically checking for result.")
|
|
230
|
+
shim_ex = ex
|
|
231
|
+
|
|
232
|
+
finally:
|
|
233
|
+
release_lock()
|
|
234
|
+
|
|
235
|
+
# the network ops being grouped by _AFTER_INVOCATION include one or more downloads.
|
|
236
|
+
with _AFTER_INVOCATION_SEMAPHORE:
|
|
237
|
+
value_or_error = memo.results.check_if_result_exists(memo_uri)
|
|
238
|
+
if not value_or_error:
|
|
239
|
+
if shim_ex:
|
|
240
|
+
raise shim_ex # re-raise the underlying exception rather than making up our own.
|
|
241
|
+
raise NoResultAfterShimSuccess(
|
|
242
|
+
f"The shim for {memo_uri} exited cleanly, but no result or exception was found."
|
|
243
|
+
)
|
|
244
|
+
return unwrap_value_or_error(ResultAndInvocationType(value_or_error, "invoked"))
|
|
245
|
+
|
|
246
|
+
return create_invocation__check_result__wait_shim
|
|
@@ -2,24 +2,24 @@ import inspect
|
|
|
2
2
|
import typing as ty
|
|
3
3
|
|
|
4
4
|
from ..core.types import Args, F, Kwargs
|
|
5
|
-
from .types import
|
|
5
|
+
from .types import Shim, ShimBuilder
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class _static_shim_builder:
|
|
9
|
-
def __init__(self, shim:
|
|
9
|
+
def __init__(self, shim: Shim) -> None:
|
|
10
10
|
self.shim = shim
|
|
11
11
|
|
|
12
|
-
def __call__(self, _f: F, _args: Args, _kwargs: Kwargs) ->
|
|
12
|
+
def __call__(self, _f: F, _args: Args, _kwargs: Kwargs) -> Shim:
|
|
13
13
|
return self.shim
|
|
14
14
|
|
|
15
15
|
def __repr__(self) -> str:
|
|
16
16
|
return f"<static_shim_builder for {self.shim}>"
|
|
17
17
|
|
|
18
18
|
|
|
19
|
-
def make_builder(
|
|
19
|
+
def make_builder(shim: ty.Union[Shim, ShimBuilder]) -> ShimBuilder:
|
|
20
20
|
"""If you have a Shim and you want to make it into the simplest possible ShimBuilder."""
|
|
21
21
|
|
|
22
|
-
if len(inspect.signature(
|
|
23
|
-
return ty.cast(ShimBuilder,
|
|
22
|
+
if len(inspect.signature(shim).parameters) == 3:
|
|
23
|
+
return ty.cast(ShimBuilder, shim)
|
|
24
24
|
|
|
25
|
-
return _static_shim_builder(ty.cast(Shim,
|
|
25
|
+
return _static_shim_builder(ty.cast(Shim, shim))
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import concurrent.futures
|
|
2
1
|
import subprocess
|
|
3
2
|
from typing import Sequence
|
|
4
3
|
|
|
@@ -20,9 +19,3 @@ def samethread_shim(shim_args: Sequence[str]) -> None:
|
|
|
20
19
|
def subprocess_shim(shim_args: Sequence[str]) -> None:
|
|
21
20
|
logger.debug("Running a mops function locally in a new subprocess.")
|
|
22
21
|
subprocess.check_call(["python", "-m", "thds.mops.pure.core.entry.main", *shim_args])
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def future_subprocess_shim(shim_args: Sequence[str]) -> concurrent.futures.Future:
|
|
26
|
-
"""Use this if you really want a Future rather than just running the process"""
|
|
27
|
-
logger.debug("Running a mops function in a new subprocess, returning a Future.")
|
|
28
|
-
return concurrent.futures.ProcessPoolExecutor().submit(samethread_shim, shim_args)
|
thds/mops/pure/runner/types.py
CHANGED
|
@@ -1,36 +1,25 @@
|
|
|
1
1
|
import typing as ty
|
|
2
2
|
|
|
3
|
-
from thds.core import futures
|
|
4
|
-
|
|
5
3
|
from ..core.metadata import ResultMetadata
|
|
6
4
|
from ..core.types import Args, F, Kwargs
|
|
7
5
|
|
|
8
|
-
|
|
9
|
-
SyncShim = ty.Callable[[ty.Sequence[str]], None]
|
|
10
|
-
Shim = ty.Union[SyncShim, FutureShim]
|
|
6
|
+
Shim = ty.Callable[[ty.Sequence[str]], ty.Any]
|
|
11
7
|
"""A runner Shim is a way of getting back into a Python process with enough
|
|
12
8
|
context to download the uploaded function and its arguments from the
|
|
13
9
|
location where a runner placed it, and then invoke the function. All
|
|
14
10
|
arguments are strings because it is assumed that this represents some
|
|
15
11
|
kind of command line invocation.
|
|
16
12
|
|
|
17
|
-
|
|
13
|
+
The Shim must be a blocking call, and its result(s) must be available
|
|
18
14
|
immediately after its return.
|
|
19
|
-
A FutureShim must return a Future (with an 'add_done_callback' method)
|
|
20
|
-
that, when resolved, means that the result(s) are available.
|
|
21
15
|
"""
|
|
22
16
|
|
|
23
|
-
S = ty.TypeVar("S", SyncShim, FutureShim, Shim, covariant=True)
|
|
24
|
-
|
|
25
17
|
|
|
26
|
-
class ShimBuilder(ty.Protocol
|
|
27
|
-
def __call__(self, __f:
|
|
18
|
+
class ShimBuilder(ty.Protocol):
|
|
19
|
+
def __call__(self, __f: F, __args: Args, __kwargs: Kwargs) -> Shim:
|
|
28
20
|
... # pragma: no cover
|
|
29
21
|
|
|
30
22
|
|
|
31
|
-
SyncShimBuilder = ShimBuilder[SyncShim]
|
|
32
|
-
FutureShimBuilder = ShimBuilder[FutureShim]
|
|
33
|
-
|
|
34
23
|
StorageRootURI = str
|
|
35
24
|
SerializeArgsKwargs = ty.Callable[[StorageRootURI, F, Args, Kwargs], bytes]
|
|
36
25
|
SerializeInvocation = ty.Callable[[StorageRootURI, F, bytes], bytes]
|
|
@@ -78,7 +78,7 @@ def _generate_log_filename(
|
|
|
78
78
|
return run_directory / filename
|
|
79
79
|
|
|
80
80
|
|
|
81
|
-
def
|
|
81
|
+
def _extract_source_uris(result: ty.Any) -> ty.Set[str]:
|
|
82
82
|
uris: ty.Set[str] = set()
|
|
83
83
|
|
|
84
84
|
def extract_uri(unknown: ty.Any) -> None:
|
|
@@ -94,16 +94,15 @@ def extract_source_uris(obj: ty.Any) -> ty.Set[str]:
|
|
|
94
94
|
material = unknown.material
|
|
95
95
|
if callable(material):
|
|
96
96
|
mat = material()
|
|
97
|
-
|
|
98
|
-
for uri in extract_source_uris(mat):
|
|
97
|
+
for uri in _extract_source_uris(mat):
|
|
99
98
|
uris.add(uri)
|
|
100
99
|
|
|
101
100
|
try:
|
|
102
|
-
pickle_visit.recursive_visit(extract_uri,
|
|
101
|
+
pickle_visit.recursive_visit(extract_uri, result)
|
|
103
102
|
except pickle.PicklingError:
|
|
104
103
|
pass
|
|
105
104
|
except Exception as exc:
|
|
106
|
-
logger.warning(f'Unexpected error trying to extract URIs from "%s"; {exc}',
|
|
105
|
+
logger.warning(f'Unexpected error trying to extract URIs from "%s"; {exc}', result)
|
|
107
106
|
|
|
108
107
|
return uris
|
|
109
108
|
|
|
@@ -116,7 +115,7 @@ def log_function_execution(
|
|
|
116
115
|
runner_prefix: str = "",
|
|
117
116
|
was_error: bool = False,
|
|
118
117
|
return_value: ty.Any = None,
|
|
119
|
-
|
|
118
|
+
args_kwargs: ty.Any = None,
|
|
120
119
|
) -> None:
|
|
121
120
|
if not run_directory:
|
|
122
121
|
logger.debug("Not writing function summary for %s", memo_uri)
|
|
@@ -146,9 +145,9 @@ def log_function_execution(
|
|
|
146
145
|
log_entry["remote_code_version"] = metadata.remote_code_version
|
|
147
146
|
# we don't bother with invoked_at or remote_started_at because they can be
|
|
148
147
|
# inferred from the timestamp and the wall times
|
|
149
|
-
if
|
|
150
|
-
log_entry["uris_in_args_kwargs"] = sorted(
|
|
151
|
-
if source_uris :=
|
|
148
|
+
if source_uris := _extract_source_uris(args_kwargs):
|
|
149
|
+
log_entry["uris_in_args_kwargs"] = sorted(source_uris)
|
|
150
|
+
if source_uris := _extract_source_uris(return_value):
|
|
152
151
|
log_entry["uris_in_rvalue"] = sorted(source_uris)
|
|
153
152
|
|
|
154
153
|
try:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: thds.mops
|
|
3
|
-
Version: 3.9.
|
|
3
|
+
Version: 3.9.20250722213952
|
|
4
4
|
Summary: ML Ops tools for Trilliant Health
|
|
5
5
|
Author-email: Trilliant Health <info@trillianthealth.com>
|
|
6
6
|
Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo
|
|
@@ -14,41 +14,37 @@ thds/mops/_utils/once.py,sha256=_LHkPbMJO4nqp0RIDj8VgIV3JoZYSQYKGdatdnT-19s,946
|
|
|
14
14
|
thds/mops/_utils/temp.py,sha256=pcHkqIPghPfcZcwAFjCIzLfUWdIgXD3XC8Au1dk5l1k,948
|
|
15
15
|
thds/mops/impure/__init__.py,sha256=VnrHPVgbOYUjkrVnnVicvNV39G6K6ENcWtCuV-BedW4,83
|
|
16
16
|
thds/mops/impure/keyfunc.py,sha256=-THL-GPa6j1zjiHBONkiFzKHuOvE_kGdIVtz0nG4Mp8,524
|
|
17
|
-
thds/mops/impure/runner.py,sha256=
|
|
18
|
-
thds/mops/k8s/__init__.py,sha256=
|
|
19
|
-
thds/mops/k8s/_launch.py,sha256=n4a3v6JNjcShFlBgRyL6JdS9My0ApggsjurtZFuKOnk,10893
|
|
17
|
+
thds/mops/impure/runner.py,sha256=kaBHDLn64FEg_INVwD9_uZk809qyCJyYK8RaYbVUisY,2812
|
|
18
|
+
thds/mops/k8s/__init__.py,sha256=y4bvR5DQLiriBV0I8CZoQdNlNrXWQ2VpcxY5lOrdnzU,669
|
|
20
19
|
thds/mops/k8s/_shared.py,sha256=MR-s6ijWUHZGjxK_fsOpHuRDB6kuofjo5xiIb7ul2VM,86
|
|
21
20
|
thds/mops/k8s/apply_yaml.py,sha256=hVW6dIVbNdzHdbGlc2VAPGkdByv_rH2oPybyIm7tKIM,820
|
|
22
21
|
thds/mops/k8s/auth.py,sha256=mXFPZvyJYEPASsBatv1r8syB9AoayuHGptHHnNUg8LE,1517
|
|
23
|
-
thds/mops/k8s/
|
|
24
|
-
thds/mops/k8s/config.py,sha256=_znocX5BW8kfG_Cbq6f3apx5FqSihD7Tmic-SBkVjMQ,2992
|
|
22
|
+
thds/mops/k8s/config.py,sha256=ha8ppDeFnDB2I9tCajiDcfZlamIk73OJe4lzD5buyXU,2993
|
|
25
23
|
thds/mops/k8s/container_registry.py,sha256=qOiGCE4t_tLYgJDGrhKV9KNv48lF_AlwCDHyFgucd2s,539
|
|
26
|
-
thds/mops/k8s/
|
|
27
|
-
thds/mops/k8s/
|
|
28
|
-
thds/mops/k8s/
|
|
29
|
-
thds/mops/k8s/logging.py,sha256=m_3Tn0XAFhUHdEicqRKnMzdK_hvSTCB7hsP1mIilKC4,10940
|
|
24
|
+
thds/mops/k8s/jobs.py,sha256=3u0jc5Fnll2ncnmcdTUHlcxJ_KYNK9s66W7r6ez49As,3271
|
|
25
|
+
thds/mops/k8s/launch.py,sha256=EjLblGExh0paElOZWgevZdVzEZie1_4jQo7fKTrE1N0,10489
|
|
26
|
+
thds/mops/k8s/logging.py,sha256=m4XnhxzLqlZa2zOObFhp_zv3juQfqfwcCughChBqcCo,9773
|
|
30
27
|
thds/mops/k8s/namespace.py,sha256=Z6trVTU9WFashto4PqIhTcxu-foOF93W0TpgqCU7WIA,383
|
|
31
28
|
thds/mops/k8s/node_selection.py,sha256=Gy2Jz8IxZblg2LmtGg8-MtKI4RmXz2AMXqFPP8OQyu0,2065
|
|
32
29
|
thds/mops/k8s/retry.py,sha256=JVfP304kItpLs5nrONHE5UWkVWlrFGlV_oFQqhq3zHg,2846
|
|
33
30
|
thds/mops/k8s/too_old_resource_version.py,sha256=S7ltVA-LrxUpQ8Q__AB0nQmezN8Mmnx5oKK62_baAKI,1500
|
|
34
|
-
thds/mops/k8s/uncertain_future.py,sha256=60v9yVlhnCDN_yUv8l4Z4KafR4TsTGxN7dprkGI8pQQ,7152
|
|
35
31
|
thds/mops/k8s/wait_job.py,sha256=_X5lSn-3CE4V-_ra0kF1WtxkAiOgqSom8mU1-0hhMio,2445
|
|
36
32
|
thds/mops/k8s/warn_image_backoff.py,sha256=ls_zLSnRbJjO4ICjq1Rk21EXh190l2dT6nKg-PT8Das,1934
|
|
37
|
-
thds/mops/k8s/watch.py,sha256=
|
|
33
|
+
thds/mops/k8s/watch.py,sha256=pLjyJg94QuDYV-CdoEJD7emkrYe5i5kDUTjtUsYSq4w,11425
|
|
38
34
|
thds/mops/k8s/tools/krsync.py,sha256=us7pXX0-bRMwD2oAno7Z6BJcPs6FgaUabHW0STyQJYg,1773
|
|
39
35
|
thds/mops/k8s/tools/krsync.sh,sha256=fWgwkdzWnJeTbzEA_uBiIIi-bNU4nXAYj3dNovyRluU,747
|
|
40
|
-
thds/mops/pure/__init__.py,sha256=
|
|
36
|
+
thds/mops/pure/__init__.py,sha256=kbG0lMvXRBS3LGbb2gPPE9-qjYMXrypyb2tJX2__aZc,1533
|
|
41
37
|
thds/mops/pure/_magic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
38
|
thds/mops/pure/_magic/api.py,sha256=kSlediIZQYsmeHB8plP6osjvUuSEVW4NWdY9ADia12Y,5094
|
|
43
|
-
thds/mops/pure/_magic/sauce.py,sha256=
|
|
44
|
-
thds/mops/pure/_magic/shims.py,sha256=
|
|
39
|
+
thds/mops/pure/_magic/sauce.py,sha256=xmO6Kch-ofVnrVFkxWm84C0-ao9vVCYq0nGGMuYeaok,6333
|
|
40
|
+
thds/mops/pure/_magic/shims.py,sha256=JI49ddv6lEUmNVsEl-XkGlsx2RpOMQoIOSSSfootYE8,1188
|
|
45
41
|
thds/mops/pure/adls/__init__.py,sha256=fw67xxwnizBurScMa-_zWb94lo5gamEVRt27V4bR0jc,54
|
|
46
42
|
thds/mops/pure/adls/_files.py,sha256=9m35Y4elWF0DjgAXVp4oi5CaY6fXWt8n67PilWxWJns,821
|
|
47
43
|
thds/mops/pure/adls/blob_store.py,sha256=ZWr7CKKcI-jz1sWZq4Jwq6LYkhFNxp-EFnNh83EJd84,7374
|
|
48
44
|
thds/mops/pure/adls/output_fqn.py,sha256=qnwdubjVwKShzZ5RruD0_85x86DtPwZNSgwADrdhrTs,748
|
|
49
45
|
thds/mops/pure/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
50
46
|
thds/mops/pure/core/content_addressed.py,sha256=RaCPvtM7bf0NnY5lNR5jPcNn2Moh-bmLtC4zOvdWjCU,1202
|
|
51
|
-
thds/mops/pure/core/deferred_work.py,sha256=
|
|
47
|
+
thds/mops/pure/core/deferred_work.py,sha256=3vjfqWFlqLLMcmX4nHVaaiidrG5N5KyAYhw6R0hoMzI,3716
|
|
52
48
|
thds/mops/pure/core/file_blob_store.py,sha256=N4m4LLrBZaqTJFR4D_eYl03a-n6yQBRsv0ID1bOS9TA,4298
|
|
53
49
|
thds/mops/pure/core/metadata.py,sha256=xAL2iz0pXrcKapmYnNrqSZ8nH2GVakA167NSpAfwiCI,8276
|
|
54
50
|
thds/mops/pure/core/output_naming.py,sha256=ntufOVNJiVPiUM-Azl9mFpDFhIxiB-V2je9dv9AUQhg,2283
|
|
@@ -61,25 +57,25 @@ thds/mops/pure/core/serialize_paths.py,sha256=bWI-AKNP_Tf29JGO7DKqshOh7b7gu51lfG
|
|
|
61
57
|
thds/mops/pure/core/source.py,sha256=b0i58gE13e25lIV6ls1yPKH67SQ7aCuZmKDEHNr9Ux4,14682
|
|
62
58
|
thds/mops/pure/core/types.py,sha256=w2g83miGhnjaWr2_4TW2Fc3BdIgoIHFbIr_wX1HC7A0,5452
|
|
63
59
|
thds/mops/pure/core/uris.py,sha256=qO9_f-ro7kax6haNOPTPe81-_aUSRFELeeZH4PMTTU4,2694
|
|
64
|
-
thds/mops/pure/core/use_runner.py,sha256=
|
|
60
|
+
thds/mops/pure/core/use_runner.py,sha256=_YeKEjj6_9uc5UIjxcm-YKLUj4joApOdaTJCMaCLC2c,1547
|
|
65
61
|
thds/mops/pure/core/entry/__init__.py,sha256=kiDcsj16CwjRSexOZW-4h4b4tDCYIS_eLS5wgu2yIlk,151
|
|
66
62
|
thds/mops/pure/core/entry/main.py,sha256=b1F5lFDK_hnpvW3bqzt5MWDcpKvCXZpWdEHI8zroC4k,2061
|
|
67
63
|
thds/mops/pure/core/entry/route_result.py,sha256=2LcS9M2mYtu56kso0YcMEZbR1mbTWZm0hFlbE2yaf4k,2741
|
|
68
|
-
thds/mops/pure/core/entry/runner_registry.py,sha256=
|
|
69
|
-
thds/mops/pure/core/lock/__init__.py,sha256=
|
|
70
|
-
thds/mops/pure/core/lock/_acquire.py,sha256=
|
|
64
|
+
thds/mops/pure/core/entry/runner_registry.py,sha256=LgEifOYXgjwox1BlBordX1U6g6QB2dAPVU6S1b_iOlk,835
|
|
65
|
+
thds/mops/pure/core/lock/__init__.py,sha256=EP_5T_CF175I01NrVR0eIaWu-k3OM-xsr9pal2i61rU,215
|
|
66
|
+
thds/mops/pure/core/lock/_acquire.py,sha256=wIHvAlh0F4lzfgjga9_eFcfZtzx3G-uCsk06H42OXZA,9093
|
|
71
67
|
thds/mops/pure/core/lock/_funcs.py,sha256=j4g8yVWnrAMPDKqLlq8nTnccM1KHSJ3g71L1iWNbV2Q,969
|
|
72
68
|
thds/mops/pure/core/lock/cli.py,sha256=uidtmgHB2y5LDkj7SQTncy_cNe1EfIseuiJPV9kcxBU,2488
|
|
73
|
-
thds/mops/pure/core/lock/maintain.py,sha256=
|
|
69
|
+
thds/mops/pure/core/lock/maintain.py,sha256=2VkqKxyp0bZkfM5wZV4Lz7zmZl7t5TOcCGNYfJNFGqs,5250
|
|
74
70
|
thds/mops/pure/core/lock/read.py,sha256=Ct5eYMlkTlEaV5Yhw6HWsDD7VrgdhDZoI6AVIQ0ts-4,1255
|
|
75
71
|
thds/mops/pure/core/lock/types.py,sha256=f32t_e2svMOXUVzcnLkEizw6Q47g3HPQsyAkGT2OKMs,993
|
|
76
|
-
thds/mops/pure/core/lock/write.py,sha256=
|
|
77
|
-
thds/mops/pure/core/memo/__init__.py,sha256=
|
|
72
|
+
thds/mops/pure/core/lock/write.py,sha256=4z3W9rsRIs5ZI-_g2Q6ZplQdez6DxCGJ-HZikQI3dHo,5614
|
|
73
|
+
thds/mops/pure/core/memo/__init__.py,sha256=OAgSWsup07EKxITr3yjwJ8eXbhU6-P1DVeZaYIgylgc,277
|
|
78
74
|
thds/mops/pure/core/memo/calls.py,sha256=kvm6kn-CbOLxZuo86BvzEJw69p7VlEJ8_mCiWd6uz-g,3631
|
|
79
75
|
thds/mops/pure/core/memo/function_memospace.py,sha256=PlQCs7dZ2Fu3gIjfzJMeOy7R5zPqYQDBp7OuViLqrpc,11644
|
|
80
76
|
thds/mops/pure/core/memo/keyfunc.py,sha256=FAOEDzMcQ-0JvW4j1eaUzixnemo_373V-16kWZl7_i0,2053
|
|
81
77
|
thds/mops/pure/core/memo/overwrite_params.py,sha256=ltuFxhr8gNo2iBoBz2eFPayjSV23gMdBuoLZD42lIAg,2425
|
|
82
|
-
thds/mops/pure/core/memo/results.py,sha256=
|
|
78
|
+
thds/mops/pure/core/memo/results.py,sha256=kaYJj542Ey5CQgiuCXVGVrKE6ZcdV5H9VD2OCSDi_38,3146
|
|
83
79
|
thds/mops/pure/core/memo/unique_name_for_function.py,sha256=NGuBmK9c-UdgQP27I-WLMRlCMWJmPSdMymRm14mT1K0,2331
|
|
84
80
|
thds/mops/pure/joblib/__init__.py,sha256=-3hSs-GsNzE_eNnwrdZBHAR_eaub5Uyl5GPYqBwEEPo,58
|
|
85
81
|
thds/mops/pure/joblib/backend.py,sha256=F__6lrdc1-VcX4n4Pw7Lz1bBgeefShtRy2DQh6Fp-eI,2671
|
|
@@ -87,17 +83,16 @@ thds/mops/pure/joblib/batching.py,sha256=tPOATD28-YW7KcWa3IqKm-fhLaILzM792ApvU-_
|
|
|
87
83
|
thds/mops/pure/pickling/__init__.py,sha256=WNdG8PdJCk-kYaXkvvPa--hjYGoUlBXG3w2X86yuhGo,156
|
|
88
84
|
thds/mops/pure/pickling/_pickle.py,sha256=YB8xbqDiwdk8ccnVZ2_4kQn98V2JSrFqw2E3J-jEHlA,8081
|
|
89
85
|
thds/mops/pure/pickling/memoize_only.py,sha256=oI5CMy6IEJc46Gb_BGWNUuAe3fysS7HxRSTajN0WssI,837
|
|
90
|
-
thds/mops/pure/pickling/mprunner.py,sha256=
|
|
91
|
-
thds/mops/pure/pickling/pickles.py,sha256=
|
|
92
|
-
thds/mops/pure/pickling/remote.py,sha256=
|
|
86
|
+
thds/mops/pure/pickling/mprunner.py,sha256=dVbwQA8hzEL7UiwYXmzoGwN3_jbEtGoHDPMkRmo_UtA,8378
|
|
87
|
+
thds/mops/pure/pickling/pickles.py,sha256=nCg7L7CqReNWDF8FAdEmCcuXVC_kLT5zuyW3V8Vvvs4,4704
|
|
88
|
+
thds/mops/pure/pickling/remote.py,sha256=SynT9gVE3D2G2KO9oROa1iopMXxCXprP6_A3xl2IEJ4,5921
|
|
93
89
|
thds/mops/pure/pickling/sha256_b64.py,sha256=HL0cPixHPZYuZDVDBscxsnI-3a2amWEfw-LseOX-PyY,2916
|
|
94
90
|
thds/mops/pure/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
95
|
-
thds/mops/pure/runner/
|
|
96
|
-
thds/mops/pure/runner/
|
|
97
|
-
thds/mops/pure/runner/
|
|
98
|
-
thds/mops/pure/runner/simple_shims.py,sha256=r-kLmpSCwzjfzF-Ku43YKvrHMLpZR5jDmweo4Vk07O4,1069
|
|
91
|
+
thds/mops/pure/runner/local.py,sha256=qdAfQVMS5EtZjjrvHfHjiXAQMoVOCxD2rzWdvc2aNbw,12004
|
|
92
|
+
thds/mops/pure/runner/shim_builder.py,sha256=DkOXbPaOWPj2uUsJhjlWmh8ijG9OQc4ciHqa-vHPfXw,709
|
|
93
|
+
thds/mops/pure/runner/simple_shims.py,sha256=oJ8sC5EVD-JFZx8CYE3_QwaQTuFa5F3IYH5PJ9mdMtY,702
|
|
99
94
|
thds/mops/pure/runner/strings.py,sha256=PYAYMxZ2ehgahKIBXJilENNE6OrdNkueNBel8LPsoh8,26
|
|
100
|
-
thds/mops/pure/runner/types.py,sha256=
|
|
95
|
+
thds/mops/pure/runner/types.py,sha256=sdeGCig5a-tm4eHrpMCTFsrmh2CBrLfI3kCMdoYqZY0,1127
|
|
101
96
|
thds/mops/pure/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
102
97
|
thds/mops/pure/tools/_pickle_dis.py,sha256=EyLgWP_dRzz1HIabGRTEGZFT_LZV5gmn4asJyFUAt4Y,6312
|
|
103
98
|
thds/mops/pure/tools/history.py,sha256=dB7C2jq-0P3Fnv5Q3nzEkLehXdX0kaZZrGl1U1ns9DU,1048
|
|
@@ -106,11 +101,11 @@ thds/mops/pure/tools/sha256_b64_addressed.py,sha256=SECAiw3xSqpsrBBZix0MgJRTQrbH
|
|
|
106
101
|
thds/mops/pure/tools/stress.py,sha256=N7C8kLpaGbImeEYlT5jsEl1metvsUu8cnfyQ8vFN0H8,2541
|
|
107
102
|
thds/mops/pure/tools/summarize/__init__.py,sha256=MSmt_5Xg84uHqzTN38JwgseJK8rsJn_11A8WD99VtEo,61
|
|
108
103
|
thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8fmDH8Imc,11524
|
|
109
|
-
thds/mops/pure/tools/summarize/run_summary.py,sha256=
|
|
104
|
+
thds/mops/pure/tools/summarize/run_summary.py,sha256=LUtvbankAYbss2NCF_XbNl05jkNgxYz_SLyERJlp4sk,5773
|
|
110
105
|
thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
111
106
|
thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
|
|
112
|
-
thds_mops-3.9.
|
|
113
|
-
thds_mops-3.9.
|
|
114
|
-
thds_mops-3.9.
|
|
115
|
-
thds_mops-3.9.
|
|
116
|
-
thds_mops-3.9.
|
|
107
|
+
thds_mops-3.9.20250722213952.dist-info/METADATA,sha256=qGHDc94PqxO9WS_21Nl0Vgsuh8X3UkvNnCPM1B96L-o,2225
|
|
108
|
+
thds_mops-3.9.20250722213952.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
109
|
+
thds_mops-3.9.20250722213952.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
|
|
110
|
+
thds_mops-3.9.20250722213952.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
|
|
111
|
+
thds_mops-3.9.20250722213952.dist-info/RECORD,,
|