thds.mops 3.8.20250709195412__py3-none-any.whl → 3.8.20250711201237__py3-none-any.whl

thds/mops/pure/core/source.py

@@ -215,7 +215,14 @@ class SourceResult(ty.NamedTuple):
     file_uri: str
 
 
-def prepare_source_result(source_: Source) -> SourceResult:
+class DuplicateSourceBasenameError(ValueError):
+    """This is not a catchable error - it will be raised inside the mops result-wrapping
+    code, and is an indication that user code has attempted to return two file-only Source objects
+    without URIs specified, and that those two files have the same basename.
+    """
+
+
+def prepare_source_result(source_: Source, existing_uris: ty.Collection[str] = tuple()) -> SourceResult:
     """Call from within the remote side of an invocation, while serializing the function return value.
 
     Forces the Source to be present at a remote URI which will be available once
@@ -229,8 +236,9 @@ def prepare_source_result(source_: Source) -> SourceResult:
         if source_.cached_path and Path(source_.cached_path).exists():
             # it exists locally - an upload may be necessary.
             file_uri = to_uri(source_.cached_path)
-            lookup_blob_store(source_.uri).putfile(source_.cached_path, source_.uri)
-            logger.info("Uploading Source to %s", source_.uri)
+            if source_.uri not in existing_uris:
+                lookup_blob_store(source_.uri).putfile(source_.cached_path, source_.uri)
+                logger.info("Uploading Source to chosen URI %s", source_.uri)
         else:
             file_uri = ""
             logger.debug("Creating a SourceResult for a URI that is presumed to already be uploaded.")
@@ -249,6 +257,15 @@ def prepare_source_result(source_: Source) -> SourceResult:
     # If users do not like this automatically assigned remote URI name, they can construct
     # the Source themselves and provide a remote URI (as well as, optionally, a
     # local_path), and we will use their remote URI.
+    if remote_uri in existing_uris:
+        raise DuplicateSourceBasenameError(
+            f"Duplicate blob store URI {remote_uri} found in SourceResultPickler."
+            " This is usually an indication that you have two files with the same name in two different directories,"
+            " and are trying to convert them into Source objects with automatically-assigned URIs."
+            " Per the documentation, all output Source objects without explicitly assigned remote URIs must be provided"
+            " with unique basenames, in order to allow retention of the basename for usability and debugging."
+        )
+
     lookup_blob_store(remote_uri).putfile(local_path, remote_uri)
     # upload must _always_ happen on remotely-returned Sources, as detailed above.
     # There is no advantage to waiting to upload past this point.

thds/mops/pure/pickling/_pickle.py

@@ -2,7 +2,6 @@
 data and also functions."""
 
 import io
-import os
 import pickle
 import typing as ty
 from functools import partial
@@ -161,34 +160,20 @@ class SourceArgumentPickler:
         return None
 
 
-class DuplicateSourceBasenameError(ValueError):
-    pass
-
-
 class SourceResultPickler:
     """Only for use on the remote side, when serializing the result."""
 
     def __init__(self) -> None:
         """There will be one of these per remote function call."""
         self._basenames_seen: set[str] = set()
+        # 'basename' is no longer a good name for what is being collected here,
+        # but we are not changing it to preserve backwards compatibility with existing results.
+        # We use this instead to collect URIs that _may_ have been uploaded by `mops`.
 
     def __call__(self, maybe_source: ty.Any) -> ty.Optional[_DeserSource]:
         if isinstance(maybe_source, source.Source):
-            src_res = prepare_source_result(maybe_source)
-            if src_res.file_uri:
-                # we need to check to make sure that this file_uri is not a duplicate
-                # - if it is, this indicates that this single function is attempting to return
-                # two Source objects that have not yet been uploaded but will be uploaded to the same name.
-                file_basename = os.path.basename(src_res.file_uri)
-                if file_basename in self._basenames_seen:
-                    raise DuplicateSourceBasenameError(
-                        f"Duplicate basename {os.path.basename(src_res.file_uri)} found in SourceResultPickler."
-                        " This is usually an indication that you have two files with the same name in two different directories,"
-                        " and are trying to convert them into Source objects with automatically-assigned URIs."
-                        " Per the documentation, all output Source objects without explicitly assigned remote URIs must be provided"
-                        " with unique basenames, in order to allow retention of the basename for usability and debugging."
-                    )
-                self._basenames_seen.add(file_basename)
+            src_res = prepare_source_result(maybe_source, self._basenames_seen)
+            self._basenames_seen.add(src_res.remote_uri)
             return ty.cast(_DeserSource, UnpickleSourceResult(*src_res))
 
         return None

{thds_mops-3.8.20250709195412.dist-info → thds_mops-3.8.20250711201237.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: thds.mops
-Version: 3.8.20250709195412
+Version: 3.8.20250711201237
 Summary: ML Ops tools for Trilliant Health
 Author-email: Trilliant Health <info@trillianthealth.com>
 Project-URL: Repository, https://github.com/TrilliantHealth/ds-monorepo

{thds_mops-3.8.20250709195412.dist-info → thds_mops-3.8.20250711201237.dist-info}/RECORD

@@ -54,7 +54,7 @@ thds/mops/pure/core/pipeline_id_mask.py,sha256=Ll2yyQM5nSgzihx8i7fCrrSNlUUnIsbAO
 thds/mops/pure/core/script_support.py,sha256=3j9Z1O5ynSSPmWSghtJgAj-Lt4GwYcA8cWcpUIRM7q0,952
 thds/mops/pure/core/serialize_big_objs.py,sha256=YcOS1ccs82ZWO7nTbeumErMzYVe4hgXCTsfvMggYmd8,2332
 thds/mops/pure/core/serialize_paths.py,sha256=bWI-AKNP_Tf29JGO7DKqshOh7b7gu51lfGryDXo3aMI,5787
-thds/mops/pure/core/source.py,sha256=7E1e4pdwuuwc1DZP0V7vfU1PSZU8j7PDxyjG3hR9ND8,13622
+thds/mops/pure/core/source.py,sha256=b0i58gE13e25lIV6ls1yPKH67SQ7aCuZmKDEHNr9Ux4,14682
 thds/mops/pure/core/types.py,sha256=w2g83miGhnjaWr2_4TW2Fc3BdIgoIHFbIr_wX1HC7A0,5452
 thds/mops/pure/core/uris.py,sha256=qO9_f-ro7kax6haNOPTPe81-_aUSRFELeeZH4PMTTU4,2694
 thds/mops/pure/core/use_runner.py,sha256=_YeKEjj6_9uc5UIjxcm-YKLUj4joApOdaTJCMaCLC2c,1547
@@ -81,7 +81,7 @@ thds/mops/pure/joblib/__init__.py,sha256=-3hSs-GsNzE_eNnwrdZBHAR_eaub5Uyl5GPYqBw
 thds/mops/pure/joblib/backend.py,sha256=F__6lrdc1-VcX4n4Pw7Lz1bBgeefShtRy2DQh6Fp-eI,2671
 thds/mops/pure/joblib/batching.py,sha256=tPOATD28-YW7KcWa3IqKm-fhLaILzM792ApvU-_zfnM,2298
 thds/mops/pure/pickling/__init__.py,sha256=WNdG8PdJCk-kYaXkvvPa--hjYGoUlBXG3w2X86yuhGo,156
-thds/mops/pure/pickling/_pickle.py,sha256=kZc70_yvEAmIsTjOJuwW8ZpExTL7icAWgGskZF5GSfI,9000
+thds/mops/pure/pickling/_pickle.py,sha256=YB8xbqDiwdk8ccnVZ2_4kQn98V2JSrFqw2E3J-jEHlA,8081
 thds/mops/pure/pickling/memoize_only.py,sha256=oI5CMy6IEJc46Gb_BGWNUuAe3fysS7HxRSTajN0WssI,837
 thds/mops/pure/pickling/mprunner.py,sha256=dVbwQA8hzEL7UiwYXmzoGwN3_jbEtGoHDPMkRmo_UtA,8378
 thds/mops/pure/pickling/pickles.py,sha256=nCg7L7CqReNWDF8FAdEmCcuXVC_kLT5zuyW3V8Vvvs4,4704
@@ -104,8 +104,8 @@ thds/mops/pure/tools/summarize/cli.py,sha256=7kDtn24ok8oBO3jFjlMmOK3jnZYpMoE_5Y8
 thds/mops/pure/tools/summarize/run_summary.py,sha256=LUtvbankAYbss2NCF_XbNl05jkNgxYz_SLyERJlp4sk,5773
 thds/mops/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/mops/testing/deferred_imports.py,sha256=f0ezCgQAtzTqW1yAOb0OWgsB9ZrlztLB894LtpWDaVw,3780
-thds_mops-3.8.20250709195412.dist-info/METADATA,sha256=r3ojo-BZukA6nKQLAProV9qEn7oog-Cir9DjTH8B1fg,2225
-thds_mops-3.8.20250709195412.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-thds_mops-3.8.20250709195412.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
-thds_mops-3.8.20250709195412.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds_mops-3.8.20250709195412.dist-info/RECORD,,
+thds_mops-3.8.20250711201237.dist-info/METADATA,sha256=ngCNk84w-g_ste8xFLBDAGvjdJlIc3w13XompxZy1sQ,2225
+thds_mops-3.8.20250711201237.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+thds_mops-3.8.20250711201237.dist-info/entry_points.txt,sha256=qKvCAaB80syXfxVR3xx6x9J0YJdaQWkIbVSw-NwFgMw,322
+thds_mops-3.8.20250711201237.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds_mops-3.8.20250711201237.dist-info/RECORD,,