thds.core 1.31.20250213162956__py3-none-any.whl → 1.32.20250218201534__py3-none-any.whl

thds/core/cm.py

@@ -0,0 +1,29 @@
+"""A keep-alive wrapper for context managers. Let's say you've got a thread pool executor
+that you've created, and you want to be able to pass it to multiple users that expect to
+'enter' the thread pool themselves, using a `with` statement. But you don't want the
+threads to be destroyed after the first use; you want to open the context yourself, but
+still pass the expected context manager to the users. This is a way to do that.
+"""
+
+import contextlib
+import typing as ty
+
+T = ty.TypeVar("T")
+
+
+class _AlreadyEnteredContext(ty.ContextManager[T]):
+    def __init__(self, entered_context: T):
+        self.entered_context = entered_context
+
+    def __enter__(self) -> T:
+        # No-op enter; just return the underlying thing
+        return self.entered_context
+
+    def __exit__(self, exc_type, exc_value, traceback):  # type: ignore
+        pass  # No-op exit
+
+
+@contextlib.contextmanager
+def keep_context(context_manager: ty.ContextManager[T]) -> ty.Iterator[ty.ContextManager[T]]:
+    with context_manager as entered_context:
+        yield _AlreadyEnteredContext(entered_context)

thds/core/files.py

@@ -19,7 +19,7 @@ FILE_SCHEME = "file://"
 logger = getLogger(__name__)
 
 
-def set_read_only(fpath: StrOrPath):
+def set_read_only(fpath: StrOrPath) -> None:
     # thank you https://stackoverflow.com/a/51262451
     logger.debug("Setting '%s' to read-only", fpath)
     perms = stat.S_IMODE(os.lstat(fpath).st_mode)
@@ -87,7 +87,7 @@ def set_file_limit(n: int):
     assert resource.getrlimit(resource.RLIMIT_NOFILE) == (n, n)
 
 
-def bump_limits():
+def bump_limits() -> None:
     """It was common to have to do this manually on our macs. Now that is no longer required."""
     set_file_limit(OPEN_FILES_LIMIT())
 

thds/core/logical_root.py

@@ -0,0 +1,36 @@
+import typing as ty
+
+
+def find_common_prefix(uris: ty.Iterable[str]) -> ty.List[str]:
+    uri_parts_list = [uri.split("/") for uri in uris]
+    if not uri_parts_list:
+        return list()
+
+    reference = uri_parts_list[0]
+
+    for i, part in enumerate(reference):
+        for uri_parts in uri_parts_list[1:]:
+            if i >= len(uri_parts) or uri_parts[i] != part:
+                if i == 0:
+                    raise ValueError(f"Paths have no common prefix: {uris}")
+                return reference[:i]
+    return reference  # the whole thing must be the common prefix
+
+
+def find(paths: ty.Iterable[str], higher_logical_root: str = "") -> str:
+    common = find_common_prefix(paths)
+    if not higher_logical_root:
+        return "/".join(common)  # lowest common root
+
+    # Split higher_logical_root into components
+    root_parts = higher_logical_root.split("/")
+
+    # Look for the sequence of parts in common
+    for i in range(len(common) - len(root_parts) + 1):
+        if common[i : i + len(root_parts)] == root_parts:
+            return "/".join(common[: i + 1])
+        # the logical root is the top level directory that corresponds to the
+        # higher_logical_root string, which may have multiple components that needed to
+        # match. So we take the common parts plus 1 to get that top level root.
+
+    raise ValueError(f"Higher root '{higher_logical_root}' not found")

thds/core/meta.json

@@ -1,8 +1,8 @@
 {
-  "git_commit": "30d8cd75a3d8e8b981101544c57544a36051476c",
+  "git_commit": "6aa6700b782d76b65b65256eaa49bd70e8217015",
   "git_branch": "main",
   "git_is_clean": true,
-  "pyproject_version": "1.31.20250213162956",
+  "pyproject_version": "1.32.20250218201534",
   "thds_user": "runner",
   "misc": {}
 }

thds/core/source/__init__.py

@@ -0,0 +1,9 @@
+"""Wrap openable, read-only data that is either locally-present or downloadable,
+
+yet will not be downloaded (if non-local) until it is actually opened or unwrapped.
+"""
+
+from . import serde, tree  # noqa: F401
+from ._construct import from_file, from_uri, path_from_uri, register_from_uri_handler  # noqa: F401
+from ._download import Downloader, register_download_handler  # noqa: F401
+from .src import Source  # noqa: F401

thds/core/source/_construct.py

@@ -0,0 +1,79 @@
+import typing as ty
+from functools import partial
+from pathlib import Path
+
+from ..files import is_file_uri, path_from_uri, to_uri
+from ..hashing import Hash
+from ..types import StrOrPath
+from . import _download
+from .src import Source
+
+# Creation from local Files or from remote URIs
+
+
+def from_file(filename: StrOrPath, hash: ty.Optional[Hash] = None, uri: str = "") -> Source:
+    """Create a read-only Source from a local file that already exists.
+
+    If URI is passed, the local file will be read and hashed, but the final URI in the
+    Source will be the one provided explicitly. NO UPLOAD IS PERFORMED. It is your
+    responsibility to ensure that your file has been uploaded to the URI you provide.
+    """
+    path = path_from_uri(filename) if isinstance(filename, str) else filename
+    assert isinstance(path, Path)
+    if not path.exists():
+        raise FileNotFoundError(path)
+
+    if uri:
+        src = from_uri(uri, _download._check_hash(hash, path))
+    else:
+        src = Source(to_uri(path), _download._check_hash(hash, path))
+    src._set_cached_path(path)  # internally, it's okay to hack around immutability.
+    return src
+
+
+class FromUri(ty.Protocol):
+    def __call__(self, hash: ty.Optional[Hash]) -> Source:
+        """Closure over a URI that creates a Source from a URI.
+
+        The Hash may be used to short-circuit creation that would result in creating
+        a Source that cannot match the expected Hash, but this is not required,
+        and the hash will be included in the Source object regardless, and will
+        be validated (if non-nil) at the time of source data access.
+        """
+
+
+class FromUriHandler(ty.Protocol):
+    def __call__(self, uri: str) -> ty.Optional[FromUri]:
+        """Returns a FromUri object containing the URI if this URI can be handled.  Returns
+        None if this URI cannot be handled.
+        """
+
+
+def register_from_uri_handler(key: str, handler: FromUriHandler):
+    """If a library wants to customize how Sources are created from URIs that it handles,
+    it can register a handler here.
+    """
+    # key is not currently used for anything other than avoiding
+    # having duplicates registered for whatever reason.
+    _FROM_URI_HANDLERS[key] = handler
+
+
+_FROM_URI_HANDLERS: ty.Dict[str, FromUriHandler] = dict()
+register_from_uri_handler(
+    "local_file", lambda uri: partial(from_file, path_from_uri(uri)) if is_file_uri(uri) else None
+)
+
+
+def from_uri(uri: str, hash: ty.Optional[Hash] = None) -> Source:
+    """Create a read-only Source from a URI. The data should already exist at this remote
+    URI, although Source itself can make no guarantee that it always represents real data
+    - only that any data it does represent is read-only.
+
+    It may be advantageous for a URI-handling library to register a more specific
+    implementation of this function, if it is capable of determining a Hash for the blob
+    represented by the URI without downloading the blob.
+    """
+    for handler in _FROM_URI_HANDLERS.values():
+        if from_uri_ := handler(uri):
+            return from_uri_(hash)
+    return Source(uri=uri, hash=hash)

thds/core/source/_download.py

@@ -0,0 +1,84 @@
+"""Wrap openable, read-only data that is either locally-present or downloadable,
+
+yet will not be downloaded (if non-local) until it is actually opened or unwrapped.
+"""
+
+import typing as ty
+from pathlib import Path
+
+from .. import log
+from ..files import is_file_uri, path_from_uri
+from ..hash_cache import filehash
+from ..hashing import Hash
+
+
+class Downloader(ty.Protocol):
+    def __call__(self, hash: ty.Optional[Hash]) -> Path:
+        """Closure over a URI that downloads a file to a local path and returns the path.
+        The file may be placed anywhere as long as the file will be readable until the
+        program exits.
+
+        If the URI points to a missing file, this MUST raise any Exception that the
+        underlying implementation desires. It MUST NOT return a Path pointing to a
+        non-existent file.
+
+        The Hash may be used to short-circuit a download that would result in downloading
+        a file that does not match the expected hash, but the Downloader need not verify
+        the Hash of the file downloaded after the fact, as that will be performed by
+        default by the Source object.
+        """
+
+
+class DownloadHandler(ty.Protocol):
+    def __call__(self, uri: str) -> ty.Optional[Downloader]:
+        """Returns a Downloader containing the URI if this URI can be handled.  Returns
+        None if this URI cannot be handled.
+        """
+
+
+def _LocalFileHandler(uri: str) -> ty.Optional[Downloader]:
+    if not is_file_uri(uri):
+        return None
+
+    def download_file(hash: ty.Optional[Hash]) -> Path:
+        lpath = path_from_uri(uri)
+        if not lpath.exists():
+            raise FileNotFoundError(lpath)
+        if hash:
+            _check_hash(hash, lpath)
+        return lpath
+
+    return download_file
+
+
+def register_download_handler(key: str, handler: DownloadHandler):
+    # key is not currently used for anything other than avoiding
+    # having duplicates registered for whatever reason.
+    _DOWNLOAD_HANDLERS[key] = handler
+
+
+_DOWNLOAD_HANDLERS: ty.Dict[str, DownloadHandler] = dict()
+register_download_handler("local_file", _LocalFileHandler)
+
+
+def _get_download_handler(uri: str) -> Downloader:
+    for handler in _DOWNLOAD_HANDLERS.values():
+        if downloader := handler(uri):
+            return downloader
+    raise ValueError(f"No SourcePath download handler for uri: {uri}")
+
+
+class SourceHashMismatchError(ValueError):
+    pass
+
+
+def _check_hash(expected_hash: ty.Optional[Hash], path: Path) -> Hash:
+    hash_algo = expected_hash.algo if expected_hash else "sha256"
+    with log.logger_context(hash_for=f"source-{hash_algo}"):
+        computed_hash = filehash(hash_algo, path)
+    if expected_hash and expected_hash != computed_hash:
+        raise SourceHashMismatchError(
+            f"{expected_hash.algo} mismatch for {path};"
+            f" got {computed_hash.bytes!r}, expected {expected_hash.bytes!r}"
+        )
+    return computed_hash

thds/core/source/serde.py

@@ -0,0 +1,105 @@
+# this should later get promoted somewhere, probably
+import json
+import typing as ty
+from functools import partial
+from pathlib import Path
+
+from thds.core import files, hashing, log, types
+
+from . import _construct
+from .src import Source
+
+_SHA256_B64 = "sha256b64"
+_MD5_B64 = "md5b64"
+
+logger = log.getLogger(__name__)
+
+
+def _from_sha256b64(d: dict) -> ty.Optional[hashing.Hash]:
+    if "sha256b64" in d:
+        return hashing.Hash(algo="sha256", bytes=hashing.db64(d[_SHA256_B64]))
+    return None
+
+
+def _from_md5b64(d: dict) -> ty.Optional[hashing.Hash]:
+    if "md5b64" in d:
+        return hashing.Hash(algo="md5", bytes=hashing.db64(d[_MD5_B64]))
+    return None
+
+
+HashParser = ty.Callable[[dict], ty.Optional[hashing.Hash]]
+_BASE_PARSERS = (_from_sha256b64, _from_md5b64)
+
+
+def base_parsers() -> ty.Tuple[HashParser, ...]:
+    return _BASE_PARSERS
+
+
+def from_json(json_source: str, hash_parsers: ty.Collection[HashParser] = base_parsers()) -> Source:
+    d = json.loads(json_source)
+    return _construct.from_uri(
+        uri=d["uri"],
+        hash=next(filter(None, (p(d) for p in hash_parsers)), None),
+    )
+
+
+def _generic_hash_serializer(
+    algo: str, stringify_hash: ty.Callable[[bytes], str], keyname: str, hash: hashing.Hash
+) -> ty.Optional[dict]:
+    if hash.algo == algo:
+        return {keyname: stringify_hash(hash.bytes)}
+    return None
+
+
+_to_sha256b64 = partial(_generic_hash_serializer, "sha256", hashing.b64, _SHA256_B64)
+_to_md5b64 = partial(_generic_hash_serializer, "md5", hashing.b64, _MD5_B64)
+
+HashSerializer = ty.Callable[[hashing.Hash], ty.Optional[dict]]
+_BASE_HASH_SERIALIZERS: ty.Tuple[HashSerializer, ...] = (_to_md5b64, _to_sha256b64)  # type: ignore
+
+
+def base_hash_serializers() -> ty.Tuple[HashSerializer, ...]:
+    return _BASE_HASH_SERIALIZERS
+
+
+def to_json(
+    source: Source, hash_serializers: ty.Collection[HashSerializer] = base_hash_serializers()
+) -> str:
+    hash_dict = (
+        next(filter(None, (ser(source.hash) for ser in hash_serializers if source.hash)), None)
+    ) or dict()
+    return json.dumps(dict(uri=source.uri, **hash_dict))
+
+
+def from_unknown_user_path(path: types.StrOrPath, desired_uri: str) -> Source:
+    """Sometimes you may want to load a Source directly from a Path provided by a user.
+
+    It _might_ represent something loadable as a from_json Source, but it might just be a
+    raw file that needs to be loaded with from_file!
+
+    This is a _reasonable_ (but not guaranteed!) way of trying to ascertain which one it
+    is, and specifying where it should live 'remotely' if such a thing becomes
+    necessary.
+
+    Your application might need to implement something more robust if the
+    actual underlying data is likely to be a JSON blob containing the key `uri`, for
+    instance.
+    """
+    with open(path) as readable:
+        try:
+            return from_json(readable.read(4096))
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            return _construct.from_file(path, uri=desired_uri)
+
+
+def write_to_json_file(source: Source, local_file: Path) -> bool:
+    """Write the canonical JSON serialization of the Source to a file."""
+    local_file.parent.mkdir(parents=True, exist_ok=True)
+    previous_source = local_file.read_text() if local_file.exists() else None
+    new_source = to_json(source) + "\n"
+    if new_source != previous_source:
+        with files.atomic_text_writer(local_file) as f:
+            logger.info(f"Writing {source} to {local_file}")
+            f.write(new_source)
+            return True
+    return False

thds/core/source/src.py

@@ -0,0 +1,86 @@
+import os
+import typing as ty
+from dataclasses import dataclass
+from pathlib import Path
+
+from .. import hashing
+from . import _download
+
+
+@dataclass(frozen=True)
+class Source(os.PathLike):
+    """Source is meant to be a consistent in-memory representation for an abstract,
+    **read-only** source of data that may not be present locally when an application
+    starts.
+
+    A Source uses `os.PathLike` (`__fspath__`) to support transparent `open(src)` calls,
+    so in many cases it will be a drop-in replacement for Path or str filenames. If you
+    need an actual Path object, you can call `path()` to get one, but you should prefer to
+    defer this until the actual location of use.
+
+    By 'wrapping' read-only data in these objects, we can unify the code around how we
+    unwrap and use them, which should allow us to more easily support different execution
+    environments and sources of data.
+
+    For instance, a Source could be a file on disk, but it could also be a file in
+    ADLS.
+
+    Furthermore, libraries which build on top of this one may use this representation to
+    identify opportunities for optimization, by representing the Source in a stable
+    and consistent format that allows different underlying data sources to fulfill the
+    request for the data based on environmental context. A library could choose to
+    transparently transform a local-path-based Source into a Source representing a
+    remote file, without changing the semantics of the object as observed by the code.
+
+    One reason a Hash is part of the interface is so that libraries interacting with the
+    object can use the hash as a canonical 'name' for the data, if one is available.
+
+    Another reason is that we can add a layer of consistency checking to data we're
+    working with, at the cost of a few compute cycles. Since Sources are meant to represent
+    read-only data, the Hash is a meaningful and persistent marker of data identity.
+
+    Do not call its constructor in application code. Use `from_file` or `from_uri` instead.
+    """
+
+    uri: str
+    hash: ty.Optional[hashing.Hash] = None
+    # hash and equality are based only on the _identity_ of the object,
+    # not on the other properties that provide some caching functionality.
+
+    @property
+    def cached_path(self) -> ty.Optional[Path]:
+        """This is part of the public interface as far as checking to see whether a file
+        is already present locally, but its existence and value is not part of equality or
+        the hash for this class - it exists purely as an optimization.
+        """
+        return getattr(self, "__cached_path", None)
+
+    def _set_cached_path(self, lpath: ty.Optional[Path]):
+        """protected interface for setting a cached Path since the attribute is not
+        available via the constructor.
+        """
+        super().__setattr__("__cached_path", lpath)  # this works around dataclass.frozen.
+        # https://noklam.github.io/blog/posts/2022-04-22-python-dataclass-partiala-immutable.html
+
+    def path(self) -> Path:
+        """Any Source can be turned into a local file path.
+
+        Remember that the resulting data is meant to be read-only. If you want to mutate
+        the data, you should first make a copy.
+
+        If not already present locally, this will incur a one-time download. Then, if the
+        Source has a Hash, the Hash will be validated against the downloaded file, and a
+        failure will raise SourceHashMismatchError.
+        """
+        if self.cached_path is None or not self.cached_path.exists():
+            lpath = _download._get_download_handler(self.uri)(self.hash)
+            # path() used to be responsible for checking the hash, but since we pass it to the downloader,
+            # it really makes more sense to allow the downloader to decide how to verify its own download,
+            # and we don't want to duplicate any effort that it may have already put in.
+            self._set_cached_path(lpath)
+
+        assert self.cached_path and self.cached_path.exists()
+        return self.cached_path
+
+    def __fspath__(self) -> str:
+        return os.fspath(self.path())

thds/core/source/tree.py

@@ -0,0 +1,106 @@
+import concurrent.futures
+import os
+import shutil
+import typing as ty
+from dataclasses import dataclass
+from pathlib import Path
+
+from .. import cm, link, logical_root, parallel, thunks, types
+from .src import Source
+
+_MAX_PARALLELISM = 90
+
+
+def _logical_tree_replication_operations(
+    local_paths: ty.Iterable[Path], logical_local_root: Path, dest_dir: Path
+) -> ty.Tuple[Path, ty.List[ty.Tuple[Path, Path]]]:
+    """
+    Pure function that determines required copy operations.
+    Returns (logical_dest, list of (src, dest) pairs)
+    """
+    logical_dest = dest_dir / logical_local_root.name
+    operations = [(src, logical_dest / src.relative_to(logical_local_root)) for src in local_paths]
+    return logical_dest, operations
+
+
+def replicate_logical_tree(
+    local_paths: ty.Iterable[Path],
+    logical_local_root: Path,
+    dest_dir: Path,
+    copy: ty.Callable[[Path, Path], ty.Any] = link.cheap_copy,
+    executor_cm: ty.Optional[ty.ContextManager[concurrent.futures.Executor]] = None,
+) -> Path:
+    """
+    Replicate only the specified files from logical_root into dest_dir.
+    Returns the path to the logical root in the new location.
+    """
+    logical_dest, operations = _logical_tree_replication_operations(
+        local_paths, logical_local_root, dest_dir
+    )
+
+    top_level_of_logical_dest_dir = dest_dir / logical_local_root.name
+    shutil.rmtree(top_level_of_logical_dest_dir, ignore_errors=True)
+
+    def copy_to(src: Path, dest: Path) -> None:
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        copy(src, dest)
+
+    for _ in parallel.failfast(
+        parallel.yield_all(
+            ((src, thunks.thunking(copy_to)(src, dest)) for src, dest in operations),
+            executor_cm=executor_cm,
+        )
+    ):
+        pass
+    return top_level_of_logical_dest_dir
+
+
+@dataclass
+class SourceTree(os.PathLike):
+    """Represent a fixed set of sources (with hashes where available) as a list of
+    sources, plus the (optional) logical root of the tree, so that they can be 'unwrapped'
+    as a local directory structure.
+    """
+
+    sources: ty.List[Source]
+    higher_logical_root: str = ""
+    # there may be cases where, rather than identifying the 'lowest common prefix' of a
+    # set of sources/URIs, we may wish to represent a 'higher' root for the sake of some
+    # consuming system.  in those cases, this can be specified and we'll find the lowest
+    # common prefix _above_ that.
+
+    def path(self, dest_dir: ty.Optional[types.StrOrPath] = None) -> Path:
+        """Return a local path to a directory that corresponds to the logical root.
+
+        This incurs a download of _all_ sources explicitly represented by the list.
+
+        If you want to _ensure_ that _only_ the listed sources are present in the
+        directory, despite any other files which may be present in an
+        implementation-specific cache, you must pass a Path to a directory that you are
+        willing to have emptied, and this method will copy the files into it.
+        """
+        with cm.keep_context(
+            concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_PARALLELISM)
+        ) as thread_pool:
+            local_paths = [
+                local_path
+                for _, local_path in parallel.failfast(
+                    parallel.yield_all(
+                        # src.path() is a thunk that downloads the data if not already present locally.
+                        # Source allows registration of download handlers by URI scheme.
+                        ((src, src.path) for src in self.sources),
+                        executor_cm=thread_pool,
+                    )
+                )
+            ]
+            local_logical_root = Path(logical_root.find(map(str, local_paths), self.higher_logical_root))
+            assert local_logical_root.is_dir()
+            if not dest_dir:
+                return local_logical_root
+
+            return replicate_logical_tree(
+                local_paths, local_logical_root, Path(dest_dir).resolve(), executor_cm=thread_pool
+            )
+
+    def __fspath__(self) -> str:  # implement the os.PathLike protocol
+        return str(self.path())

thds/core/source_serde.py

@@ -1,104 +1,2 @@
-# this should later get promoted somewhere, probably
-import json
-import typing as ty
-from functools import partial
-from pathlib import Path
-
-from thds.core import files, hashing, log, source, types
-
-_SHA256_B64 = "sha256b64"
-_MD5_B64 = "md5b64"
-
-logger = log.getLogger(__name__)
-
-
-def _from_sha256b64(d: dict) -> ty.Optional[hashing.Hash]:
-    if "sha256b64" in d:
-        return hashing.Hash(algo="sha256", bytes=hashing.db64(d[_SHA256_B64]))
-    return None
-
-
-def _from_md5b64(d: dict) -> ty.Optional[hashing.Hash]:
-    if "md5b64" in d:
-        return hashing.Hash(algo="md5", bytes=hashing.db64(d[_MD5_B64]))
-    return None
-
-
-HashParser = ty.Callable[[dict], ty.Optional[hashing.Hash]]
-_BASE_PARSERS = (_from_sha256b64, _from_md5b64)
-
-
-def base_parsers() -> ty.Tuple[HashParser, ...]:
-    return _BASE_PARSERS
-
-
-def from_json(
-    json_source: str, hash_parsers: ty.Collection[HashParser] = base_parsers()
-) -> source.Source:
-    d = json.loads(json_source)
-    return source.from_uri(
-        uri=d["uri"],
-        hash=next(filter(None, (p(d) for p in hash_parsers)), None),
-    )
-
-
-def _generic_hash_serializer(
-    algo: str, stringify_hash: ty.Callable[[bytes], str], keyname: str, hash: hashing.Hash
-) -> ty.Optional[dict]:
-    if hash.algo == algo:
-        return {keyname: stringify_hash(hash.bytes)}
-    return None
-
-
-_to_sha256b64 = partial(_generic_hash_serializer, "sha256", hashing.b64, _SHA256_B64)
-_to_md5b64 = partial(_generic_hash_serializer, "md5", hashing.b64, _MD5_B64)
-
-HashSerializer = ty.Callable[[hashing.Hash], ty.Optional[dict]]
-_BASE_HASH_SERIALIZERS: ty.Tuple[HashSerializer, ...] = (_to_md5b64, _to_sha256b64)  # type: ignore
-
-
-def base_hash_serializers() -> ty.Tuple[HashSerializer, ...]:
-    return _BASE_HASH_SERIALIZERS
-
-
-def to_json(
-    source: source.Source, hash_serializers: ty.Collection[HashSerializer] = base_hash_serializers()
-) -> str:
-    hash_dict = (
-        next(filter(None, (ser(source.hash) for ser in hash_serializers if source.hash)), None)
-    ) or dict()
-    return json.dumps(dict(uri=source.uri, **hash_dict))
-
-
-def from_unknown_user_path(path: types.StrOrPath, desired_uri: str) -> source.Source:
-    """Sometimes you may want to load a Source directly from a Path provided by a user.
-
-    It _might_ represent something loadable as a from_json Source, but it might just be a
-    raw file that needs to be loaded with from_file!
-
-    This is a _reasonable_ (but not guaranteed!) way of trying to ascertain which one it
-    is, and specifying where it should live 'remotely' if such a thing becomes
-    necessary.
-
-    Your application might need to implement something more robust if the
-    actual underlying data is likely to be a JSON blob containing the key `uri`, for
-    instance.
-    """
-    with open(path) as readable:
-        try:
-            return from_json(readable.read(4096))
-        except (json.JSONDecodeError, UnicodeDecodeError):
-            return source.from_file(path, uri=desired_uri)
-
-
-def write_to_json_file(source: source.Source, local_file: Path) -> bool:
-    """Write the canonical JSON serialization of the Source to a file."""
-    local_file.parent.mkdir(parents=True, exist_ok=True)
-    previous_source = local_file.read_text() if local_file.exists() else None
-    new_source = to_json(source) + "\n"
-    if new_source != previous_source:
-        with files.atomic_text_writer(local_file) as f:
-            logger.info(f"Writing {source} to {local_file}")
-            f.write(new_source)
-            return True
-    return False
+# deprecated alias for source.serde
+from .source.serde import from_json, from_unknown_user_path, to_json, write_to_json_file  # noqa: F401

{thds.core-1.31.20250213162956.dist-info → thds.core-1.32.20250218201534.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: thds.core
-Version: 1.31.20250213162956
+Version: 1.32.20250218201534
 Summary: Core utilities.
 Author: Trilliant Health
 Description-Content-Type: text/markdown

{thds.core-1.31.20250213162956.dist-info → thds.core-1.32.20250218201534.dist-info}/RECORD

@@ -2,13 +2,14 @@ thds/core/__init__.py,sha256=imKpmnrBV0_7-1d1Pc2yR5jxbvOrmIplLm7Ig_eK1OU,934
 thds/core/ansi_esc.py,sha256=QZ3CptZbX4N_hyP2IgqfTbNt9tBPaqy7ReTMQIzGbrc,870
 thds/core/cache.py,sha256=nL0oAyZrhPqyBBLevnOWSWVoEBrftaG3aE6Qq6tvmAA,7153
 thds/core/calgitver.py,sha256=HklIz-SczK92Vm2rXtTSDiVxAcxUW_GPVCRRGt4BmBA,2324
+thds/core/cm.py,sha256=WZB8eQU0DaBYj9s97nc3PuCtai9guovfyiQH68zhLzY,1086
 thds/core/concurrency.py,sha256=NQunF_tJ_z8cfVyhzkTPlb-nZrgu-vIk9_3XffgscKQ,3520
 thds/core/config.py,sha256=N-WVpPDrfTSFKz0m7WrqZPBdd17dycpDx9nhbATkf3c,9092
 thds/core/decos.py,sha256=VpFTKTArXepICxN4U8C8J6Z5KDq-yVjFZQzqs2jeVAk,1341
 thds/core/dict_utils.py,sha256=MAVkGJg4KQN1UGBLEKuPdQucZaXg_jJakujQ-GUrYzw,6471
 thds/core/env.py,sha256=M36CYkPZ5AUf_-n8EqjsMGwWOzaKEn0KgRwnqUK7jS4,1094
 thds/core/exit_after.py,sha256=0lz63nz2NTiIdyBDYyRa9bQShxQKe7eISy8VhXeW4HU,3485
-thds/core/files.py,sha256=8amNymTRuW17o6Qay73juxwkcW1DysWXeAByeofadkM,4540
+thds/core/files.py,sha256=35vhbaDv4OkL_n1PCM-ki54708aibFMrnURpth_5UsA,4556
 thds/core/fretry.py,sha256=Tui2q6vXV6c7mjTa1czLrXiugHUEwQp-sZdiwXfxvmM,3829
 thds/core/generators.py,sha256=rcdFpPj0NMJWSaSZTnBfTeZxTTORNB633Lng-BW1284,1939
 thds/core/git.py,sha256=I6kaEvwcvVxCLYHhTTfnHle-GkmgOR9_fHs03QxgBfI,2792
@@ -21,8 +22,9 @@ thds/core/inspect.py,sha256=vCxKqw8XG2W1cuj0MwjdXhe9TLQrGdjRraS6UEYsbf8,1955
 thds/core/iterators.py,sha256=d3iTQDR0gCW1nMRmknQeodR_4THzR9Ajmp8F8KCCFgg,208
 thds/core/lazy.py,sha256=e1WvG4LsbEydV0igEr_Vl1cq05zlQNIE8MFYT90yglE,3289
 thds/core/link.py,sha256=kmFJIFvEZc16-7S7IGvtTpzwl3VuvFl3yPlE6WJJ03w,5404
+thds/core/logical_root.py,sha256=gWkIYRv9kNQfzbpxJaYiwNXVz1neZ2NvnvProtOn9d8,1399
 thds/core/merge_args.py,sha256=7oj7dtO1-XVkfTM3aBlq3QlZbo8tb6X7E3EVIR-60t8,5781
-thds/core/meta.json,sha256=HhmqF-oKGnmptGrl7rwZkxD9SzwePtjo82gyFSAfp48,196
+thds/core/meta.json,sha256=yYe9d8_WiKFL5-fGrUD1hBo8tCJtbY-wet_cGdgcy_A,196
 thds/core/meta.py,sha256=IPLAKrH06HooPMNf5FeqJvUcM-JljTGXddrAQ5oAX8E,16896
 thds/core/parallel.py,sha256=HXAn9aIYqNE5rnRN5ypxR6CUucdfzE5T5rJ_MUv-pFk,7590
 thds/core/pickle_visit.py,sha256=QNMWIi5buvk2zsvx1-D-FKL7tkrFUFDs387vxgGebgU,833
@@ -32,8 +34,7 @@ thds/core/protocols.py,sha256=4na2EeWUDWfLn5-SxfMmKegDSndJ5z-vwMhDavhCpEM,409
 thds/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 thds/core/scaling.py,sha256=f7CtdgK0sN6nroTq5hLAkG8xwbWhbCZUULstSKjoxO0,1615
 thds/core/scope.py,sha256=iPRhS-lIe-axDctqxBtEPeF0PM_w-0tRS-9kPweUGBY,7205
-thds/core/source.py,sha256=ACMe_e-0tuwUknOobomC9_q6vHM3hMTouWsSsJ-qHQk,9884
-thds/core/source_serde.py,sha256=TqW4MTrXQ49JJaXrkmFcymSBIWvBlkxO2JOPNQap_F4,3523
+thds/core/source_serde.py,sha256=X4c7LiT3VidejqtTel9YB6dWGB3x-ct39KF9E50Nbx4,139
 thds/core/stack_context.py,sha256=17lPOuYWclUpZ-VXRkPgI4WbiMzq7_ZY6Kj1QK_1oNo,1332
 thds/core/thunks.py,sha256=p1OvMBJ4VGMsD8BVA7zwPeAp0L3y_nxVozBF2E78t3M,1053
 thds/core/timer.py,sha256=1FfcQ4-Gp6WQFXR0GKeT_8jwtamEfnTukdSbDKTAJVM,5432
@@ -45,6 +46,12 @@ thds/core/log/json_formatter.py,sha256=C5bRsSbAqaQqfTm88jc3mYe3vwKZZLAxET8s7_u7a
 thds/core/log/kw_formatter.py,sha256=9-MVOd2r5NEkYNne9qWyFMeR5lac3w7mjHXsDa681i0,3379
 thds/core/log/kw_logger.py,sha256=CyZVPnkUMtrUL2Lyk261AIEPmoP-buf_suFAhQlU1io,4063
 thds/core/log/logfmt.py,sha256=qS6BbdlOZPRnxmHenVL3uK43OQ30NJUnz92S6d_Yh2A,10360
+thds/core/source/__init__.py,sha256=RiaUHNunoaw4XJUrwR5vJzSS6HGxOUKUONR_ipX5654,424
+thds/core/source/_construct.py,sha256=klN6-fSJrsbbUhp92wzhJcF73h_PKKJItNLC__vwlIs,3122
+thds/core/source/_download.py,sha256=pUhkphHdB7y4ZpxZZ6ITIS5giXMHuRf420yYAJwx6aE,2924
+thds/core/source/serde.py,sha256=wXCfuv_Dv3QvJJr-uebGmTrfhCU_1a8VX3VJnXhVHfU,3539
+thds/core/source/src.py,sha256=A1PSR5vANLwnUWLsFNVLkkeUdaidzRAzq8vri_a5w9E,4141
+thds/core/source/tree.py,sha256=vjAqnQXGE0XiI0WvlLyXGqEAZbyjq6XmdUeWAR0HI4M,4144
 thds/core/sqlite/__init__.py,sha256=tDMzuO76qTtckJHldPQ6nPZ6kcvhhoJrVuuW42JtaSQ,606
 thds/core/sqlite/connect.py,sha256=l4QaSAI8RjP7Qh2FjmJ3EwRgfGf65Z3-LjtC9ocHM_U,977
 thds/core/sqlite/copy.py,sha256=y3IRQTBrWDfKuVIfW7fYuEgwRCRKHjN0rxVFkIb9VrQ,1155
@@ -60,8 +67,8 @@ thds/core/sqlite/structured.py,sha256=swCbDoyVT6cE7Kl79Wh_rg5Z1-yrUDJbiVJF4bjset
 thds/core/sqlite/types.py,sha256=oUkfoKRYNGDPZRk29s09rc9ha3SCk2SKr_K6WKebBFs,1308
 thds/core/sqlite/upsert.py,sha256=BmKK6fsGVedt43iY-Lp7dnAu8aJ1e9CYlPVEQR2pMj4,5827
 thds/core/sqlite/write.py,sha256=z0219vDkQDCnsV0WLvsj94keItr7H4j7Y_evbcoBrWU,3458
-thds.core-1.31.20250213162956.dist-info/METADATA,sha256=NVfvS6EbBrUw_-SsR3Ckb87WNCQ9LPcXiX_ukJ5Gep4,2123
-thds.core-1.31.20250213162956.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-thds.core-1.31.20250213162956.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
-thds.core-1.31.20250213162956.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
-thds.core-1.31.20250213162956.dist-info/RECORD,,
+thds.core-1.32.20250218201534.dist-info/METADATA,sha256=fBdWYnrnIKHNioHwRHhFjWY_tUm-atCCLoHJgdoSqds,2123
+thds.core-1.32.20250218201534.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+thds.core-1.32.20250218201534.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
+thds.core-1.32.20250218201534.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
+thds.core-1.32.20250218201534.dist-info/RECORD,,

thds/core/source.py

@@ -1,238 +0,0 @@
-"""Wrap openable, read-only data that is either locally-present or downloadable,
-
-yet will not be downloaded (if non-local) until it is actually opened or unwrapped.
-"""
-
-import os
-import typing as ty
-from dataclasses import dataclass
-from functools import partial
-from pathlib import Path
-
-from . import log
-from .files import is_file_uri, path_from_uri, to_uri
-from .hash_cache import filehash
-from .hashing import Hash
-from .types import StrOrPath
-
-
-class Downloader(ty.Protocol):
-    def __call__(self, hash: ty.Optional[Hash]) -> Path:
-        """Closure over a URI that downloads a file to a local path and returns the path.
-        The file may be placed anywhere as long as the file will be readable until the
-        program exits.
-
-        If the URI points to a missing file, this MUST raise any Exception that the
-        underlying implementation desires. It MUST NOT return a Path pointing to a
-        non-existent file.
-
-        The Hash may be used to short-circuit a download that would result in downloading
-        a file that does not match the expected hash, but the Downloader need not verify
-        the Hash of the file downloaded after the fact, as that will be performed by
-        default by the Source object.
-        """
-
-
-class DownloadHandler(ty.Protocol):
-    def __call__(self, uri: str) -> ty.Optional[Downloader]:
-        """Returns a Downloader containing the URI if this URI can be handled.  Returns
-        None if this URI cannot be handled.
-        """
-
-
-def _LocalFileHandler(uri: str) -> ty.Optional[Downloader]:
-    if not is_file_uri(uri):
-        return None
-
-    def download_file(hash: ty.Optional[Hash]) -> Path:
-        lpath = path_from_uri(uri)
-        if not lpath.exists():
-            raise FileNotFoundError(lpath)
-        if hash:
-            _check_hash(hash, lpath)
-        return lpath
-
-    return download_file
-
-
-def register_download_handler(key: str, handler: DownloadHandler):
-    # key is not currently used for anything other than avoiding
-    # having duplicates registered for whatever reason.
-    _DOWNLOAD_HANDLERS[key] = handler
-
-
-_DOWNLOAD_HANDLERS: ty.Dict[str, DownloadHandler] = dict()
-register_download_handler("local_file", _LocalFileHandler)
-
-
-def _get_download_handler(uri: str) -> Downloader:
-    for handler in _DOWNLOAD_HANDLERS.values():
-        if downloader := handler(uri):
-            return downloader
-    raise ValueError(f"No SourcePath download handler for uri: {uri}")
-
-
-class SourceHashMismatchError(ValueError):
-    pass
-
-
-def _check_hash(expected_hash: ty.Optional[Hash], path: Path) -> Hash:
-    hash_algo = expected_hash.algo if expected_hash else "sha256"
-    with log.logger_context(hash_for=f"source-{hash_algo}"):
-        computed_hash = filehash(hash_algo, path)
-    if expected_hash and expected_hash != computed_hash:
-        raise SourceHashMismatchError(
-            f"{expected_hash.algo} mismatch for {path};"
-            f" got {computed_hash.bytes!r}, expected {expected_hash.bytes!r}"
-        )
-    return computed_hash
-
-
-@dataclass(frozen=True)
-class Source(os.PathLike):
-    """Source is meant to be a consistent in-memory representation for an abstract,
-    **read-only** source of data that may not be present locally when an application
-    starts.
-
-    A Source uses `os.PathLike` (`__fspath__`) to support transparent `open(src)` calls,
-    so in many cases it will be a drop-in replacement for Path or str filenames. If you
-    need an actual Path object, you can call `path()` to get one, but you should prefer to
-    defer this until the actual location of use.
-
-    By 'wrapping' read-only data in these objects, we can unify the code around how we
-    unwrap and use them, which should allow us to more easily support different execution
-    environments and sources of data.
-
-    For instance, a Source could be a file on disk, but it could also be a file in
-    ADLS.
-
-    Furthermore, libraries which build on top of this one may use this representation to
-    identify opportunities for optimization, by representing the Source in a stable
-    and consistent format that allows different underlying data sources to fulfill the
-    request for the data based on environmental context. A library could choose to
-    transparently transform a local-path-based Source into a Source representing a
-    remote file, without changing the semantics of the object as observed by the code.
-
-    One reason a Hash is part of the interface is so that libraries interacting with the
-    object can use the hash as a canonical 'name' for the data, if one is available.
-
-    Another reason is that we can add a layer of consistency checking to data we're
-    working with, at the cost of a few compute cycles. Since Sources are meant to represent
-    read-only data, the Hash is a meaningful and persistent marker of data identity.
-
-    Do not call its constructor in application code. Use `from_file` or `from_uri` instead.
-    """
-
-    uri: str
-    hash: ty.Optional[Hash] = None
-    # hash and equality are based only on the _identity_ of the object,
-    # not on the other properties that provide some caching functionality.
-
-    @property
-    def cached_path(self) -> ty.Optional[Path]:
-        """This is part of the public interface as far as checking to see whether a file
-        is already present locally, but its existence and value is not part of equality or
-        the hash for this class - it exists purely as an optimization.
-        """
-        return getattr(self, "__cached_path", None)
-
-    def _set_cached_path(self, lpath: ty.Optional[Path]):
-        """protected interface for setting a cached Path since the attribute is not
-        available via the constructor.
-        """
-        super().__setattr__("__cached_path", lpath)  # this works around dataclass.frozen.
-        # https://noklam.github.io/blog/posts/2022-04-22-python-dataclass-partiala-immutable.html
-
-    def path(self) -> Path:
-        """Any Source can be turned into a local file path.
-
-        Remember that the resulting data is meant to be read-only. If you want to mutate
-        the data, you should first make a copy.
-
-        If not already present locally, this will incur a one-time download. Then, if the
-        Source has a Hash, the Hash will be validated against the downloaded file, and a
-        failure will raise SourceHashMismatchError.
-        """
-        if self.cached_path is None or not self.cached_path.exists():
-            lpath = _get_download_handler(self.uri)(self.hash)
-            # path() used to be responsible for checking the hash, but since we pass it to the downloader,
-            # it really makes more sense to allow the downloader to decide how to verify its own download,
-            # and we don't want to duplicate any effort that it may have already put in.
-            self._set_cached_path(lpath)
-
-        assert self.cached_path and self.cached_path.exists()
-        return self.cached_path
-
-    def __fspath__(self) -> str:
-        return os.fspath(self.path())
-
-
-# Creation from local Files or from remote URIs
-
-
-def from_file(filename: StrOrPath, hash: ty.Optional[Hash] = None, uri: str = "") -> Source:
-    """Create a read-only Source from a local file that already exists.
-
-    If URI is passed, the local file will be read and hashed, but the final URI in the
-    Source will be the one provided explicitly. NO UPLOAD IS PERFORMED. It is your
-    responsibility to ensure that your file has been uploaded to the URI you provide.
-    """
-    path = path_from_uri(filename) if isinstance(filename, str) else filename
-    assert isinstance(path, Path)
-    if not path.exists():
-        raise FileNotFoundError(path)
-
-    if uri:
-        src = from_uri(uri, _check_hash(hash, path))
-    else:
-        src = Source(to_uri(path), _check_hash(hash, path))
-    src._set_cached_path(path)  # internally, it's okay to hack around immutability.
-    return src
-
-
-class FromUri(ty.Protocol):
-    def __call__(self, hash: ty.Optional[Hash]) -> Source:
-        """Closure over a URI that creates a Source from a URI.
-
-        The Hash may be used to short-circuit creation that would result in creating
-        a Source that cannot match the expected Hash, but this is not required,
-        and the hash will be included in the Source object regardless, and will
-        be validated (if non-nil) at the time of source data access.
-        """
-
-
-class FromUriHandler(ty.Protocol):
-    def __call__(self, uri: str) -> ty.Optional[FromUri]:
-        """Returns a FromUri object containing the URI if this URI can be handled.  Returns
-        None if this URI cannot be handled.
-        """
-
-
-def register_from_uri_handler(key: str, handler: FromUriHandler):
-    """If a library wants to customize how Sources are created from URIs that it handles,
-    it can register a handler here.
-    """
-    # key is not currently used for anything other than avoiding
-    # having duplicates registered for whatever reason.
-    _FROM_URI_HANDLERS[key] = handler
-
-
-_FROM_URI_HANDLERS: ty.Dict[str, FromUriHandler] = dict()
-register_from_uri_handler(
-    "local_file", lambda uri: partial(from_file, path_from_uri(uri)) if is_file_uri(uri) else None
-)
-
-
-def from_uri(uri: str, hash: ty.Optional[Hash] = None) -> Source:
-    """Create a read-only Source from a URI. The data should already exist at this remote
-    URI, although Source itself can make no guarantee that it always represents real data
-    - only that any data it does represent is read-only.
-
-    It may be advantageous for a URI-handling library to register a more specific
-    implementation of this function, if it is capable of determining a Hash for the blob
-    represented by the URI without downloading the blob.
-    """
-    for handler in _FROM_URI_HANDLERS.values():
-        if from_uri_ := handler(uri):
-            return from_uri_(hash)
-    return Source(uri=uri, hash=hash)