marin-rigging 0.2.1.dev202605292307__tar.gz

marin_rigging-0.2.1.dev202605292307/.gitignore

@@ -0,0 +1,244 @@
+# redundant, but Ray looks for this otherwise.
+.git
+
+logs/
+
+# CPU profiles
+prof/
+
+# Downloaded build tools (zig, etc.)
+.tools/
+
+tests/snapshots/outputs
+tests/snapshots/diffs
+
+# don't log data/MD outputs to git
+data/*
+output/*
+outputs/*
+
+# Snapshot diffs and outputs
+tests/snapshots/*/outputs/*
+tests/snapshots/*/diffs/*
+
+# This is mainly for Ray and using submodule
+*/**/.git
+
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# pypa/gh-action-pypi-publish caches its Docker action manifest here.
+.github/.tmp/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+local_store/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Ruff
+.ruff_cache/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+*.iml
+
+# IDE Config
+.vscode/
+
+# Mac OS
+.DS_Store
+
+# Secrets
+credentials.json
+marin/crawl/bigquery-gcs-key.json
+
+# Archive
+archive/
+
+# Caches and Outputs
+!/scripts/web/output/
+!/output/
+
+# csv
+*.csv
+
+# wandb logs
+wandb
+artifacts
+
+# Ignore generated credentials from google-github-actions/auth
+gha-creds-*.json
+
+.aider*
+.git/*
+
+*.jsonl
+**/*.jsonl
+scr/*
+.weaver/
+
+# Local host Marin config
+.marin.yaml
+
+/scratch
+
+.forge
+.claude
+!.claude/skills
+.agents/tmp/
+.codex
+.entire
+
+.worktrees
+.obsidian
+.cw_env

marin_rigging-0.2.1.dev202605292307/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: marin-rigging
+Version: 0.2.1.dev202605292307
+Requires-Python: <3.13,>=3.11
+Requires-Dist: connect-python>=0.9.0
+Requires-Dist: fsspec>=2024.0.0
+Requires-Dist: gcsfs>=2024.0.0
+Requires-Dist: s3fs>=2024.0.0

marin_rigging-0.2.1.dev202605292307/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "marin-rigging"
+version = "0.2.1.dev202605292307"
+requires-python = ">=3.11,<3.13"
+dependencies = [
+    "connect-python>=0.9.0",
+    "fsspec>=2024.0.0",
+    "gcsfs>=2024.0.0",
+    "s3fs>=2024.0.0",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.3.2",
+    "pytest-asyncio",
+    "pytest-timeout",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/rigging"]
+
+[tool.pytest.ini_options]
+timeout = 10

marin_rigging-0.2.1.dev202605292307/src/rigging/__init__.py

@@ -0,0 +1,2 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0

marin_rigging-0.2.1.dev202605292307/src/rigging/config_discovery.py

@@ -0,0 +1,154 @@
+# Copyright The Marin Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Path-agnostic config discovery for cluster YAML files.
+
+Generic YAML config discovery helpers. Callers (e.g. iris) pass the
+directories to search; this module knows nothing about any particular
+marin sub-package.
+"""
+
+import functools
+import logging
+from collections.abc import Sequence
+from pathlib import Path
+
+import tomllib
+
+logger = logging.getLogger(__name__)
+
+_YAML_SUFFIXES = (".yaml", ".yml")
+
+
+@functools.lru_cache(maxsize=128)
+def find_project_root(start: Path | str | None = None) -> Path | None:
+    """Find the marin workspace root.
+
+    Walks up from ``start`` (or the current working directory) looking for a
+    ``pyproject.toml`` that declares ``[tool.uv.workspace]``. This uniquely
+    identifies the top-level marin root and avoids matching a workspace
+    member's pyproject (e.g. ``lib/iris/pyproject.toml``).
+
+    Returns:
+        The marin root ``Path``, or ``None`` when running outside a marin
+        checkout (e.g. from an installed pip package).
+    """
+    current = Path(start).resolve() if start is not None else Path.cwd().resolve()
+
+    for directory in (current, *current.parents):
+        pp = directory / "pyproject.toml"
+        if pp.is_file() and _declares_uv_workspace(pp):
+            logger.debug("Found marin workspace root: %s", directory)
+            return directory
+
+    logger.debug("No marin workspace root found starting from %s", current)
+    return None
+
+
+def _declares_uv_workspace(pyproject_path: Path) -> bool:
+    """Return True if ``pyproject_path`` declares ``[tool.uv.workspace]``."""
+    try:
+        with pyproject_path.open("rb") as f:
+            data = tomllib.load(f)
+    except (OSError, tomllib.TOMLDecodeError):
+        return False
+    return "workspace" in data.get("tool", {}).get("uv", {})
+
+
+def _resolve_dirs(dirs: Sequence[Path | str]) -> list[Path]:
+    """Expand ``~`` and resolve relative dirs against the marin project root.
+
+    An empty string resolves to the project root itself. Absolute paths are
+    returned unchanged. Relative paths are joined onto the marin project root
+    when one is found, and fall back to the current working directory otherwise.
+    """
+    root = find_project_root()
+    resolved: list[Path] = []
+    for raw in dirs:
+        p = Path(raw).expanduser()
+        if p.is_absolute():
+            resolved.append(p)
+        elif root is not None:
+            resolved.append(root / p)
+        else:
+            resolved.append(Path.cwd() / p)
+    return resolved
+
+
+def find_configs(
+    dirs: Sequence[Path | str],
+    name: str | None = None,
+) -> dict[str, Path]:
+    """Discover YAML config files across ``dirs``.
+
+    Relative ``dirs`` are resolved against the marin project root (see
+    :func:`find_project_root`); absolute paths are used as-is; ``~`` is
+    expanded. An empty string resolves to the project root itself.
+
+    Args:
+        dirs: Directories to search, in priority order.
+        name: When given, only return entries whose stem equals ``name``.
+
+    Returns:
+        A dict mapping config stem (filename without ``.yaml``/``.yml``) to
+        its resolved ``Path``. When the same stem appears in multiple dirs,
+        the first (highest-priority) match wins.
+    """
+    configs: dict[str, Path] = {}
+    for directory in _resolve_dirs(dirs):
+        if not directory.is_dir():
+            continue
+        for path in sorted(directory.iterdir()):
+            if path.suffix not in _YAML_SUFFIXES:
+                continue
+            stem = path.stem
+            if name is not None and stem != name:
+                continue
+            if stem not in configs:
+                configs[stem] = path
+    return configs
+
+
+def resolve_cluster_config(name: str, dirs: Sequence[Path | str]) -> Path:
+    """Resolve a cluster name (or path) to an existing YAML config file.
+
+    If ``name`` is already an existing file path, it is returned directly.
+    Otherwise ``dirs`` are searched for a file whose stem matches ``name``
+    (with ``.yaml`` or ``.yml`` extensions stripped from ``name`` before
+    comparison).
+
+    Args:
+        name: Cluster name (e.g. ``"marin-dev"``) or path to an existing file.
+        dirs: Directories to search.
+
+    Returns:
+        The resolved ``Path`` to the config file.
+
+    Raises:
+        FileNotFoundError: When no matching config file is found, with a
+            message listing all searched locations.
+    """
+    candidate = Path(name).expanduser()
+    if candidate.is_file():
+        return candidate
+
+    # Allow callers to pass either "marin-dev" or "marin-dev.yaml".
+    name_path = Path(name)
+    search_stem = name_path.stem if name_path.suffix in _YAML_SUFFIXES else name
+
+    matches = find_configs(dirs, name=search_stem)
+    if search_stem in matches:
+        logger.debug("Resolved cluster config %r -> %s", name, matches[search_stem])
+        return matches[search_stem]
+
+    searched_str = "\n  ".join(str(d) for d in _resolve_dirs(dirs))
+    raise FileNotFoundError(f"No config file found for cluster {name!r}.\nSearched directories:\n  {searched_str}")
+
+
+def list_cluster_configs(dirs: Sequence[Path | str]) -> dict[str, Path]:
+    """List all YAML cluster configs across ``dirs``.
+
+    Thin alias over :func:`find_configs` for callers that want the full
+    name-to-path mapping.
+    """
+    return find_configs(dirs)