globusfs 0.1.0__tar.gz

globusfs-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,38 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install -e ".[dev]"
+      - run: ruff check .
+      - run: ruff format --check .
+      # Network tests hit a third-party public collection whose backend is
+      # intermittently degraded; they are informative, not gating.
+      - name: Unit tests
+        run: pytest -m "not network" -q
+
+  network:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install -e ".[dev]"
+      - name: Characterization tests against a live Globus collection
+        run: pytest -m network -q -rs

globusfs-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,41 @@
+name: Release
+
+# Publishes to PyPI on a version tag. Uses trusted publishing (OIDC),
+# so there is no API token stored anywhere -- PyPI verifies the identity
+# of this specific workflow in this specific repo.
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install build
+      - run: python -m build
+      # Fail early on malformed metadata rather than at upload.
+      - run: pip install twine && twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    # Must match the environment name registered with the PyPI publisher.
+    environment: pypi
+    permissions:
+      # Required for trusted publishing; grants nothing else.
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

globusfs-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.py[cod]
+.venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/

globusfs-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sam Foreman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

globusfs-0.1.0/PKG-INFO

@@ -0,0 +1,195 @@
+Metadata-Version: 2.5
+Name: globusfs
+Version: 0.1.0
+Summary: fsspec filesystem for Globus collections
+Project-URL: Homepage, https://github.com/saforem2/globusfs
+Project-URL: Source, https://github.com/saforem2/globusfs
+Project-URL: Issues, https://github.com/saforem2/globusfs/issues
+Author-email: Sam Foreman <saforem2@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Sam Foreman
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: data,filesystem,fsspec,globus,hpc
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Filesystems
+Requires-Python: >=3.10
+Requires-Dist: aiohttp
+Requires-Dist: fsspec>=2023.1
+Provides-Extra: auth
+Requires-Dist: globus-sdk>=3.30; extra == 'auth'
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: requests; extra == 'dev'
+Requires-Dist: ruff>=0.14; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# globusfs
+
+An [fsspec](https://filesystem-spec.readthedocs.io/) filesystem for
+[Globus](https://www.globus.org/) collections.
+
+> **Status: working.** Verified against three live collections —
+> including **ALCF Eagle** (`alcf#dtn_eagle`, 747 project directories),
+> where `ls`, `glob`, `info`, `open()` with mid-file seek, and sparse
+> ranged reads all work against production Lustre. Writes (`PUT`/`DELETE`)
+> round-trip on Globus Tutorial Collection 1. Anonymous pyarrow column
+> projection works on a public collection — one column of sixty, a few KB
+> instead of 360 KB — while that collection was intermittently returning
+> backend-fault 404s.
+
+```python
+import globusfs
+
+# Browser login once; tokens persist to ~/.globusfs/tokens.json
+fs = globusfs.filesystem("<collection-uuid>")
+
+fs.ls("/")
+with fs.open("data/file.parquet", "rb") as f:
+    ...
+```
+
+Public collections need no credentials — and no `globus-sdk`:
+
+```python
+import fsspec, pyarrow.parquet as pq
+
+fs = fsspec.filesystem(
+    "globus",
+    collection_id="isaac",
+    https_url="https://g-05a4b6.2d513.8443.data.globus.org",
+)
+
+with fs.open("isaac/ability/ALL_2007-01.parquet", "rb") as f:
+    # Reads only the bytes this column needs, over HTTP range requests.
+    table = pq.ParquetFile(f).read(columns=["author"])
+```
+
+## Why
+
+Globus is how large scientific datasets actually move between facilities,
+but there is no fsspec backend for it — so pyarrow, pandas, dask, and
+grain can't read a Globus collection the way they read `s3://` or
+`gs://`. This fills that gap with one backend that serves all of them.
+
+## How it works
+
+Globus Connect Server exposes two services, and this needs both:
+
+| Concern | Service | Why |
+|---|---|---|
+| Reading bytes | HTTPS collection endpoint | Serves full HTTP range semantics: 206, `Content-Range`, mid-file seeks, multipart |
+| Listing / metadata | Transfer API | The HTTPS interface has no directory listings |
+
+The read path subclasses `fsspec`'s `HTTPFileSystem`, which already
+speaks exactly the range dialect GCS serves.
+
+### Server quirks this works around
+
+Verified against a live collection:
+
+- **Backend flakiness surfaces as a `404`.** GCS load-balances across
+  GridFTP backends; a failing one returns `ENDPOINT_ERROR` / `GCS Manager
+  Internal Error` rendered as HTTP 404 — byte-identical in status to a
+  genuinely missing file, and sticky for the life of a connection.
+  Observed failure rates on the public test collection swung from 0/20 to
+  20/20 within minutes, hitting files, directories, and the collection
+  root alike. Retries need a *fresh* connection, and the only way to tell
+  a transient error from a real miss is to parse the body. **A client
+  that treats 404 as "absent" will report healthy data as missing.**
+- **Suffix ranges (`bytes=-8`) return `416`**, which is how parquet
+  readers typically seek to the footer. Because `info()` knows the true
+  size, readers can use absolute offsets instead.
+- **`HEAD` is unusable.** A HEAD 404 carries no body — and the body is
+  the *only* thing distinguishing a backend fault from a real miss. So
+  HEAD results are permanently ambiguous. Size and existence come from
+  the Transfer API, or from a ranged `GET` (which does return a body and
+  carries the total in `Content-Range`).
+- **The HTTPS interface has no directory listings at all** — hence the
+  Transfer API for metadata.
+
+## Known ALCF collections
+
+Resolved via `endpoint_search`; ALCF's docs list names, not UUIDs.
+
+| Collection | UUID | Type |
+|---|---|---|
+| `alcf#dtn_eagle` | `05d2c76a-e867-4f67-aa57-76edeb0beda0` | mapped |
+| `alcf#dtn_flare` | `f39a7a0f-5bfc-46ce-9615-ba9f8592814f` | mapped |
+| `alcf#dtn_grand` | `3caddd4a-bb35-4c3d-9101-d9a0ad7f3a30` | mapped |
+| Globus Tutorials on ALCF Eagle | `a6f165fa-aee2-4fe5-95f3-97429c28bf82` | guest, public |
+
+Eagle's collection root is already `/eagle/projects`, so paths are
+project-relative: `fs.ls("/datascience")`, not `/eagle/projects/datascience`.
+
+Expect roughly **2 s per metadata or read operation** through the DTN —
+fine for sparse reads and exploration, not for per-record access in a
+training loop. See the note on training workloads below.
+
+## Credentials
+
+Token acquisition is pluggable, because it varies more than anything
+else: a public collection needs nothing, a portal already has a token,
+an interactive user needs a browser.
+
+| Provider | Use |
+|---|---|
+| `AnonymousCredentials` | Public collections (default) |
+| `StaticToken` | A token you already hold |
+| `CallableToken` | Fetch on demand — the pickle-safe option |
+| `AppCredentials` | Wraps `globus_sdk` `UserApp`/`ClientApp` |
+
+Two constraints worth knowing, both from the globus-sdk docs:
+
+- **`GlobusApp` is not thread-safe**, but fsspec shares one filesystem
+  across threads. `AppCredentials` serializes every call through a lock.
+- **fsspec pickles filesystems to worker processes.** A live token in the
+  constructor args would be copied into every worker payload, so
+  `StaticToken` refuses to pickle; use `CallableToken` reading from the
+  environment or shared storage so workers re-read rather than receive.
+
+Also set `request_refresh_tokens=True` — it defaults to `False`, and
+without it a long run dies when the access token expires mid-epoch.
+
+## A note on training workloads
+
+This is built for *remote and sparse* reads: column projection,
+exploration, data not yet staged. For distributed training, staging with
+Globus Transfer to node-local scratch beats per-record HTTPS on every
+axis — no per-record latency, no token expiry mid-epoch, and it works
+with formats like ArrayRecord whose readers do their own seeking.
+
+## Tests
+
+Characterization tests hit a real public collection and are marked
+`network`:
+
+```bash
+pytest                    # everything
+pytest -m "not network"   # offline only
+```
+
+## License
+
+MIT

globusfs-0.1.0/README.md

@@ -0,0 +1,148 @@
+# globusfs
+
+An [fsspec](https://filesystem-spec.readthedocs.io/) filesystem for
+[Globus](https://www.globus.org/) collections.
+
+> **Status: working.** Verified against three live collections —
+> including **ALCF Eagle** (`alcf#dtn_eagle`, 747 project directories),
+> where `ls`, `glob`, `info`, `open()` with mid-file seek, and sparse
+> ranged reads all work against production Lustre. Writes (`PUT`/`DELETE`)
+> round-trip on Globus Tutorial Collection 1. Anonymous pyarrow column
+> projection works on a public collection — one column of sixty, a few KB
+> instead of 360 KB — while that collection was intermittently returning
+> backend-fault 404s.
+
+```python
+import globusfs
+
+# Browser login once; tokens persist to ~/.globusfs/tokens.json
+fs = globusfs.filesystem("<collection-uuid>")
+
+fs.ls("/")
+with fs.open("data/file.parquet", "rb") as f:
+    ...
+```
+
+Public collections need no credentials — and no `globus-sdk`:
+
+```python
+import fsspec, pyarrow.parquet as pq
+
+fs = fsspec.filesystem(
+    "globus",
+    collection_id="isaac",
+    https_url="https://g-05a4b6.2d513.8443.data.globus.org",
+)
+
+with fs.open("isaac/ability/ALL_2007-01.parquet", "rb") as f:
+    # Reads only the bytes this column needs, over HTTP range requests.
+    table = pq.ParquetFile(f).read(columns=["author"])
+```
+
+## Why
+
+Globus is how large scientific datasets actually move between facilities,
+but there is no fsspec backend for it — so pyarrow, pandas, dask, and
+grain can't read a Globus collection the way they read `s3://` or
+`gs://`. This fills that gap with one backend that serves all of them.
+
+## How it works
+
+Globus Connect Server exposes two services, and this needs both:
+
+| Concern | Service | Why |
+|---|---|---|
+| Reading bytes | HTTPS collection endpoint | Serves full HTTP range semantics: 206, `Content-Range`, mid-file seeks, multipart |
+| Listing / metadata | Transfer API | The HTTPS interface has no directory listings |
+
+The read path subclasses `fsspec`'s `HTTPFileSystem`, which already
+speaks exactly the range dialect GCS serves.
+
+### Server quirks this works around
+
+Verified against a live collection:
+
+- **Backend flakiness surfaces as a `404`.** GCS load-balances across
+  GridFTP backends; a failing one returns `ENDPOINT_ERROR` / `GCS Manager
+  Internal Error` rendered as HTTP 404 — byte-identical in status to a
+  genuinely missing file, and sticky for the life of a connection.
+  Observed failure rates on the public test collection swung from 0/20 to
+  20/20 within minutes, hitting files, directories, and the collection
+  root alike. Retries need a *fresh* connection, and the only way to tell
+  a transient error from a real miss is to parse the body. **A client
+  that treats 404 as "absent" will report healthy data as missing.**
+- **Suffix ranges (`bytes=-8`) return `416`**, which is how parquet
+  readers typically seek to the footer. Because `info()` knows the true
+  size, readers can use absolute offsets instead.
+- **`HEAD` is unusable.** A HEAD 404 carries no body — and the body is
+  the *only* thing distinguishing a backend fault from a real miss. So
+  HEAD results are permanently ambiguous. Size and existence come from
+  the Transfer API, or from a ranged `GET` (which does return a body and
+  carries the total in `Content-Range`).
+- **The HTTPS interface has no directory listings at all** — hence the
+  Transfer API for metadata.
+
+## Known ALCF collections
+
+Resolved via `endpoint_search`; ALCF's docs list names, not UUIDs.
+
+| Collection | UUID | Type |
+|---|---|---|
+| `alcf#dtn_eagle` | `05d2c76a-e867-4f67-aa57-76edeb0beda0` | mapped |
+| `alcf#dtn_flare` | `f39a7a0f-5bfc-46ce-9615-ba9f8592814f` | mapped |
+| `alcf#dtn_grand` | `3caddd4a-bb35-4c3d-9101-d9a0ad7f3a30` | mapped |
+| Globus Tutorials on ALCF Eagle | `a6f165fa-aee2-4fe5-95f3-97429c28bf82` | guest, public |
+
+Eagle's collection root is already `/eagle/projects`, so paths are
+project-relative: `fs.ls("/datascience")`, not `/eagle/projects/datascience`.
+
+Expect roughly **2 s per metadata or read operation** through the DTN —
+fine for sparse reads and exploration, not for per-record access in a
+training loop. See the note on training workloads below.
+
+## Credentials
+
+Token acquisition is pluggable, because it varies more than anything
+else: a public collection needs nothing, a portal already has a token,
+an interactive user needs a browser.
+
+| Provider | Use |
+|---|---|
+| `AnonymousCredentials` | Public collections (default) |
+| `StaticToken` | A token you already hold |
+| `CallableToken` | Fetch on demand — the pickle-safe option |
+| `AppCredentials` | Wraps `globus_sdk` `UserApp`/`ClientApp` |
+
+Two constraints worth knowing, both from the globus-sdk docs:
+
+- **`GlobusApp` is not thread-safe**, but fsspec shares one filesystem
+  across threads. `AppCredentials` serializes every call through a lock.
+- **fsspec pickles filesystems to worker processes.** A live token in the
+  constructor args would be copied into every worker payload, so
+  `StaticToken` refuses to pickle; use `CallableToken` reading from the
+  environment or shared storage so workers re-read rather than receive.
+
+Also set `request_refresh_tokens=True` — it defaults to `False`, and
+without it a long run dies when the access token expires mid-epoch.
+
+## A note on training workloads
+
+This is built for *remote and sparse* reads: column projection,
+exploration, data not yet staged. For distributed training, staging with
+Globus Transfer to node-local scratch beats per-record HTTPS on every
+axis — no per-record latency, no token expiry mid-epoch, and it works
+with formats like ArrayRecord whose readers do their own seeking.
+
+## Tests
+
+Characterization tests hit a real public collection and are marked
+`network`:
+
+```bash
+pytest                    # everything
+pytest -m "not network"   # offline only
+```
+
+## License
+
+MIT

globusfs-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "globusfs"
+version = "0.1.0"
+description = "fsspec filesystem for Globus collections"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+authors = [{ name = "Sam Foreman", email = "saforem2@gmail.com" }]
+keywords = ["globus", "fsspec", "filesystem", "hpc", "data"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Science/Research",
+  "Programming Language :: Python :: 3",
+  "Topic :: System :: Filesystems",
+]
+dependencies = [
+  # HTTPFileSystem (the ranged-read path we subclass) lives in fsspec but
+  # needs aiohttp, which fsspec does not pull in on its own.
+  "fsspec>=2023.1",
+  "aiohttp",
+]
+
+[project.urls]
+Homepage = "https://github.com/saforem2/globusfs"
+Source = "https://github.com/saforem2/globusfs"
+Issues = "https://github.com/saforem2/globusfs/issues"
+
+[project.optional-dependencies]
+# Only needed for Transfer-API listing and GlobusApp-backed credentials.
+# Anonymous reads work without it, which is what keeps the test fixture
+# usable in CI with no secrets.
+auth = ["globus-sdk>=3.30"]
+dev = ["pytest", "pytest-asyncio", "ruff>=0.14", "requests"]
+
+# fsspec discovers backends through this entry point group, so
+# `fsspec.open("globus://...")` works after a plain install with no
+# explicit register_implementation() call by the user.
+[project.entry-points."fsspec.specs"]
+globus = "globusfs:GlobusFileSystem"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/globusfs"]
+
+[tool.pytest.ini_options]
+markers = [
+  "network: hits the public ISAAC collection; deselect with -m 'not network'",
+]

globusfs-0.1.0/src/globusfs/__init__.py

@@ -0,0 +1,29 @@
+"""fsspec filesystem for Globus collections."""
+
+from .core import GlobusFileSystem
+from .credentials import (
+    AnonymousCredentials,
+    AppCredentials,
+    CallableToken,
+    GlobusCredentials,
+    StaticToken,
+)
+from .errors import GlobusFSError, TransientBackendError
+from .login import filesystem, login
+from .transfer import TransferMetadata
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AnonymousCredentials",
+    "AppCredentials",
+    "CallableToken",
+    "GlobusCredentials",
+    "GlobusFSError",
+    "GlobusFileSystem",
+    "StaticToken",
+    "TransferMetadata",
+    "TransientBackendError",
+    "filesystem",
+    "login",
+]