nomad-fsspec 0.1.0__tar.gz

nomad_fsspec-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+dist/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/

nomad_fsspec-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 nomad-fsspec contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nomad_fsspec-0.1.0/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.5
+Name: nomad-fsspec
+Version: 0.1.0
+Summary: An fsspec filesystem for the public raw data of NOMAD.
+Project-URL: Homepage, https://github.com/arash77/nomad-fsspec
+Project-URL: Issues, https://github.com/arash77/nomad-fsspec/issues
+Author: nomad-fsspec contributors
+License-Expression: MIT
+License-File: LICENSE
+Keywords: filesystem,fsspec,materials science,nomad
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.10
+Requires-Dist: fsspec>=2024.1.0
+Requires-Dist: requests>=2.31
+Requires-Dist: urllib3>=1.26
+Description-Content-Type: text/markdown
+
+# nomad-fsspec
+
+An [fsspec](https://filesystem-spec.readthedocs.io) filesystem for the public raw data of
+[NOMAD](https://nomad-lab.eu), the materials science repository. It is read-only.
+
+```python
+import fsspec
+
+fs = fsspec.filesystem("nomad")  # https://nomad-lab.eu/prod/v1 by default
+fs.ls("/")  # public datasets
+fs.ls("/wWgAnNZNQxOHLf97H62dgw")  # the uploads of one dataset
+fs.get_file(
+    "/wWgAnNZNQxOHLf97H62dgw/Xa2LB__ESiKu-odRBPJhjA/nomad-demo-data/BrK_svSe/TBCC006.ABC/vasprun.xml.relax2",
+    "vasprun.xml",
+)
+```
+
+Paths are `/<dataset id>/<upload id>/<path inside the upload>`. Datasets and uploads have their
+names in `display_name`. To use a NOMAD Oasis, pass `base_url`:
+
+```python
+fs = fsspec.filesystem("nomad", base_url="https://my-oasis.example.org/nomad-oasis")
+```
+
+Only public data is listed. NOMAD's rate limits (HTTP 429 and 503) are retried.
+
+## Development
+
+```bash
+just install
+just test        # unit tests
+just test-live   # a few calls to nomad-lab.eu
+just lint
+```
+
+## License
+
+MIT

nomad_fsspec-0.1.0/README.md

@@ -0,0 +1,38 @@
+# nomad-fsspec
+
+An [fsspec](https://filesystem-spec.readthedocs.io) filesystem for the public raw data of
+[NOMAD](https://nomad-lab.eu), the materials science repository. It is read-only.
+
+```python
+import fsspec
+
+fs = fsspec.filesystem("nomad")  # https://nomad-lab.eu/prod/v1 by default
+fs.ls("/")  # public datasets
+fs.ls("/wWgAnNZNQxOHLf97H62dgw")  # the uploads of one dataset
+fs.get_file(
+    "/wWgAnNZNQxOHLf97H62dgw/Xa2LB__ESiKu-odRBPJhjA/nomad-demo-data/BrK_svSe/TBCC006.ABC/vasprun.xml.relax2",
+    "vasprun.xml",
+)
+```
+
+Paths are `/<dataset id>/<upload id>/<path inside the upload>`. Datasets and uploads have their
+names in `display_name`. To use a NOMAD Oasis, pass `base_url`:
+
+```python
+fs = fsspec.filesystem("nomad", base_url="https://my-oasis.example.org/nomad-oasis")
+```
+
+Only public data is listed. NOMAD's rate limits (HTTP 429 and 503) are retried.
+
+## Development
+
+```bash
+just install
+just test        # unit tests
+just test-live   # a few calls to nomad-lab.eu
+just lint
+```
+
+## License
+
+MIT

nomad_fsspec-0.1.0/pyproject.toml

@@ -0,0 +1,70 @@
+[build-system]
+requires = ["hatchling>=1.21"]
+build-backend = "hatchling.build"
+
+[project]
+name = "nomad-fsspec"
+version = "0.1.0"
+description = "An fsspec filesystem for the public raw data of NOMAD."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "nomad-fsspec contributors" }]
+keywords = ["nomad", "fsspec", "materials science", "filesystem"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Scientific/Engineering :: Physics",
+]
+dependencies = [
+    "fsspec>=2024.1.0",
+    "requests>=2.31",
+    "urllib3>=1.26",
+]
+
+[project.entry-points."fsspec.specs"]
+nomad = "nomad_fsspec:NomadFileSystem"
+
+[project.urls]
+Homepage = "https://github.com/arash77/nomad-fsspec"
+Issues = "https://github.com/arash77/nomad-fsspec/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "responses>=0.25",
+    "ruff>=0.6",
+    "mypy>=1.11",
+    "types-requests",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nomad_fsspec"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src", "tests", "README.md", "LICENSE"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W", "UP", "B", "C4", "SIM"]
+
+[tool.mypy]
+python_version = "3.10"
+warn_unused_ignores = true
+ignore_missing_imports = true
+files = ["src/nomad_fsspec"]
+
+[tool.pytest.ini_options]
+minversion = "8.0"
+testpaths = ["tests"]
+addopts = "-ra --strict-markers"
+markers = ["live: tests that call the public NOMAD at nomad-lab.eu"]

nomad_fsspec-0.1.0/src/nomad_fsspec/__init__.py

@@ -0,0 +1,5 @@
+from . import filesystem
+from .errors import NomadError
+from .filesystem import NomadFileSystem
+
+__all__ = ["NomadError", "NomadFileSystem", "filesystem"]

nomad_fsspec-0.1.0/src/nomad_fsspec/client.py

@@ -0,0 +1,95 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any
+from urllib.parse import quote, urlparse
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from .errors import NomadError
+
+DEFAULT_BASE_URL = "https://nomad-lab.eu/prod/v1"
+API_PATH = "/api/v1"
+
+
+class NomadClient:
+    """A small client for NOMAD's REST API."""
+
+    def __init__(
+        self, base_url: str = DEFAULT_BASE_URL, timeout: tuple[float, float] = (10, 120)
+    ) -> None:
+        base = base_url.strip().rstrip("/")
+        if base.endswith(API_PATH):
+            base = base[: -len(API_PATH)]
+        self.api_url = base + API_PATH
+        self.host = urlparse(base).netloc or base
+        self.timeout = timeout
+        # NOMAD refuses bursts with 429 or 503, usually with Retry-After
+        retry = Retry(
+            total=5,
+            backoff_factor=1,
+            status_forcelist=[429, 503],
+            allowed_methods={"GET", "POST"},
+            respect_retry_after_header=True,
+            raise_on_status=False,
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        self.session = requests.Session()
+        self.session.mount("https://", adapter)
+        self.session.mount("http://", adapter)
+
+    def url(self, parts: list[str]) -> str:
+        return "/".join([self.api_url, *(quote(part, safe="") for part in parts)])
+
+    def get_json(self, parts: list[str], params: dict[str, Any] | None = None) -> dict[str, Any]:
+        with self.request("GET", parts, params=params) as response:
+            return self._json(response)
+
+    def post_json(self, parts: list[str], body: dict[str, Any]) -> dict[str, Any]:
+        with self.request("POST", parts, json=body) as response:
+            return self._json(response)
+
+    @contextmanager
+    def request(self, method: str, parts: list[str], **kwargs: Any) -> Iterator[requests.Response]:
+        label = "/" + "/".join(parts)
+        try:
+            with self.session.request(
+                method, self.url(parts), timeout=self.timeout, **kwargs
+            ) as response:
+                self._raise_for_status(response, label)
+                yield response
+        except requests.RequestException as e:
+            raise NomadError(f"Could not reach NOMAD at {self.host}: {e}") from e
+
+    def _raise_for_status(self, response: requests.Response, label: str) -> None:
+        if response.ok:
+            return
+        detail = _detail(response)
+        if response.status_code == 404:
+            raise FileNotFoundError(f"{label}: {detail}")
+        if response.status_code in (401, 403):
+            raise PermissionError(f"NOMAD at {self.host} refused access to {label}: {detail}")
+        raise NomadError(
+            f"NOMAD at {self.host} answered HTTP {response.status_code} for {label}: {detail}"
+        )
+
+    def _json(self, response: requests.Response) -> dict[str, Any]:
+        message = f"{self.host} did not answer like a NOMAD API; check the base URL"
+        try:
+            body = response.json()
+        except ValueError as e:
+            raise NomadError(message) from e
+        if not isinstance(body, dict):
+            raise NomadError(message)
+        return body
+
+
+def _detail(response: requests.Response) -> str:
+    try:
+        detail = response.json().get("detail")
+    except (ValueError, AttributeError):
+        detail = None
+    if isinstance(detail, str):
+        return detail
+    return response.reason or f"HTTP {response.status_code}"

nomad_fsspec-0.1.0/src/nomad_fsspec/errors.py

@@ -0,0 +1,6 @@
+class NomadError(Exception):
+    """NOMAD could not be reached or answered with an error.
+
+    Not an ``OSError`` on purpose: fsspec's ``walk`` skips folders that raise ``OSError``, which
+    would show a failed request as an empty folder.
+    """

nomad_fsspec-0.1.0/src/nomad_fsspec/filesystem.py

@@ -0,0 +1,217 @@
+from typing import IO, Any
+
+from fsspec import AbstractFileSystem
+
+from .client import DEFAULT_BASE_URL, NomadClient
+
+# NOMAD reads a whole directory for every page, so ask for all of it at once
+PAGE_SIZE = 10000
+AGGREGATION_PAGE_SIZE = 2500
+CHUNK_SIZE = 2**20
+
+
+class NomadFileSystem(AbstractFileSystem):
+    """Read-only access to the public datasets of a NOMAD deployment.
+
+    Paths are ``/<dataset_id>/<upload_id>/<path inside the upload>``. Datasets and uploads carry
+    their names in ``display_name``.
+    """
+
+    protocol = "nomad"
+    root_marker = "/"
+
+    def __init__(
+        self,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: tuple[float, float] = (10, 120),
+        **storage_options: Any,
+    ) -> None:
+        super().__init__(**storage_options)
+        self.client = NomadClient(base_url, timeout)
+
+    @classmethod
+    def _strip_protocol(cls, path: Any) -> Any:
+        if isinstance(path, list):
+            return [cls._strip_protocol(p) for p in path]
+        return "/" + super()._strip_protocol(path).strip("/")
+
+    def ls(self, path: str, detail: bool = True, **kwargs: Any) -> list[Any]:
+        path = self._strip_protocol(path)
+        # not _ls_from_cache: it answers a folder with its own entry once the parent is cached
+        key = path.rstrip("/")
+        try:
+            entries = self.dircache[key]
+        except KeyError:
+            entries = self._fetch(path)
+            self.dircache[key] = entries
+        return entries if detail else [entry["name"] for entry in entries]
+
+    def get_file(
+        self,
+        rpath: str,
+        lpath: Any = None,
+        callback: Any = None,
+        outfile: IO[bytes] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        parts = self._file_parts(rpath)
+        params = {"ignore_mime_type": "true"}
+        with self.client.request("GET", _raw(parts), params=params, stream=True) as response:
+            if outfile is not None:
+                _copy(response.iter_content(CHUNK_SIZE), outfile)
+            else:
+                with open(lpath, "wb") as target:
+                    _copy(response.iter_content(CHUNK_SIZE), target)
+
+    def cat_file(
+        self, path: str, start: int | None = None, end: int | None = None, **kwargs: Any
+    ) -> bytes:
+        parts = self._file_parts(path)
+        start = start or 0
+        if start < 0 or (end is not None and end < 0):
+            size = int(self.info(path)["size"])
+            start = max(size + start, 0) if start < 0 else start
+            end = size + end if end is not None and end < 0 else end
+        if end is not None and end <= start:
+            return b""
+        # NOMAD ignores HTTP Range headers; it takes offset and length instead
+        params: dict[str, Any] = {"ignore_mime_type": "true"}
+        if start:
+            params["offset"] = start
+        if end is not None:
+            params["length"] = end - start
+        with self.client.request("GET", _raw(parts), params=params) as response:
+            return response.content
+
+    def _open(self, path: str, mode: str = "rb", **kwargs: Any) -> Any:
+        if mode != "rb":
+            raise PermissionError(f"NOMAD is read-only; cannot open {path} with mode {mode!r}")
+        return super()._open(path, mode=mode, **kwargs)
+
+    def _file_parts(self, path: str) -> list[str]:
+        parts = _split(self._strip_protocol(path))
+        if len(parts) < 3:
+            raise IsADirectoryError(path)
+        return parts
+
+    def _fetch(self, path: str) -> list[dict[str, Any]]:
+        parts = _split(path)
+        if not parts:
+            return self._list_datasets()
+        if len(parts) == 1:
+            return self._list_uploads(parts[0])
+        return self._list_upload_directory(parts)
+
+    def _list_datasets(self) -> list[dict[str, Any]]:
+        public = {bucket["value"] for bucket in self._aggregate("datasets.dataset_id", {})}
+        entries: list[dict[str, Any]] = []
+        offset = 0
+        while True:
+            params = {
+                "page_size": PAGE_SIZE,
+                "page_offset": offset,
+                "order_by": "dataset_name",
+                "order": "asc",
+            }
+            body = self.client.get_json(["datasets", ""], params)
+            page = body["data"]
+            entries.extend(
+                _directory(f"/{dataset['dataset_id']}", dataset.get("dataset_name"))
+                for dataset in page
+                if dataset["dataset_id"] in public
+            )
+            offset += len(page)
+            if not page or offset >= body["pagination"]["total"]:
+                return entries
+
+    def _list_uploads(self, dataset_id: str) -> list[dict[str, Any]]:
+        query = {"datasets.dataset_id": dataset_id}
+        buckets = self._aggregate("upload_id", query, include=["upload_name"])
+        if not buckets:
+            raise FileNotFoundError(f"/{dataset_id}")
+        uploads = [
+            _directory(
+                f"/{dataset_id}/{bucket['value']}",
+                (bucket.get("entries") or [{}])[0].get("upload_name"),
+            )
+            for bucket in buckets
+        ]
+        return sorted(
+            uploads, key=lambda upload: (upload["display_name"].casefold(), upload["name"])
+        )
+
+    def _list_upload_directory(self, parts: list[str]) -> list[dict[str, Any]]:
+        path = "/" + "/".join(parts)
+        entries: list[dict[str, Any]] = []
+        offset = 0
+        while True:
+            params = {"page_size": PAGE_SIZE, "page_offset": offset}
+            body = self.client.get_json(["uploads", parts[1], "rawdir", *parts[2:], ""], params)
+            if "directory_metadata" not in body:
+                size = (body.get("file_metadata") or {}).get("size")
+                return [{"name": path, "size": int(size or 0), "type": "file"}]
+            content = body["directory_metadata"]["content"]
+            entries.extend(
+                {
+                    "name": f"{path}/{item['name']}",
+                    "size": int(item.get("size") or 0),
+                    "type": "file" if item.get("is_file") else "directory",
+                }
+                for item in content
+                if item.get("name")
+            )
+            offset += len(content)
+            # an empty page or the total ends the listing; NOMAD answers 400 past the last page
+            if not content or offset >= body["pagination"]["total"]:
+                return entries
+
+    def _aggregate(
+        self, quantity: str, query: dict[str, Any], include: list[str] | None = None
+    ) -> list[dict[str, Any]]:
+        buckets: list[dict[str, Any]] = []
+        after = None
+        while True:
+            pagination: dict[str, Any] = {"page_size": AGGREGATION_PAGE_SIZE}
+            if after is not None:
+                pagination["page_after_value"] = after
+            terms: dict[str, Any] = {"quantity": quantity, "pagination": pagination}
+            if include:
+                terms["entries"] = {"size": 1, "required": {"include": include}}
+            body = self.client.post_json(
+                ["entries", "query"],
+                {
+                    "owner": "public",
+                    "query": query,
+                    "pagination": {"page_size": 0},
+                    "aggregations": {"buckets": {"terms": terms}},
+                },
+            )
+            result = body["aggregations"]["buckets"]["terms"]
+            buckets.extend(result["data"])
+            # NOMAD sets a cursor even on the last page, so a short page is the end
+            if len(result["data"]) < AGGREGATION_PAGE_SIZE:
+                return buckets
+            after = (result.get("pagination") or {}).get("next_page_after_value")
+            if after is None:
+                return buckets
+
+
+def _split(path: str) -> list[str]:
+    parts = [part for part in path.split("/") if part]
+    if any(part in (".", "..") for part in parts):
+        raise ValueError(f"Invalid NOMAD path: {path}")
+    return parts
+
+
+def _directory(path: str, display_name: str | None) -> dict[str, Any]:
+    name = display_name or path.rsplit("/", 1)[-1]
+    return {"name": path, "size": 0, "type": "directory", "display_name": name}
+
+
+def _raw(parts: list[str]) -> list[str]:
+    return ["uploads", parts[1], "raw", *parts[2:]]
+
+
+def _copy(chunks: Any, target: IO[bytes]) -> None:
+    for chunk in chunks:
+        target.write(chunk)

nomad_fsspec-0.1.0/tests/test_client.py

@@ -0,0 +1,58 @@
+import pytest
+import requests
+import responses
+from responses import registries
+
+from nomad_fsspec.client import NomadClient
+from nomad_fsspec.errors import NomadError
+
+API = "https://nomad.example.org/oasis/api/v1"
+
+
+@pytest.mark.parametrize(
+    "base_url",
+    [
+        "https://nomad.example.org/oasis",
+        "https://nomad.example.org/oasis/",
+        "https://nomad.example.org/oasis/api/v1",
+        " https://nomad.example.org/oasis/api/v1/ ",
+    ],
+)
+def test_base_url_spellings_reach_one_api_root(base_url):
+    client = NomadClient(base_url)
+    assert client.api_url == API
+    assert client.host == "nomad.example.org"
+
+
+def test_path_segments_are_encoded_one_at_a_time():
+    url = NomadClient(API).url(["uploads", "u1", "raw", "run #1 100%.csv", ""])
+    assert url == f"{API}/uploads/u1/raw/run%20%231%20100%25.csv/"
+
+
+@responses.activate
+def test_errors_are_mapped():
+    client = NomadClient(API)
+    responses.get(f"{API}/a", status=404, json={"detail": "The specified upload_id was not found."})
+    responses.get(f"{API}/b", status=403, json={"detail": "Forbidden"})
+    responses.get(f"{API}/c", status=500, body="<html>proxy error</html>")
+    responses.get(f"{API}/d", body="<html>the NOMAD GUI</html>")
+    responses.get(f"{API}/e", body=requests.ConnectionError("refused"))
+    with pytest.raises(FileNotFoundError, match="upload_id was not found"):
+        client.get_json(["a"])
+    with pytest.raises(PermissionError):
+        client.get_json(["b"])
+    with pytest.raises(NomadError, match="HTTP 500"):
+        client.get_json(["c"])
+    with pytest.raises(NomadError, match="did not answer like a NOMAD API"):
+        client.get_json(["d"])
+    with pytest.raises(NomadError, match="Could not reach NOMAD"):
+        client.get_json(["e"])
+    assert not issubclass(NomadError, OSError)
+
+
+@responses.activate(registry=registries.OrderedRegistry)
+def test_rate_limited_and_unavailable_answers_are_retried():
+    responses.get(f"{API}/x", status=503)
+    responses.get(f"{API}/x", status=429, headers={"Retry-After": "0"})
+    responses.get(f"{API}/x", json={"ok": True})
+    assert NomadClient(API).get_json(["x"]) == {"ok": True}

nomad_fsspec-0.1.0/tests/test_filesystem.py

@@ -0,0 +1,246 @@
+import pytest
+import responses
+from responses import matchers, registries
+
+from nomad_fsspec import NomadError, NomadFileSystem, filesystem
+
+BASE = "https://nomad.example.org/oasis"
+API = f"{BASE}/api/v1"
+
+
+def make_fs() -> NomadFileSystem:
+    return NomadFileSystem(base_url=BASE, skip_instance_cache=True)
+
+
+def aggregation_body(quantity, query, page_size, after=None, include=None):
+    pagination = {"page_size": page_size}
+    if after is not None:
+        pagination["page_after_value"] = after
+    terms = {"quantity": quantity, "pagination": pagination}
+    if include:
+        terms["entries"] = {"size": 1, "required": {"include": include}}
+    return {
+        "owner": "public",
+        "query": query,
+        "pagination": {"page_size": 0},
+        "aggregations": {"buckets": {"terms": terms}},
+    }
+
+
+def aggregation(*values, names=None, after=None):
+    data = []
+    for value in values:
+        bucket = {"value": value, "count": 1}
+        if names and value in names:
+            bucket["entries"] = [{"upload_name": names[value]}]
+        data.append(bucket)
+    terms = {"data": data, "pagination": {"next_page_after_value": after}}
+    return {"aggregations": {"buckets": {"terms": terms}}}
+
+
+@responses.activate
+def test_root_lists_public_datasets_by_name_and_caches():
+    # strict matchers matter: NOMAD ignores a parameter it does not know and answers with everything
+    responses.post(
+        f"{API}/entries/query",
+        match=[matchers.json_params_matcher(aggregation_body("datasets.dataset_id", {}, 2500))],
+        json=aggregation("ds1", "ds2"),
+    )
+    responses.get(
+        f"{API}/datasets/",
+        match=[
+            matchers.query_param_matcher(
+                {
+                    "page_size": "10000",
+                    "page_offset": "0",
+                    "order_by": "dataset_name",
+                    "order": "asc",
+                }
+            )
+        ],
+        json={
+            "pagination": {"total": 3},
+            "data": [
+                {"dataset_id": "ds1", "dataset_name": "demo data"},
+                {"dataset_id": "ds2"},
+                {"dataset_id": "ds3", "dataset_name": "no public entries"},
+            ],
+        },
+    )
+    fs = make_fs()
+    entries = fs.ls("nomad://")
+    assert [(e["name"], e["display_name"], e["type"]) for e in entries] == [
+        ("/ds1", "demo data", "directory"),
+        ("/ds2", "ds2", "directory"),
+    ]
+    assert fs.ls("/", detail=False) == ["/ds1", "/ds2"]
+    assert len(responses.calls) == 2
+
+
+@responses.activate(registry=registries.OrderedRegistry)
+def test_a_dataset_lists_its_uploads_across_pages(monkeypatch):
+    monkeypatch.setattr(filesystem, "AGGREGATION_PAGE_SIZE", 2)
+    query = {"datasets.dataset_id": "ds1"}
+    include = ["upload_name"]
+    responses.post(
+        f"{API}/entries/query",
+        match=[
+            matchers.json_params_matcher(aggregation_body("upload_id", query, 2, include=include))
+        ],
+        json=aggregation("up1", "up2", names={"up1": "b.zip", "up2": "a.zip"}, after="up2"),
+    )
+    responses.post(
+        f"{API}/entries/query",
+        match=[
+            matchers.json_params_matcher(
+                aggregation_body("upload_id", query, 2, after="up2", include=include)
+            )
+        ],
+        json=aggregation("up3", after="up3"),
+    )
+    entries = make_fs().ls("/ds1")
+    assert [(e["name"], e["display_name"]) for e in entries] == [
+        ("/ds1/up2", "a.zip"),
+        ("/ds1/up1", "b.zip"),
+        ("/ds1/up3", "up3"),
+    ]
+
+
+@responses.activate
+def test_a_dataset_without_public_uploads_is_not_found():
+    responses.post(f"{API}/entries/query", json=aggregation())
+    with pytest.raises(FileNotFoundError):
+        make_fs().ls("/no-such-dataset")
+
+
+def test_nomad_error_is_exported():
+    assert issubclass(NomadError, Exception)
+
+
+def rawdir(*content, total=None):
+    return {
+        "directory_metadata": {"content": list(content)},
+        "pagination": {"total": len(content) if total is None else total},
+    }
+
+
+def page(size, offset):
+    return matchers.query_param_matcher({"page_size": str(size), "page_offset": str(offset)})
+
+
+@responses.activate
+def test_upload_directory_pages_until_the_total(monkeypatch):
+    monkeypatch.setattr(filesystem, "PAGE_SIZE", 2)
+    url = f"{API}/uploads/up1/rawdir/data/"
+    first = rawdir(
+        {"name": "a.txt", "size": 5, "is_file": True},
+        {"name": "sub", "size": 9, "is_file": False},
+        total=4,
+    )
+    second = rawdir({"name": "", "is_file": False}, {"name": "z.txt", "is_file": True}, total=4)
+    responses.get(url, match=[page(2, 0)], json=first)
+    responses.get(url, match=[page(2, 2)], json=second)
+    fs = make_fs()
+    entries = fs.ls("/ds1/up1/data")
+    assert [(e["name"], e["type"], e["size"]) for e in entries] == [
+        ("/ds1/up1/data/a.txt", "file", 5),
+        ("/ds1/up1/data/sub", "directory", 9),
+        ("/ds1/up1/data/z.txt", "file", 0),
+    ]
+    assert fs.info("/ds1/up1/data/a.txt")["size"] == 5
+    assert len(responses.calls) == 2
+
+
+@responses.activate
+def test_an_empty_page_ends_the_listing(monkeypatch):
+    monkeypatch.setattr(filesystem, "PAGE_SIZE", 1)
+    url = f"{API}/uploads/up1/rawdir/"
+    responses.get(url, match=[page(1, 0)], json=rawdir({"name": "a", "is_file": True}, total=5))
+    responses.get(url, match=[page(1, 1)], json=rawdir(total=5))
+    assert [e["name"] for e in make_fs().ls("/ds1/up1")] == ["/ds1/up1/a"]
+
+
+@responses.activate
+def test_listing_a_file_returns_the_file():
+    responses.get(
+        f"{API}/uploads/up1/rawdir/a.txt/", json={"file_metadata": {"name": "", "size": 7}}
+    )
+    assert make_fs().ls("/ds1/up1/a.txt") == [{"name": "/ds1/up1/a.txt", "size": 7, "type": "file"}]
+
+
+@responses.activate
+def test_dot_segments_are_refused():
+    with pytest.raises(ValueError):
+        make_fs().ls("/ds1/up1/../up2")
+
+
+@responses.activate
+def test_a_failing_folder_is_not_hidden_by_walk():
+    responses.get(f"{API}/uploads/up1/rawdir/", json=rawdir({"name": "sub", "is_file": False}))
+    responses.get(f"{API}/uploads/up1/rawdir/sub/", status=500, json={"detail": "boom"})
+    with pytest.raises(NomadError, match="boom"):
+        list(make_fs().walk("/ds1/up1"))
+
+
+@responses.activate
+def test_a_folder_lists_its_contents_after_its_parent_was_listed():
+    responses.get(f"{API}/uploads/up1/rawdir/", json=rawdir({"name": "sub", "is_file": False}))
+    responses.get(f"{API}/uploads/up1/rawdir/sub/", json=rawdir({"name": "b.txt", "is_file": True}))
+    fs = make_fs()
+    fs.ls("/ds1/up1")
+    assert fs.ls("/ds1/up1/sub", detail=False) == ["/ds1/up1/sub/b.txt"]
+
+
+def raw_params(**extra):
+    return matchers.query_param_matcher({"ignore_mime_type": "true", **extra})
+
+
+@responses.activate
+def test_get_file_streams_one_request(tmp_path):
+    responses.get(
+        f"{API}/uploads/up1/raw/data/run%20%231%20100%25.csv",
+        match=[raw_params()],
+        body=b"a,b\n1,2\n",
+    )
+    target = tmp_path / "out.csv"
+    make_fs().get_file("/ds1/up1/data/run #1 100%.csv", str(target))
+    assert target.read_bytes() == b"a,b\n1,2\n"
+    assert len(responses.calls) == 1
+
+
+@responses.activate
+def test_get_file_refuses_a_dataset_or_an_upload(tmp_path):
+    with pytest.raises(IsADirectoryError):
+        make_fs().get_file("/ds1/up1", str(tmp_path / "x"))
+
+
+@responses.activate
+def test_partial_reads_use_offset_and_length():
+    url = f"{API}/uploads/up1/raw/a.bin"
+    responses.get(
+        f"{API}/uploads/up1/rawdir/", json=rawdir({"name": "a.bin", "size": 10, "is_file": True})
+    )
+    responses.get(url, match=[raw_params(offset="2", length="3")], body=b"cde")
+    responses.get(url, match=[raw_params(length="2")], body=b"ab")
+    responses.get(url, match=[raw_params(offset="8")], body=b"ij")
+    responses.get(url, match=[raw_params(length="10")], body=b"abcdefghij")
+    fs = make_fs()
+    assert fs.cat_file("/ds1/up1/a.bin", start=2, end=5) == b"cde"
+    assert fs.cat_file("/ds1/up1/a.bin", end=2) == b"ab"
+    assert fs.cat_file("/ds1/up1/a.bin", start=3, end=3) == b""
+    assert fs.cat_file("/ds1/up1/a.bin", start=-2) == b"ij"
+    with fs.open("/ds1/up1/a.bin", "rb") as handle:
+        assert handle.read() == b"abcdefghij"
+
+
+@responses.activate
+def test_writing_is_refused():
+    with pytest.raises(PermissionError):
+        make_fs().open("/ds1/up1/new.txt", "wb")
+
+
+@responses.activate(registry=registries.OrderedRegistry)
+def test_a_full_aggregation_page_without_a_cursor_ends_the_listing(monkeypatch):
+    monkeypatch.setattr(filesystem, "AGGREGATION_PAGE_SIZE", 2)
+    responses.post(f"{API}/entries/query", json=aggregation("up1", "up2"))
+    assert make_fs().ls("/ds1", detail=False) == ["/ds1/up1", "/ds1/up2"]

nomad_fsspec-0.1.0/tests/test_live.py

@@ -0,0 +1,37 @@
+import pytest
+import requests
+
+from nomad_fsspec import NomadFileSystem
+
+pytestmark = pytest.mark.live
+
+DEMO_DATASET = "/wWgAnNZNQxOHLf97H62dgw"
+DEMO_FILE = (
+    f"{DEMO_DATASET}/Xa2LB__ESiKu-odRBPJhjA/nomad-demo-data/BrK_svSe/TBCC006.ABC/vasprun.xml.relax2"
+)
+
+
+@pytest.fixture(scope="module")
+def fs():
+    try:
+        requests.get("https://nomad-lab.eu/prod/v1/api/v1/info", timeout=20).raise_for_status()
+    except requests.RequestException as e:
+        pytest.skip(f"nomad-lab.eu is not reachable: {e}")
+    return NomadFileSystem(skip_instance_cache=True)
+
+
+def test_root_lists_public_datasets_by_name(fs):
+    root = fs.ls("/")
+    assert len(root) > 1000
+    assert any(entry["display_name"] == "demo example data" for entry in root)
+
+
+def test_demo_file_downloads_and_reads_in_parts(fs, tmp_path):
+    uploads = fs.ls(DEMO_DATASET)
+    assert [upload["display_name"] for upload in uploads] == ["nomad-demo-data.zip"]
+    assert fs.info(DEMO_FILE)["size"] == 424235
+    target = tmp_path / "vasprun.xml"
+    fs.get_file(DEMO_FILE, str(target))
+    data = target.read_bytes()
+    assert len(data) == 424235
+    assert fs.cat_file(DEMO_FILE, start=100, end=150) == data[100:150]

nomad_fsspec-0.1.0/tests/test_package.py

@@ -0,0 +1,7 @@
+import fsspec
+
+from nomad_fsspec import NomadFileSystem
+
+
+def test_registered_as_the_nomad_protocol():
+    assert isinstance(fsspec.filesystem("nomad", skip_instance_cache=True), NomadFileSystem)