essreduce 25.9.0__tar.gz → 25.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. {essreduce-25.9.0 → essreduce-25.10.0}/PKG-INFO +1 -1
  2. essreduce-25.10.0/src/ess/reduce/data/__init__.py +17 -0
  3. essreduce-25.10.0/src/ess/reduce/data/_registry.py +355 -0
  4. {essreduce-25.9.0 → essreduce-25.10.0}/src/essreduce.egg-info/PKG-INFO +1 -1
  5. {essreduce-25.9.0 → essreduce-25.10.0}/src/essreduce.egg-info/SOURCES.txt +3 -1
  6. essreduce-25.10.0/tests/conftest.py +103 -0
  7. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_generator_test.py +14 -7
  8. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/workflow_test.py +26 -20
  9. essreduce-25.9.0/src/ess/reduce/data.py +0 -154
  10. {essreduce-25.9.0 → essreduce-25.10.0}/.copier-answers.ess.yml +0 -0
  11. {essreduce-25.9.0 → essreduce-25.10.0}/.copier-answers.yml +0 -0
  12. {essreduce-25.9.0 → essreduce-25.10.0}/.github/ISSUE_TEMPLATE/high-level-requirement.yml +0 -0
  13. {essreduce-25.9.0 → essreduce-25.10.0}/.github/dependabot.yml +0 -0
  14. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/ci.yml +0 -0
  15. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/docs.yml +0 -0
  16. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/nightly_at_main.yml +0 -0
  17. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/nightly_at_main_lower_bound.yml +0 -0
  18. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/nightly_at_release.yml +0 -0
  19. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/python-version-ci +0 -0
  20. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/release.yml +0 -0
  21. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/test.yml +0 -0
  22. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/unpinned.yml +0 -0
  23. {essreduce-25.9.0 → essreduce-25.10.0}/.github/workflows/weekly_windows_macos.yml +0 -0
  24. {essreduce-25.9.0 → essreduce-25.10.0}/.gitignore +0 -0
  25. {essreduce-25.9.0 → essreduce-25.10.0}/.pre-commit-config.yaml +0 -0
  26. {essreduce-25.9.0 → essreduce-25.10.0}/.python-version +0 -0
  27. {essreduce-25.9.0 → essreduce-25.10.0}/CODE_OF_CONDUCT.md +0 -0
  28. {essreduce-25.9.0 → essreduce-25.10.0}/CONTRIBUTING.md +0 -0
  29. {essreduce-25.9.0 → essreduce-25.10.0}/LICENSE +0 -0
  30. {essreduce-25.9.0 → essreduce-25.10.0}/MANIFEST.in +0 -0
  31. {essreduce-25.9.0 → essreduce-25.10.0}/README.md +0 -0
  32. {essreduce-25.9.0 → essreduce-25.10.0}/docs/_static/anaconda-icon.js +0 -0
  33. {essreduce-25.9.0 → essreduce-25.10.0}/docs/_static/favicon.svg +0 -0
  34. {essreduce-25.9.0 → essreduce-25.10.0}/docs/_static/logo-dark.svg +0 -0
  35. {essreduce-25.9.0 → essreduce-25.10.0}/docs/_static/logo.svg +0 -0
  36. {essreduce-25.9.0 → essreduce-25.10.0}/docs/_templates/class-template.rst +0 -0
  37. {essreduce-25.9.0 → essreduce-25.10.0}/docs/_templates/doc_version.html +0 -0
  38. {essreduce-25.9.0 → essreduce-25.10.0}/docs/_templates/module-template.rst +0 -0
  39. {essreduce-25.9.0 → essreduce-25.10.0}/docs/about/index.md +0 -0
  40. {essreduce-25.9.0 → essreduce-25.10.0}/docs/api-reference/index.md +0 -0
  41. {essreduce-25.9.0 → essreduce-25.10.0}/docs/conf.py +0 -0
  42. {essreduce-25.9.0 → essreduce-25.10.0}/docs/developer/coding-conventions.md +0 -0
  43. {essreduce-25.9.0 → essreduce-25.10.0}/docs/developer/dependency-management.md +0 -0
  44. {essreduce-25.9.0 → essreduce-25.10.0}/docs/developer/getting-started.md +0 -0
  45. {essreduce-25.9.0 → essreduce-25.10.0}/docs/developer/gui.ipynb +0 -0
  46. {essreduce-25.9.0 → essreduce-25.10.0}/docs/developer/index.md +0 -0
  47. {essreduce-25.9.0 → essreduce-25.10.0}/docs/index.md +0 -0
  48. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/index.md +0 -0
  49. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/installation.md +0 -0
  50. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/reduction-workflow-guidelines.md +0 -0
  51. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/tof/dream.ipynb +0 -0
  52. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/tof/frame-unwrapping.ipynb +0 -0
  53. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/tof/index.md +0 -0
  54. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/tof/wfm.ipynb +0 -0
  55. {essreduce-25.9.0 → essreduce-25.10.0}/docs/user-guide/widget.md +0 -0
  56. {essreduce-25.9.0 → essreduce-25.10.0}/pyproject.toml +0 -0
  57. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/base.in +0 -0
  58. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/base.txt +0 -0
  59. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/basetest.in +0 -0
  60. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/basetest.txt +0 -0
  61. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/ci.in +0 -0
  62. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/ci.txt +0 -0
  63. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/dev.in +0 -0
  64. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/dev.txt +0 -0
  65. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/docs.in +0 -0
  66. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/docs.txt +0 -0
  67. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/make_base.py +0 -0
  68. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/mypy.in +0 -0
  69. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/mypy.txt +0 -0
  70. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/nightly.in +0 -0
  71. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/nightly.txt +0 -0
  72. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/static.in +0 -0
  73. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/static.txt +0 -0
  74. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/test.in +0 -0
  75. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/test.txt +0 -0
  76. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/wheels.in +0 -0
  77. {essreduce-25.9.0 → essreduce-25.10.0}/requirements/wheels.txt +0 -0
  78. {essreduce-25.9.0 → essreduce-25.10.0}/resources/logo.svg +0 -0
  79. {essreduce-25.9.0 → essreduce-25.10.0}/setup.cfg +0 -0
  80. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/__init__.py +0 -0
  81. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/live/__init__.py +0 -0
  82. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/live/raw.py +0 -0
  83. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/live/roi.py +0 -0
  84. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/live/workflow.py +0 -0
  85. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/logging.py +0 -0
  86. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/nexus/__init__.py +0 -0
  87. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/nexus/_nexus_loader.py +0 -0
  88. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/nexus/json_generator.py +0 -0
  89. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/nexus/json_nexus.py +0 -0
  90. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/nexus/types.py +0 -0
  91. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/nexus/workflow.py +0 -0
  92. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/parameter.py +0 -0
  93. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/py.typed +0 -0
  94. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/scripts/grow_nexus.py +0 -0
  95. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/streaming.py +0 -0
  96. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/__init__.py +0 -0
  97. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/eto_to_tof.py +0 -0
  98. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/fakes.py +0 -0
  99. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/interpolator_numba.py +0 -0
  100. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/interpolator_scipy.py +0 -0
  101. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/lut.py +0 -0
  102. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/resample.py +0 -0
  103. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/types.py +0 -0
  104. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/time_of_flight/workflow.py +0 -0
  105. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/ui.py +0 -0
  106. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/uncertainty.py +0 -0
  107. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/__init__.py +0 -0
  108. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_base.py +0 -0
  109. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_binedges_widget.py +0 -0
  110. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_bounds_widget.py +0 -0
  111. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_config.py +0 -0
  112. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_filename_widget.py +0 -0
  113. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_linspace_widget.py +0 -0
  114. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_optional_widget.py +0 -0
  115. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_spinner.py +0 -0
  116. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_string_widget.py +0 -0
  117. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_switchable_widget.py +0 -0
  118. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/widgets/_vector_widget.py +0 -0
  119. {essreduce-25.9.0 → essreduce-25.10.0}/src/ess/reduce/workflow.py +0 -0
  120. {essreduce-25.9.0 → essreduce-25.10.0}/src/essreduce.egg-info/dependency_links.txt +0 -0
  121. {essreduce-25.9.0 → essreduce-25.10.0}/src/essreduce.egg-info/entry_points.txt +0 -0
  122. {essreduce-25.9.0 → essreduce-25.10.0}/src/essreduce.egg-info/requires.txt +0 -0
  123. {essreduce-25.9.0 → essreduce-25.10.0}/src/essreduce.egg-info/top_level.txt +0 -0
  124. {essreduce-25.9.0 → essreduce-25.10.0}/tests/accumulators_test.py +0 -0
  125. {essreduce-25.9.0 → essreduce-25.10.0}/tests/live/raw_test.py +0 -0
  126. {essreduce-25.9.0 → essreduce-25.10.0}/tests/live/roi_test.py +0 -0
  127. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_examples/array_dataset.json +0 -0
  128. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_examples/dataset.json +0 -0
  129. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_examples/detector.json +0 -0
  130. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_examples/entry.json +0 -0
  131. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_examples/event_data.json +0 -0
  132. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_examples/instrument.json +0 -0
  133. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_examples/log.json +0 -0
  134. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/json_nexus_test.py +0 -0
  135. {essreduce-25.9.0 → essreduce-25.10.0}/tests/nexus/nexus_loader_test.py +0 -0
  136. {essreduce-25.9.0 → essreduce-25.10.0}/tests/package_test.py +0 -0
  137. {essreduce-25.9.0 → essreduce-25.10.0}/tests/scripts/test_grow_nexus.py +0 -0
  138. {essreduce-25.9.0 → essreduce-25.10.0}/tests/streaming_test.py +0 -0
  139. {essreduce-25.9.0 → essreduce-25.10.0}/tests/time_of_flight/interpolator_test.py +0 -0
  140. {essreduce-25.9.0 → essreduce-25.10.0}/tests/time_of_flight/lut_test.py +0 -0
  141. {essreduce-25.9.0 → essreduce-25.10.0}/tests/time_of_flight/resample_tests.py +0 -0
  142. {essreduce-25.9.0 → essreduce-25.10.0}/tests/time_of_flight/unwrap_test.py +0 -0
  143. {essreduce-25.9.0 → essreduce-25.10.0}/tests/time_of_flight/wfm_test.py +0 -0
  144. {essreduce-25.9.0 → essreduce-25.10.0}/tests/time_of_flight/workflow_test.py +0 -0
  145. {essreduce-25.9.0 → essreduce-25.10.0}/tests/uncertainty_test.py +0 -0
  146. {essreduce-25.9.0 → essreduce-25.10.0}/tests/widget_test.py +0 -0
  147. {essreduce-25.9.0 → essreduce-25.10.0}/tools/shrink_nexus.py +0 -0
  148. {essreduce-25.9.0 → essreduce-25.10.0}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: essreduce
3
- Version: 25.9.0
3
+ Version: 25.10.0
4
4
  Summary: Common data reduction tools for the ESS facility
5
5
  Author: Scipp contributors
6
6
  License-Expression: BSD-3-Clause
@@ -0,0 +1,17 @@
1
+ # SPDX-License-Identifier: BSD-3-Clause
2
+ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp)
3
+ """Data files bundled with ESSreduce.
4
+
5
+ This module requires the Pooch package which is not a hard dependency of ESSreduce.
6
+ It has to be installed separately with either pip or conda.
7
+ """
8
+
9
+ from ._registry import Entry, LocalRegistry, PoochRegistry, Registry, make_registry
10
+
11
+ __all__ = [
12
+ 'Entry',
13
+ 'LocalRegistry',
14
+ 'PoochRegistry',
15
+ 'Registry',
16
+ 'make_registry',
17
+ ]
@@ -0,0 +1,355 @@
1
+ # SPDX-License-Identifier: BSD-3-Clause
2
+ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp)
3
+
4
+ from __future__ import annotations
5
+
6
+ import hashlib
7
+ import os
8
+ from abc import ABC, abstractmethod
9
+ from collections.abc import Mapping
10
+ from dataclasses import dataclass
11
+ from functools import cache
12
+ from pathlib import Path
13
+ from typing import Any
14
+
15
+ _LOCAL_CACHE_ENV_VAR = "SCIPP_DATA_DIR"
16
+ _LOCAL_REGISTRY_ENV_VAR = "SCIPP_OVERRIDE_DATA_DIR"
17
+
18
+
19
+ def make_registry(
20
+ prefix: str,
21
+ files: Mapping[str, str | Entry],
22
+ *,
23
+ version: str,
24
+ base_url: str = "https://public.esss.dk/groups/scipp",
25
+ retry_if_failed: int = 3,
26
+ ) -> Registry:
27
+ """Create a file registry object.
28
+
29
+ By default, this function creates a :class:`PoochRegistry` to download files
30
+ via HTTP from an online file store.
31
+ This can be overridden by setting the environment variable `SCIPP_DATA_DIR` to a
32
+ path on the local file system.
33
+ In this case, a :class:`LocalRegistry` is returned.
34
+
35
+ Files are specified as a dict using either the Pooch string format explicitly
36
+ constructed :class:`Entry` objects:
37
+
38
+ >>> from ess.reduce.data import Entry
39
+ >>> files = {
40
+ ... "file1.dat": "md5:1234567890abcdef",
41
+ ... "file2.csv": Entry(alg="md5", chk="abcdef123456789"),
42
+ ... "folder/nested.dat": "blake2b:1234567890abcdef",
43
+ ... "zipped.zip": Entry(alg="blake2b", chk="abcdef123456789", unzip=True),
44
+ ... }
45
+
46
+ In the example above, the specifications for ``file1.dat`` and ``file2.csv`` are
47
+ essentially equivalent.
48
+ ``folder/nested.dat`` is a file in a subfolder.
49
+ Paths like this must always use forward slashes (/) even on Windows.
50
+
51
+ As shown above, it is possible to automatically unzip
52
+ files by specifying ``unzip=True``.
53
+ When calling ``registry.get_path("zipped.zip")`` the file will be unzipped and
54
+ a path to the content is returned.
55
+ This expects that there is only a single file in the zip archive.
56
+
57
+ The complete path to the source file is constructed as follows:
58
+
59
+ - Pooch: ``{base_url}/{prefix}/{version}/{name}``
60
+ - Local: ``{SCIPP_OVERRIDE_DATA_DIR}/{prefix}/{version}/{name}``
61
+
62
+ When using Pooch, files are downloaded to the user's cache directory.
63
+ This can be controlled with the environment variable ``SCIPP_CACHE_DIR``.
64
+
65
+ Parameters
66
+ ----------
67
+ prefix:
68
+ Prefix to add to all file names.
69
+ files:
70
+ Mapping of file names to checksums or :class:`Entry` objects.
71
+ version:
72
+ A version string for the files.
73
+ base_url:
74
+ URL for the online file store.
75
+ Ignored if the override environment variable is set.
76
+ retry_if_failed:
77
+ Number of retries when downloading a file.
78
+ Ignored if the override environment variable is set.
79
+
80
+ Returns
81
+ -------
82
+ :
83
+ Either a :class:`PoochRegistry` or :class:`LocalRegistry`.
84
+ """
85
+ if (override := os.environ.get(_LOCAL_REGISTRY_ENV_VAR)) is not None:
86
+ return LocalRegistry(
87
+ _check_local_override_path(override),
88
+ prefix,
89
+ files,
90
+ version=version,
91
+ base_url=base_url,
92
+ retry_if_failed=retry_if_failed,
93
+ )
94
+ return PoochRegistry(
95
+ prefix,
96
+ files,
97
+ version=version,
98
+ base_url=base_url,
99
+ retry_if_failed=retry_if_failed,
100
+ )
101
+
102
+
103
+ def _check_local_override_path(override: str) -> Path:
104
+ path = Path(override)
105
+ if not path.is_dir():
106
+ raise ValueError(
107
+ f"The data override path '{override}' is not a directory. If you want to "
108
+ "download files instead, unset the environment variable "
109
+ f"{_LOCAL_REGISTRY_ENV_VAR}."
110
+ )
111
+ return path
112
+
113
+
114
+ @dataclass(frozen=True, slots=True)
115
+ class Entry:
116
+ """An entry in a registry."""
117
+
118
+ chk: str
119
+ """Checksum."""
120
+ alg: str
121
+ """Checksum algorithm."""
122
+ unzip: bool = False
123
+ """Whether to unzip the file."""
124
+
125
+ @classmethod
126
+ def from_pooch_string(cls, pooch_string: str) -> Entry:
127
+ alg, chk = pooch_string.split(":")
128
+ return cls(chk=chk, alg=alg)
129
+
130
+
131
+ class Registry(ABC):
132
+ def __init__(self, files: Mapping[str, str | Entry]) -> None:
133
+ self._files = _to_file_entries(files)
134
+
135
+ @abstractmethod
136
+ def get_path(self, name: str) -> Path:
137
+ """Get the path to a file in the registry.
138
+
139
+ Depending on the implementation, the file is downloaded if necessary.
140
+
141
+ Note that implementations are allowed to cache return values of this method
142
+ to avoid recomputing potentially expensive checksums.
143
+ This usually means that the ``Registry`` object itself gets stored until the
144
+ Python interpreter shuts down.
145
+ However, registries are small and do not own resources.
146
+
147
+ Parameters
148
+ ----------
149
+ name:
150
+ Name of the file to get the path for.
151
+
152
+ Returns
153
+ -------
154
+ :
155
+ The Path to the file.
156
+ """
157
+
158
+ def _needs_unzip(self, name: str) -> bool:
159
+ return self._files[name].unzip
160
+
161
+
162
+ class PoochRegistry(Registry):
163
+ def __init__(
164
+ self,
165
+ prefix: str,
166
+ files: Mapping[str, str | Entry],
167
+ *,
168
+ version: str,
169
+ base_url: str,
170
+ retry_if_failed: int = 3,
171
+ ) -> None:
172
+ self._registry = _create_pooch(
173
+ prefix,
174
+ files,
175
+ version=version,
176
+ base_url=base_url,
177
+ retry_if_failed=retry_if_failed,
178
+ )
179
+ self._unzip_processor = _import_pooch().Unzip()
180
+ super().__init__(files)
181
+
182
+ @cache # noqa: B019
183
+ def get_path(self, name: str) -> Path:
184
+ """Get the path to a file in the registry.
185
+
186
+ Downloads the file if necessary.
187
+ """
188
+ if self._needs_unzip(name):
189
+ paths: list[str] = self._registry.fetch( # type: ignore[assignment]
190
+ name, processor=self._unzip_processor
191
+ )
192
+ return Path(_expect_single_unzipped(paths, name))
193
+ return Path(self._registry.fetch(name))
194
+
195
+
196
+ class LocalRegistry(Registry):
197
+ def __init__(
198
+ self,
199
+ source_path: Path,
200
+ prefix: str,
201
+ files: Mapping[str, str | Entry],
202
+ *,
203
+ version: str,
204
+ base_url: str,
205
+ retry_if_failed: int = 3,
206
+ ) -> None:
207
+ # Piggyback off of Pooch to determine the cache directory.
208
+ pooch_registry = _create_pooch(
209
+ prefix,
210
+ files,
211
+ version=version,
212
+ base_url=base_url,
213
+ retry_if_failed=retry_if_failed,
214
+ )
215
+ pooch = _import_pooch()
216
+ self._unzip_processor = pooch.processors.Unzip(extract_dir=pooch_registry.path)
217
+ self._source_path = source_path.resolve().joinpath(*prefix.split("/"), version)
218
+ super().__init__(files)
219
+
220
+ @cache # noqa: B019
221
+ def get_path(self, name: str) -> Path:
222
+ """Get the path to a file in the registry."""
223
+ try:
224
+ entry = self._files[name]
225
+ except KeyError:
226
+ raise ValueError(f"File '{name}' is not in the registry.") from None
227
+
228
+ path = self._local_path(name)
229
+ if not path.exists():
230
+ raise FileNotFoundError(
231
+ f"File '{name}' is registered but does not exist on the file system. "
232
+ f"Expected it at '{path}'."
233
+ )
234
+
235
+ _check_hash(name, path, entry)
236
+
237
+ if self._needs_unzip(name):
238
+ return Path(
239
+ _expect_single_unzipped(
240
+ self._unzip_processor(os.fspath(path), "download", None), path
241
+ )
242
+ )
243
+ return path
244
+
245
+ def _local_path(self, name: str) -> Path:
246
+ # Split on "/" because `name` is always a POSIX-style path, but the return
247
+ # value is a system path, i.e., it can be a Windows-style path.
248
+ return self._source_path.joinpath(*name.split("/"))
249
+
250
+
251
+ def _import_pooch() -> Any:
252
+ try:
253
+ import pooch
254
+ except ImportError:
255
+ raise ImportError(
256
+ "You need to install Pooch to access test and tutorial files. "
257
+ "See https://www.fatiando.org/pooch/latest/index.html"
258
+ ) from None
259
+
260
+ return pooch
261
+
262
+
263
+ def _create_pooch(
264
+ prefix: str,
265
+ files: Mapping[str, str | Entry],
266
+ *,
267
+ version: str,
268
+ base_url: str,
269
+ retry_if_failed: int = 3,
270
+ ) -> Any:
271
+ pooch = _import_pooch()
272
+ return pooch.create(
273
+ path=pooch.os_cache(prefix),
274
+ env=_LOCAL_CACHE_ENV_VAR,
275
+ base_url=f'{base_url}/{prefix}/{version}/',
276
+ registry=_to_pooch_registry(files),
277
+ retry_if_failed=retry_if_failed,
278
+ )
279
+
280
+
281
+ def _pooch_unzip_processor(extract_dir: Path) -> Any:
282
+ try:
283
+ import pooch
284
+ except ImportError:
285
+ raise ImportError("You need to install Pooch to unzip files.") from None
286
+
287
+ return pooch.processors.Unzip(extract_dir=os.fspath(extract_dir))
288
+
289
+
290
+ def _expect_single_unzipped(paths: list[str], archive: str | os.PathLike) -> str:
291
+ if len(paths) != 1:
292
+ raise ValueError(
293
+ f"Expected exactly one file to unzip, got {len(paths)} in "
294
+ f"'{os.fspath(archive)}'."
295
+ )
296
+ return paths[0]
297
+
298
+
299
+ def _check_hash(name: str, path: Path, entry: Entry) -> None:
300
+ new_chk = _checksum_of_file(path, algorithm=entry.alg)
301
+ if new_chk.lower() != entry.chk.lower():
302
+ raise ValueError(
303
+ f"{entry.alg} hash of file '{name}' does not match the known hash: "
304
+ f"expected {entry.chk}, got {new_chk}."
305
+ )
306
+
307
+
308
+ def _to_file_entries(files: Mapping[str, str | Entry]) -> dict[str, Entry]:
309
+ return {
310
+ name: entry if isinstance(entry, Entry) else Entry.from_pooch_string(entry)
311
+ for name, entry in files.items()
312
+ }
313
+
314
+
315
+ def _to_pooch_registry(files: Mapping[str, str | Entry]) -> dict[str, str]:
316
+ return {
317
+ name: f"{entry.alg}:{entry.chk}" if isinstance(entry, Entry) else entry
318
+ for name, entry in files.items()
319
+ }
320
+
321
+
322
+ # Code taken from Scitacean and Pooch.
323
+ def _checksum_of_file(path: Path, *, algorithm: str) -> str:
324
+ """Compute the checksum of a local file.
325
+
326
+ Parameters
327
+ ----------
328
+ path:
329
+ Path of the file.
330
+ algorithm:
331
+ Hash algorithm to use.
332
+ Can be any algorithm supported by :func:`hashlib.new`.
333
+
334
+ Returns
335
+ -------
336
+ :
337
+ The hex digest of the hash.
338
+ """
339
+ chk = _new_hash(algorithm)
340
+ # size based on http://git.savannah.gnu.org/gitweb/?p=coreutils.git;a=blob;f=src/ioblksize.h;h=ed2f4a9c4d77462f357353eb73ee4306c28b37f1;hb=HEAD#l23 # noqa: E501
341
+ buffer = memoryview(bytearray(128 * 1024))
342
+ with open(path, "rb", buffering=0) as file:
343
+ for n in iter(lambda: file.readinto(buffer), 0):
344
+ chk.update(buffer[:n])
345
+ return chk.hexdigest() # type: ignore[no-any-return]
346
+
347
+
348
+ def _new_hash(algorithm: str) -> Any:
349
+ # Try to use a named constructor instead of hashlib.new where possible
350
+ # because that is supposed to be faster, according to
351
+ # https://docs.python.org/3/library/hashlib.html#hashlib.new
352
+ try:
353
+ return getattr(hashlib, algorithm)()
354
+ except AttributeError:
355
+ return hashlib.new(algorithm, usedforsecurity=False)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: essreduce
3
- Version: 25.9.0
3
+ Version: 25.10.0
4
4
  Summary: Common data reduction tools for the ESS facility
5
5
  Author: Scipp contributors
6
6
  License-Expression: BSD-3-Clause
@@ -69,7 +69,6 @@ requirements/wheels.in
69
69
  requirements/wheels.txt
70
70
  resources/logo.svg
71
71
  src/ess/reduce/__init__.py
72
- src/ess/reduce/data.py
73
72
  src/ess/reduce/logging.py
74
73
  src/ess/reduce/parameter.py
75
74
  src/ess/reduce/py.typed
@@ -77,6 +76,8 @@ src/ess/reduce/streaming.py
77
76
  src/ess/reduce/ui.py
78
77
  src/ess/reduce/uncertainty.py
79
78
  src/ess/reduce/workflow.py
79
+ src/ess/reduce/data/__init__.py
80
+ src/ess/reduce/data/_registry.py
80
81
  src/ess/reduce/live/__init__.py
81
82
  src/ess/reduce/live/raw.py
82
83
  src/ess/reduce/live/roi.py
@@ -116,6 +117,7 @@ src/essreduce.egg-info/entry_points.txt
116
117
  src/essreduce.egg-info/requires.txt
117
118
  src/essreduce.egg-info/top_level.txt
118
119
  tests/accumulators_test.py
120
+ tests/conftest.py
119
121
  tests/package_test.py
120
122
  tests/streaming_test.py
121
123
  tests/uncertainty_test.py
@@ -0,0 +1,103 @@
1
+ # SPDX-License-Identifier: BSD-3-Clause
2
+ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp)
3
+
4
+ from pathlib import Path
5
+
6
+ import pytest
7
+
8
+ from ess.reduce.data import Registry, make_registry
9
+
10
+
11
+ @pytest.fixture(scope='session')
12
+ def bifrost_registry() -> Registry:
13
+ return make_registry(
14
+ 'ess/bifrost',
15
+ files={
16
+ "BIFROST_20240914T053723.h5": "md5:0f2fa5c9a851f8e3a4fa61defaa3752e",
17
+ },
18
+ version='1',
19
+ )
20
+
21
+
22
+ @pytest.fixture(scope='session')
23
+ def dream_registry() -> Registry:
24
+ return make_registry(
25
+ 'ess/dream',
26
+ files={
27
+ "TEST_977695_00068064.hdf": "md5:9e6ee9ec70d7c5e8c0c93b9e07e8949f",
28
+ },
29
+ version='2',
30
+ )
31
+
32
+
33
+ @pytest.fixture(scope='session')
34
+ def loki_registry() -> Registry:
35
+ return make_registry(
36
+ 'ess/loki',
37
+ files={
38
+ # Files from LoKI@Larmor detector test experiment
39
+ #
40
+ # Background run 1 (no sample, sample holder/can only, no transmission monitor) # noqa: E501
41
+ '60248-2022-02-28_2215.nxs': 'md5:d9f17b95274a0fc6468df7e39df5bf03',
42
+ # Sample run 1 (sample + sample holder/can, no transmission monitor in beam)
43
+ '60250-2022-02-28_2215.nxs': 'md5:6a519ceaacbae702a6d08241e86799b1',
44
+ # Sample run 2 (sample + sample holder/can, no transmission monitor in beam)
45
+ '60339-2022-02-28_2215.nxs': 'md5:03c86f6389566326bb0cbbd80b8f8c4f',
46
+ # Background transmission run (sample holder/can + transmission monitor)
47
+ '60392-2022-02-28_2215.nxs': 'md5:9ecc1a9a2c05a880144afb299fc11042',
48
+ # Background run 2 (no sample, sample holder/can only, no transmission monitor) # noqa: E501
49
+ '60393-2022-02-28_2215.nxs': 'md5:bf550d0ba29931f11b7450144f658652',
50
+ # Sample transmission run (sample + sample holder/can + transmission monitor) # noqa: E501
51
+ '60394-2022-02-28_2215.nxs': 'md5:c40f38a62337d86957af925296c4c615',
52
+ # Analytical model for the I(Q) of the Poly-Gauss sample
53
+ 'PolyGauss_I0-50_Rg-60.h5': 'md5:f5d60d9c2286cb197b8cd4dc82db3d7e',
54
+ # XML file for the pixel mask
55
+ 'mask_new_July2022.xml': 'md5:421b6dc9db74126ffbc5d88164d017b0',
56
+ },
57
+ version='2',
58
+ )
59
+
60
+
61
+ @pytest.fixture(scope='session')
62
+ def bifrost_simulated_elastic(bifrost_registry: Registry) -> Path:
63
+ """McStas simulation with elastic incoherent scattering + phonon."""
64
+ return bifrost_registry.get_path('BIFROST_20240914T053723.h5')
65
+
66
+
67
+ @pytest.fixture(scope='session')
68
+ def loki_tutorial_sample_run_60250(loki_registry: Registry) -> Path:
69
+ """Sample run with sample and sample holder/can, no transmission monitor in beam."""
70
+ return loki_registry.get_path('60250-2022-02-28_2215.nxs')
71
+
72
+
73
+ @pytest.fixture(scope='session')
74
+ def loki_tutorial_sample_run_60339(loki_registry: Registry) -> Path:
75
+ """Sample run with sample and sample holder/can, no transmission monitor in beam."""
76
+ return loki_registry.get_path('60339-2022-02-28_2215.nxs')
77
+
78
+
79
+ @pytest.fixture(scope='session')
80
+ def loki_tutorial_background_run_60248(loki_registry: Registry) -> Path:
81
+ """Background run with sample holder/can only, no transmission monitor."""
82
+ return loki_registry.get_path('60248-2022-02-28_2215.nxs')
83
+
84
+
85
+ @pytest.fixture(scope='session')
86
+ def loki_tutorial_background_run_60393(loki_registry: Registry) -> Path:
87
+ """Background run with sample holder/can only, no transmission monitor."""
88
+ return loki_registry.get_path('60393-2022-02-28_2215.nxs')
89
+
90
+
91
+ @pytest.fixture(scope='session')
92
+ def loki_tutorial_sample_transmission_run(loki_registry: Registry) -> Path:
93
+ """Sample transmission run (sample + sample holder/can + transmission monitor)."""
94
+ return loki_registry.get_path('60394-2022-02-28_2215.nxs')
95
+
96
+
97
+ @pytest.fixture(scope='session')
98
+ def dream_coda_test_file(dream_registry: Registry) -> Path:
99
+ """CODA file for DREAM where most pulses have been removed.
100
+
101
+ See ``tools/shrink_nexus.py``.
102
+ """
103
+ return dream_registry.get_path('TEST_977695_00068064.hdf')
@@ -1,16 +1,19 @@
1
1
  # SPDX-License-Identifier: BSD-3-Clause
2
2
  # Copyright (c) 2024 Scipp contributors (https://github.com/scipp)
3
+ from pathlib import Path
4
+
3
5
  import pytest
4
6
  import scippnexus as snx
5
7
  from scipp.testing import assert_identical
6
8
 
7
- from ess.reduce import data
8
9
  from ess.reduce.nexus.json_generator import event_data_generator
9
10
  from ess.reduce.nexus.json_nexus import json_nexus_group
10
11
 
11
12
 
12
- def test_event_data_generator_monitor_events_round_trip() -> None:
13
- filename = data.loki_tutorial_sample_run_60250()
13
+ def test_event_data_generator_monitor_events_round_trip(
14
+ loki_tutorial_sample_run_60250: Path,
15
+ ) -> None:
16
+ filename = loki_tutorial_sample_run_60250
14
17
  monitor = snx.load(filename, root='entry/instrument/monitor_1/monitor_1_events')
15
18
  generator = event_data_generator(monitor)
16
19
  for i in range(len(monitor)):
@@ -20,8 +23,10 @@ def test_event_data_generator_monitor_events_round_trip() -> None:
20
23
  next(generator)
21
24
 
22
25
 
23
- def test_event_data_generator_detector_events_round_trip() -> None:
24
- filename = data.loki_tutorial_sample_run_60250()
26
+ def test_event_data_generator_detector_events_round_trip(
27
+ loki_tutorial_sample_run_60250: Path,
28
+ ) -> None:
29
+ filename = loki_tutorial_sample_run_60250
25
30
  detector = snx.load(
26
31
  filename, root='entry/instrument/larmor_detector/larmor_detector_events'
27
32
  )
@@ -31,8 +36,10 @@ def test_event_data_generator_detector_events_round_trip() -> None:
31
36
  assert_identical(group[()], detector[i : i + 1])
32
37
 
33
38
 
34
- def test_event_data_generator_without_event_id_yields_ones() -> None:
35
- filename = data.loki_tutorial_sample_run_60250()
39
+ def test_event_data_generator_without_event_id_yields_ones(
40
+ loki_tutorial_sample_run_60250: Path,
41
+ ) -> None:
42
+ filename = loki_tutorial_sample_run_60250
36
43
  base = snx.load(filename, root='entry/instrument/monitor_1/monitor_1_events')
37
44
  monitor = base.bins.drop_coords('event_id')
38
45
  generator = event_data_generator(monitor)
@@ -1,13 +1,13 @@
1
1
  # SPDX-License-Identifier: BSD-3-Clause
2
2
  # Copyright (c) 2024 Scipp contributors (https://github.com/scipp)
3
3
  from datetime import UTC, datetime
4
+ from pathlib import Path
4
5
 
5
6
  import pytest
6
7
  import scipp as sc
7
8
  import scippnexus as snx
8
9
  from scipp.testing import assert_identical
9
10
 
10
- from ess.reduce import data
11
11
  from ess.reduce.nexus import compute_component_position, workflow
12
12
  from ess.reduce.nexus.types import (
13
13
  BackgroundRun,
@@ -176,11 +176,11 @@ def test_to_transform_raises_if_interval_does_not_yield_unique_value(
176
176
  )
177
177
 
178
178
 
179
- def test_given_no_sample_load_nexus_sample_returns_group_with_origin_depends_on() -> (
180
- None
181
- ):
179
+ def test_given_no_sample_load_nexus_sample_returns_group_with_origin_depends_on(
180
+ loki_tutorial_sample_run_60250: Path,
181
+ ) -> None:
182
182
  filespec = workflow.file_path_to_file_spec(
183
- data.loki_tutorial_sample_run_60250(), preopen=True
183
+ loki_tutorial_sample_run_60250, preopen=True
184
184
  )
185
185
  spec = workflow.unique_component_spec(filespec)
186
186
  assert spec.filename['/entry'][snx.NXsample] == {}
@@ -549,9 +549,9 @@ def test_assemble_monitor_preserves_masks(calibrated_monitor, monitor_event_data
549
549
  assert 'mymask' in monitor_data.masks
550
550
 
551
551
 
552
- def test_load_event_monitor_workflow() -> None:
552
+ def test_load_event_monitor_workflow(loki_tutorial_sample_run_60250: Path) -> None:
553
553
  wf = LoadMonitorWorkflow(run_types=[SampleRun], monitor_types=[FrameMonitor1])
554
- wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250()
554
+ wf[Filename[SampleRun]] = loki_tutorial_sample_run_60250
555
555
  wf[NeXusName[FrameMonitor1]] = 'monitor_1'
556
556
  da = wf.compute(MonitorData[SampleRun, FrameMonitor1])
557
557
  assert 'position' in da.coords
@@ -561,9 +561,9 @@ def test_load_event_monitor_workflow() -> None:
561
561
  assert da.bins.constituents['data'].variances is not None
562
562
 
563
563
 
564
- def test_load_histogram_monitor_workflow() -> None:
564
+ def test_load_histogram_monitor_workflow(dream_coda_test_file: Path) -> None:
565
565
  wf = LoadMonitorWorkflow(run_types=[SampleRun], monitor_types=[FrameMonitor1])
566
- wf[Filename[SampleRun]] = data.dream_coda_test_file()
566
+ wf[Filename[SampleRun]] = dream_coda_test_file
567
567
  wf[NeXusName[FrameMonitor1]] = 'monitor_bunker'
568
568
  da = wf.compute(MonitorData[SampleRun, FrameMonitor1])
569
569
  assert 'position' in da.coords
@@ -575,9 +575,9 @@ def test_load_histogram_monitor_workflow() -> None:
575
575
  assert da.variances is not None
576
576
 
577
577
 
578
- def test_load_detector_workflow() -> None:
578
+ def test_load_detector_workflow(loki_tutorial_sample_run_60250: Path) -> None:
579
579
  wf = LoadDetectorWorkflow(run_types=[SampleRun], monitor_types=[])
580
- wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250()
580
+ wf[Filename[SampleRun]] = loki_tutorial_sample_run_60250
581
581
  wf[NeXusName[snx.NXdetector]] = 'larmor_detector'
582
582
  da = wf.compute(DetectorData[SampleRun])
583
583
  assert 'position' in da.coords
@@ -588,9 +588,11 @@ def test_load_detector_workflow() -> None:
588
588
 
589
589
 
590
590
  @pytest.mark.parametrize('preopen', [True, False])
591
- def test_generic_nexus_workflow(preopen: bool) -> None:
591
+ def test_generic_nexus_workflow(
592
+ preopen: bool, loki_tutorial_sample_run_60250: Path
593
+ ) -> None:
592
594
  wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[FrameMonitor1])
593
- wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250()
595
+ wf[Filename[SampleRun]] = loki_tutorial_sample_run_60250
594
596
  wf[NeXusName[FrameMonitor1]] = 'monitor_1'
595
597
  wf[NeXusName[snx.NXdetector]] = 'larmor_detector'
596
598
  wf[PreopenNeXusFile] = preopen
@@ -607,9 +609,9 @@ def test_generic_nexus_workflow(preopen: bool) -> None:
607
609
  assert da.dims == ('event_time_zero',)
608
610
 
609
611
 
610
- def test_generic_nexus_workflow_load_choppers() -> None:
612
+ def test_generic_nexus_workflow_load_choppers(bifrost_simulated_elastic: Path) -> None:
611
613
  wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[])
612
- wf[Filename[SampleRun]] = data.bifrost_simulated_elastic()
614
+ wf[Filename[SampleRun]] = bifrost_simulated_elastic
613
615
  choppers = wf.compute(RawChoppers[SampleRun])
614
616
 
615
617
  assert choppers.keys() == {
@@ -626,9 +628,11 @@ def test_generic_nexus_workflow_load_choppers() -> None:
626
628
  assert chopper['slit_edges'].shape == (2,)
627
629
 
628
630
 
629
- def test_generic_nexus_workflow_load_beamline_metadata() -> None:
631
+ def test_generic_nexus_workflow_load_beamline_metadata(
632
+ bifrost_simulated_elastic: Path,
633
+ ) -> None:
630
634
  wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[])
631
- wf[Filename[SampleRun]] = data.bifrost_simulated_elastic()
635
+ wf[Filename[SampleRun]] = bifrost_simulated_elastic
632
636
  beamline = wf.compute(Beamline)
633
637
 
634
638
  assert beamline.name == 'BIFROST'
@@ -636,10 +640,12 @@ def test_generic_nexus_workflow_load_beamline_metadata() -> None:
636
640
  assert beamline.site == 'ESS'
637
641
 
638
642
 
639
- def test_generic_nexus_workflow_load_measurement_metadata() -> None:
643
+ def test_generic_nexus_workflow_load_measurement_metadata(
644
+ loki_tutorial_sample_run_60250: Path, loki_tutorial_background_run_60248: Path
645
+ ) -> None:
640
646
  wf = GenericNeXusWorkflow(run_types=[SampleRun], monitor_types=[])
641
- wf[Filename[SampleRun]] = data.loki_tutorial_sample_run_60250()
642
- wf[Filename[BackgroundRun]] = data.loki_tutorial_background_run_60248()
647
+ wf[Filename[SampleRun]] = loki_tutorial_sample_run_60250
648
+ wf[Filename[BackgroundRun]] = loki_tutorial_background_run_60248
643
649
  measurement = wf.compute(Measurement)
644
650
 
645
651
  assert measurement.title == 'My experiment'