hcx 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hcx-0.1.0/PKG-INFO +87 -0
- hcx-0.1.0/README.md +78 -0
- hcx-0.1.0/pyproject.toml +89 -0
- hcx-0.1.0/src/hcx/__init__.py +30 -0
- hcx-0.1.0/src/hcx/batch.py +35 -0
- hcx-0.1.0/src/hcx/conformance.py +220 -0
- hcx-0.1.0/src/hcx/models/__init__.py +3 -0
- hcx-0.1.0/src/hcx/models/lstm.py +165 -0
- hcx-0.1.0/src/hcx/output.py +15 -0
- hcx-0.1.0/src/hcx/protocol.py +29 -0
- hcx-0.1.0/src/hcx/py.typed +0 -0
- hcx-0.1.0/src/hcx/specifications.py +92 -0
- hcx-0.1.0/src/hcx/synthetic.py +90 -0
hcx-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: hcx
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: hcx
|
|
5
|
+
Requires-Dist: numpy
|
|
6
|
+
Requires-Dist: torch
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
|
|
10
|
+
# hcx
|
|
11
|
+
|
|
12
|
+
hcx defines typed, runtime-usable contracts for hydrological model batches,
|
|
13
|
+
forecasts, output specifications, models, and model factories.
|
|
14
|
+
|
|
15
|
+
The [contract specification](docs/spec.md) is normative. Public docstrings and
|
|
16
|
+
this README are explanatory summaries of that specification.
|
|
17
|
+
|
|
18
|
+
The distribution combines three parts: the normative specification, an
|
|
19
|
+
inline-typed Python contract marked with `py.typed`, and a standalone
|
|
20
|
+
conformance harness. Independent model packages can import the same contract
|
|
21
|
+
and prove their behavior without an application, training engine, or external
|
|
22
|
+
dataset.
|
|
23
|
+
|
|
24
|
+
## Authoring a model package
|
|
25
|
+
|
|
26
|
+
1. Add `hcx` as a runtime dependency of the model package.
|
|
27
|
+
2. Implement a `torch.nn.Module` whose `forward(batch: Batch) -> Forecast`
|
|
28
|
+
behavior satisfies `ForecastModel`, including the normative shape,
|
|
29
|
+
dtype/device, and metadata identity rules.
|
|
30
|
+
3. Implement a callable matching `ModelFactory`. It accepts a model-specific
|
|
31
|
+
`dict[str, object]` and the keyword-only ordered `dynamic_inputs` and
|
|
32
|
+
`static_inputs`, resolved `input_size`, `static_size`, and `output_size`, and
|
|
33
|
+
a resolved `OutputSpecification[object]`. It returns the module. Preserve
|
|
34
|
+
name order; the sizes are resolved from the first batch.
|
|
35
|
+
4. Declare the factory in the model package's `pyproject.toml`:
|
|
36
|
+
|
|
37
|
+
```toml
|
|
38
|
+
[project.entry-points.'hcx.models']
|
|
39
|
+
my_model = "my_model_package.factory:create_model"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
5. Exercise the complete contract with synthetic data:
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from importlib.metadata import entry_points
|
|
46
|
+
|
|
47
|
+
from hcx import Point, assert_conforms, make_synthetic_batch
|
|
48
|
+
|
|
49
|
+
batch = make_synthetic_batch()
|
|
50
|
+
assert batch.scalar_dynamic is not None
|
|
51
|
+
assert batch.scalar_static is not None
|
|
52
|
+
|
|
53
|
+
factory = entry_points(group="hcx.models", name="my_model")[0].load()
|
|
54
|
+
dynamic_inputs = [
|
|
55
|
+
f"dynamic_{index}" for index in range(batch.scalar_dynamic.shape[-1])
|
|
56
|
+
]
|
|
57
|
+
static_inputs = [
|
|
58
|
+
f"static_{index}" for index in range(batch.scalar_static.shape[-1])
|
|
59
|
+
]
|
|
60
|
+
model = factory(
|
|
61
|
+
{},
|
|
62
|
+
dynamic_inputs=dynamic_inputs,
|
|
63
|
+
static_inputs=static_inputs,
|
|
64
|
+
input_size=batch.scalar_dynamic.shape[-1],
|
|
65
|
+
static_size=batch.scalar_static.shape[-1],
|
|
66
|
+
output_size=batch.target.shape[-1],
|
|
67
|
+
output_specification=Point(),
|
|
68
|
+
)
|
|
69
|
+
assert_conforms(model, batch)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The factory can also be imported and called directly in this smoke test. hcx
|
|
73
|
+
packages `scalar_lstm` as a reference entry point, not as a required base class.
|
|
74
|
+
Entry-point names are consumer-facing identifiers and should remain stable and
|
|
75
|
+
unique within an environment.
|
|
76
|
+
|
|
77
|
+
See the normative [factory and entry-point clauses](docs/spec.md#6-model-factory-and-hcxmodels-entry-points)
|
|
78
|
+
and the [changelog](CHANGELOG.md) for version history.
|
|
79
|
+
|
|
80
|
+
## Releases
|
|
81
|
+
|
|
82
|
+
Maintainers bump the version with bump-my-version and update the changelog in
|
|
83
|
+
the same commit. A human then creates and publishes the GitHub Release, which
|
|
84
|
+
creates the `vX.Y.Z` tag. The GitHub Actions workflow builds distributions with
|
|
85
|
+
`uv build` and publishes them through OIDC trusted publishing. Prereleases and
|
|
86
|
+
manual workflow dispatches target TestPyPI; published non-prereleases target
|
|
87
|
+
PyPI. Local publishing and hand-made tags are forbidden.
|
hcx-0.1.0/README.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# hcx
|
|
2
|
+
|
|
3
|
+
hcx defines typed, runtime-usable contracts for hydrological model batches,
|
|
4
|
+
forecasts, output specifications, models, and model factories.
|
|
5
|
+
|
|
6
|
+
The [contract specification](docs/spec.md) is normative. Public docstrings and
|
|
7
|
+
this README are explanatory summaries of that specification.
|
|
8
|
+
|
|
9
|
+
The distribution combines three parts: the normative specification, an
|
|
10
|
+
inline-typed Python contract marked with `py.typed`, and a standalone
|
|
11
|
+
conformance harness. Independent model packages can import the same contract
|
|
12
|
+
and prove their behavior without an application, training engine, or external
|
|
13
|
+
dataset.
|
|
14
|
+
|
|
15
|
+
## Authoring a model package
|
|
16
|
+
|
|
17
|
+
1. Add `hcx` as a runtime dependency of the model package.
|
|
18
|
+
2. Implement a `torch.nn.Module` whose `forward(batch: Batch) -> Forecast`
|
|
19
|
+
behavior satisfies `ForecastModel`, including the normative shape,
|
|
20
|
+
dtype/device, and metadata identity rules.
|
|
21
|
+
3. Implement a callable matching `ModelFactory`. It accepts a model-specific
|
|
22
|
+
`dict[str, object]` and the keyword-only ordered `dynamic_inputs` and
|
|
23
|
+
`static_inputs`, resolved `input_size`, `static_size`, and `output_size`, and
|
|
24
|
+
a resolved `OutputSpecification[object]`. It returns the module. Preserve
|
|
25
|
+
name order; the sizes are resolved from the first batch.
|
|
26
|
+
4. Declare the factory in the model package's `pyproject.toml`:
|
|
27
|
+
|
|
28
|
+
```toml
|
|
29
|
+
[project.entry-points.'hcx.models']
|
|
30
|
+
my_model = "my_model_package.factory:create_model"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
5. Exercise the complete contract with synthetic data:
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from importlib.metadata import entry_points
|
|
37
|
+
|
|
38
|
+
from hcx import Point, assert_conforms, make_synthetic_batch
|
|
39
|
+
|
|
40
|
+
batch = make_synthetic_batch()
|
|
41
|
+
assert batch.scalar_dynamic is not None
|
|
42
|
+
assert batch.scalar_static is not None
|
|
43
|
+
|
|
44
|
+
factory = entry_points(group="hcx.models", name="my_model")[0].load()
|
|
45
|
+
dynamic_inputs = [
|
|
46
|
+
f"dynamic_{index}" for index in range(batch.scalar_dynamic.shape[-1])
|
|
47
|
+
]
|
|
48
|
+
static_inputs = [
|
|
49
|
+
f"static_{index}" for index in range(batch.scalar_static.shape[-1])
|
|
50
|
+
]
|
|
51
|
+
model = factory(
|
|
52
|
+
{},
|
|
53
|
+
dynamic_inputs=dynamic_inputs,
|
|
54
|
+
static_inputs=static_inputs,
|
|
55
|
+
input_size=batch.scalar_dynamic.shape[-1],
|
|
56
|
+
static_size=batch.scalar_static.shape[-1],
|
|
57
|
+
output_size=batch.target.shape[-1],
|
|
58
|
+
output_specification=Point(),
|
|
59
|
+
)
|
|
60
|
+
assert_conforms(model, batch)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The factory can also be imported and called directly in this smoke test. hcx
|
|
64
|
+
packages `scalar_lstm` as a reference entry point, not as a required base class.
|
|
65
|
+
Entry-point names are consumer-facing identifiers and should remain stable and
|
|
66
|
+
unique within an environment.
|
|
67
|
+
|
|
68
|
+
See the normative [factory and entry-point clauses](docs/spec.md#6-model-factory-and-hcxmodels-entry-points)
|
|
69
|
+
and the [changelog](CHANGELOG.md) for version history.
|
|
70
|
+
|
|
71
|
+
## Releases
|
|
72
|
+
|
|
73
|
+
Maintainers bump the version with bump-my-version and update the changelog in
|
|
74
|
+
the same commit. A human then creates and publishes the GitHub Release, which
|
|
75
|
+
creates the `vX.Y.Z` tag. The GitHub Actions workflow builds distributions with
|
|
76
|
+
`uv build` and publishes them through OIDC trusted publishing. Prereleases and
|
|
77
|
+
manual workflow dispatches target TestPyPI; published non-prereleases target
|
|
78
|
+
PyPI. Local publishing and hand-made tags are forbidden.
|
hcx-0.1.0/pyproject.toml
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hcx"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "hcx"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"numpy",
|
|
9
|
+
"torch",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
[project.entry-points.'hcx.models']
|
|
13
|
+
scalar_lstm = "hcx.models.lstm:factory"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
[build-system]
|
|
17
|
+
requires = ["uv_build>=0.9.13,<0.10.0"]
|
|
18
|
+
build-backend = "uv_build"
|
|
19
|
+
|
|
20
|
+
[dependency-groups]
|
|
21
|
+
dev = [
|
|
22
|
+
"pytest>=8.3", # tests
|
|
23
|
+
"pytest-cov>=6.2.1",
|
|
24
|
+
"ruff>=0.12", # linter + formatter
|
|
25
|
+
"bump-my-version>=1.2.7",
|
|
26
|
+
"ty>=0.0.58",
|
|
27
|
+
"pyyaml>=6.0.3",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# -----------------------
|
|
31
|
+
# Ruff (lint + format)
|
|
32
|
+
# -----------------------
|
|
33
|
+
[tool.ruff]
|
|
34
|
+
line-length = 120
|
|
35
|
+
target-version = "py311"
|
|
36
|
+
# extend-exclude = ["notebooks/", "experiments/"]
|
|
37
|
+
|
|
38
|
+
[tool.ruff.lint]
|
|
39
|
+
# Errors, warnings, naming, imports, modern py, bugbear, comprehensions, simplify
|
|
40
|
+
select = ["E", "F", "W", "N", "I", "UP", "B", "C4", "SIM"]
|
|
41
|
+
ignore = [
|
|
42
|
+
"E501", # let formatter handle wrapping
|
|
43
|
+
"N803", # allow X, Y for ML-style params
|
|
44
|
+
"N806", # allow X, Y for local vars
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[tool.ruff.format]
|
|
48
|
+
quote-style = "double"
|
|
49
|
+
indent-style = "space"
|
|
50
|
+
|
|
51
|
+
# # -----------------------
|
|
52
|
+
# # ty (fast static type checker)
|
|
53
|
+
# # -----------------------
|
|
54
|
+
# [tool.ty]
|
|
55
|
+
# [tool.ty.rules]
|
|
56
|
+
# possibly-unresolved-reference = "warn"
|
|
57
|
+
|
|
58
|
+
# -----------------------
|
|
59
|
+
# Pytest
|
|
60
|
+
# -----------------------
|
|
61
|
+
[tool.pytest.ini_options]
|
|
62
|
+
pythonpath = ["src"]
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
python_files = ["test_*.py", "*_test.py"]
|
|
65
|
+
python_classes = ["Test*"]
|
|
66
|
+
python_functions = ["test_*"]
|
|
67
|
+
addopts = ["--strict-markers", "--strict-config", "-ra"]
|
|
68
|
+
|
|
69
|
+
# -----------------------
|
|
70
|
+
# Bump My Version
|
|
71
|
+
# -----------------------
|
|
72
|
+
[tool.bumpversion]
|
|
73
|
+
current_version = "0.1.0"
|
|
74
|
+
commit = false
|
|
75
|
+
tag = false
|
|
76
|
+
allow_dirty = true
|
|
77
|
+
|
|
78
|
+
[[tool.bumpversion.files]]
|
|
79
|
+
filename = "pyproject.toml"
|
|
80
|
+
search = 'version = "{current_version}"'
|
|
81
|
+
replace = 'version = "{new_version}"'
|
|
82
|
+
|
|
83
|
+
[[tool.bumpversion.files]]
|
|
84
|
+
filename = "src/hcx/__init__.py"
|
|
85
|
+
search = '__version__ = "{current_version}"'
|
|
86
|
+
replace = '__version__ = "{new_version}"'
|
|
87
|
+
|
|
88
|
+
[tool.uv.sources]
|
|
89
|
+
# ctrl-freak = { git = "https://github.com/hydrosolutions/ctrl-freak.git", rev = "main" } # example of how to add a git dependency
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
2
|
+
|
|
3
|
+
from hcx.batch import Batch, BatchMetadata, GriddedDynamic, GriddedStatic
|
|
4
|
+
from hcx.conformance import ConformanceCheck, ConformanceError, assert_conforms, check_conformance
|
|
5
|
+
from hcx.output import Forecast
|
|
6
|
+
from hcx.protocol import MODEL_ENTRY_POINT_GROUP, ForecastModel, ModelFactory
|
|
7
|
+
from hcx.specifications import Gaussian, GaussianParameters, OutputSpecification, Point, PointParameters
|
|
8
|
+
from hcx.synthetic import make_synthetic_batch
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"MODEL_ENTRY_POINT_GROUP",
|
|
12
|
+
"Batch",
|
|
13
|
+
"BatchMetadata",
|
|
14
|
+
"ConformanceCheck",
|
|
15
|
+
"ConformanceError",
|
|
16
|
+
"Forecast",
|
|
17
|
+
"ForecastModel",
|
|
18
|
+
"Gaussian",
|
|
19
|
+
"GaussianParameters",
|
|
20
|
+
"GriddedDynamic",
|
|
21
|
+
"GriddedStatic",
|
|
22
|
+
"ModelFactory",
|
|
23
|
+
"OutputSpecification",
|
|
24
|
+
"Point",
|
|
25
|
+
"PointParameters",
|
|
26
|
+
"assert_conforms",
|
|
27
|
+
"check_conformance",
|
|
28
|
+
"make_synthetic_batch",
|
|
29
|
+
"__version__",
|
|
30
|
+
]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class GriddedDynamic:
|
|
9
|
+
values: torch.Tensor
|
|
10
|
+
coordinates: torch.Tensor
|
|
11
|
+
padding_mask: torch.Tensor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class GriddedStatic:
|
|
16
|
+
values: torch.Tensor
|
|
17
|
+
coordinates: torch.Tensor
|
|
18
|
+
padding_mask: torch.Tensor
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class BatchMetadata:
|
|
23
|
+
sample_ids: tuple[str, ...]
|
|
24
|
+
input_end_indices: np.ndarray
|
|
25
|
+
target_fill_mask: np.ndarray
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Batch:
|
|
30
|
+
scalar_dynamic: torch.Tensor | None
|
|
31
|
+
scalar_static: torch.Tensor | None
|
|
32
|
+
gridded_dynamic: dict[str, GriddedDynamic]
|
|
33
|
+
gridded_static: dict[str, GriddedStatic]
|
|
34
|
+
target: torch.Tensor
|
|
35
|
+
metadata: BatchMetadata
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
from contextlib import suppress
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
from hcx.batch import Batch
|
|
8
|
+
from hcx.output import Forecast
|
|
9
|
+
from hcx.protocol import ForecastModel
|
|
10
|
+
|
|
11
|
+
_SKIPPED = "skipped: forward did not return a Forecast"
|
|
12
|
+
_QUADRANTS = ("scalar_dynamic", "scalar_static", "gridded_dynamic", "gridded_static")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class ConformanceCheck:
|
|
17
|
+
name: str
|
|
18
|
+
passed: bool
|
|
19
|
+
detail: str
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ConformanceError(AssertionError):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _consumed_quadrants(model: torch.nn.Module, batch: Batch) -> ConformanceCheck:
|
|
27
|
+
declared = getattr(model, "consumed_quadrants", None)
|
|
28
|
+
if declared is None:
|
|
29
|
+
return ConformanceCheck("consumed_quadrants_present", True, "no consumed quadrants declared")
|
|
30
|
+
populated = {
|
|
31
|
+
"scalar_dynamic": batch.scalar_dynamic is not None,
|
|
32
|
+
"scalar_static": batch.scalar_static is not None,
|
|
33
|
+
"gridded_dynamic": bool(batch.gridded_dynamic),
|
|
34
|
+
"gridded_static": bool(batch.gridded_static),
|
|
35
|
+
}
|
|
36
|
+
try:
|
|
37
|
+
names = tuple(declared)
|
|
38
|
+
invalid = [name for name in names if name not in _QUADRANTS]
|
|
39
|
+
absent = [name for name in names if name in populated and not populated[name]]
|
|
40
|
+
except Exception as exc:
|
|
41
|
+
return ConformanceCheck("consumed_quadrants_present", False, f"declaration iteration failed: {exc!r}")
|
|
42
|
+
if invalid:
|
|
43
|
+
return ConformanceCheck("consumed_quadrants_present", False, f"invalid quadrants: {invalid!r}")
|
|
44
|
+
if absent:
|
|
45
|
+
return ConformanceCheck("consumed_quadrants_present", False, f"absent quadrants: {absent!r}")
|
|
46
|
+
return ConformanceCheck("consumed_quadrants_present", True, "all declared quadrants are populated")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _identity(name: str, got: object, original: object, snapshot: object) -> ConformanceCheck:
|
|
50
|
+
same_object = got is original
|
|
51
|
+
if isinstance(snapshot, tuple):
|
|
52
|
+
same_value = got == snapshot
|
|
53
|
+
elif isinstance(got, np.ndarray) and isinstance(snapshot, np.ndarray):
|
|
54
|
+
same_value = bool(np.array_equal(got, snapshot))
|
|
55
|
+
else:
|
|
56
|
+
same_value = False
|
|
57
|
+
passed = bool(same_object and same_value)
|
|
58
|
+
detail = "carried verbatim (original object, unchanged values)" if passed else "rebuilt or changed"
|
|
59
|
+
return ConformanceCheck(name, passed, detail)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _finiteness(out: Forecast) -> ConformanceCheck:
|
|
63
|
+
prediction = out.prediction
|
|
64
|
+
valid = isinstance(prediction, torch.Tensor) and bool(torch.isfinite(prediction).all())
|
|
65
|
+
detail = "prediction is finite"
|
|
66
|
+
if out.variance is not None:
|
|
67
|
+
variance = out.variance
|
|
68
|
+
valid_variance = (
|
|
69
|
+
isinstance(variance, torch.Tensor)
|
|
70
|
+
and isinstance(prediction, torch.Tensor)
|
|
71
|
+
and variance.shape == prediction.shape
|
|
72
|
+
and variance.dtype == prediction.dtype
|
|
73
|
+
and variance.device == prediction.device
|
|
74
|
+
and bool(torch.isfinite(variance).all())
|
|
75
|
+
and bool((variance > 0).all())
|
|
76
|
+
)
|
|
77
|
+
valid = valid and valid_variance
|
|
78
|
+
detail = "prediction finite; variance must match prediction and be finite and strictly positive"
|
|
79
|
+
return ConformanceCheck("finiteness", bool(valid), detail)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _trainability(model: torch.nn.Module, batch: Batch) -> ConformanceCheck:
|
|
83
|
+
try:
|
|
84
|
+
model.train()
|
|
85
|
+
model.zero_grad(set_to_none=True)
|
|
86
|
+
out = model.forward(batch)
|
|
87
|
+
if not isinstance(out, Forecast):
|
|
88
|
+
return ConformanceCheck("trainability_grad_flow", False, f"train forward returned {type(out).__name__}")
|
|
89
|
+
requires_grad = isinstance(out.prediction, torch.Tensor) and bool(out.prediction.requires_grad)
|
|
90
|
+
if requires_grad:
|
|
91
|
+
out.prediction.sum().backward()
|
|
92
|
+
gradient = any(
|
|
93
|
+
parameter.grad is not None
|
|
94
|
+
and bool(torch.isfinite(parameter.grad).all())
|
|
95
|
+
and float(parameter.grad.abs().sum()) > 0
|
|
96
|
+
for parameter in model.parameters()
|
|
97
|
+
)
|
|
98
|
+
return ConformanceCheck(
|
|
99
|
+
"trainability_grad_flow",
|
|
100
|
+
requires_grad and gradient,
|
|
101
|
+
f"requires_grad={requires_grad}, finite nonzero gradient={gradient}",
|
|
102
|
+
)
|
|
103
|
+
except Exception as exc:
|
|
104
|
+
return ConformanceCheck("trainability_grad_flow", False, f"train forward/backward raised {exc!r}")
|
|
105
|
+
finally:
|
|
106
|
+
with suppress(Exception):
|
|
107
|
+
model.zero_grad(set_to_none=True)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def check_conformance(model: torch.nn.Module, batch: Batch) -> tuple[list[ConformanceCheck], Forecast | None]:
|
|
111
|
+
was_training = model.training
|
|
112
|
+
metadata = batch.metadata
|
|
113
|
+
sample_ids = metadata.sample_ids
|
|
114
|
+
input_end_indices = metadata.input_end_indices
|
|
115
|
+
target_fill_mask = metadata.target_fill_mask
|
|
116
|
+
input_end_snapshot = input_end_indices.copy()
|
|
117
|
+
target_fill_snapshot = target_fill_mask.copy()
|
|
118
|
+
checks = [
|
|
119
|
+
ConformanceCheck("structural_protocol", isinstance(model, ForecastModel), "isinstance(model, ForecastModel)"),
|
|
120
|
+
_consumed_quadrants(model, batch),
|
|
121
|
+
]
|
|
122
|
+
output: Forecast | None = None
|
|
123
|
+
try:
|
|
124
|
+
model.eval()
|
|
125
|
+
try:
|
|
126
|
+
with torch.no_grad():
|
|
127
|
+
produced = model.forward(batch)
|
|
128
|
+
if isinstance(produced, Forecast):
|
|
129
|
+
output = produced
|
|
130
|
+
checks.append(ConformanceCheck("returns_forecast", True, "forward returned Forecast"))
|
|
131
|
+
else:
|
|
132
|
+
checks.append(
|
|
133
|
+
ConformanceCheck("returns_forecast", False, f"forward returned {type(produced).__name__}")
|
|
134
|
+
)
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
checks.append(ConformanceCheck("returns_forecast", False, f"forward raised {exc!r}"))
|
|
137
|
+
|
|
138
|
+
if output is None:
|
|
139
|
+
for name in (
|
|
140
|
+
"prediction_tensor_dtype_device",
|
|
141
|
+
"prediction_shape_exact",
|
|
142
|
+
"identity_sample_ids_verbatim",
|
|
143
|
+
"identity_input_end_indices_verbatim",
|
|
144
|
+
"identity_target_fill_mask_verbatim",
|
|
145
|
+
"finiteness",
|
|
146
|
+
):
|
|
147
|
+
checks.append(ConformanceCheck(name, False, _SKIPPED))
|
|
148
|
+
checks.append(ConformanceCheck("stable_across_two_eval_forwards", False, _SKIPPED))
|
|
149
|
+
else:
|
|
150
|
+
prediction = output.prediction
|
|
151
|
+
tensor_match = (
|
|
152
|
+
isinstance(prediction, torch.Tensor)
|
|
153
|
+
and prediction.dtype == batch.target.dtype
|
|
154
|
+
and prediction.device == batch.target.device
|
|
155
|
+
)
|
|
156
|
+
checks.append(
|
|
157
|
+
ConformanceCheck("prediction_tensor_dtype_device", tensor_match, "must match target dtype/device")
|
|
158
|
+
)
|
|
159
|
+
shape = tuple(prediction.shape) if isinstance(prediction, torch.Tensor) else None
|
|
160
|
+
expected = (batch.target.shape[0], batch.target.shape[-1])
|
|
161
|
+
checks.append(
|
|
162
|
+
ConformanceCheck("prediction_shape_exact", shape == expected, f"expected {expected}, got {shape}")
|
|
163
|
+
)
|
|
164
|
+
checks.append(_identity("identity_sample_ids_verbatim", output.sample_ids, sample_ids, sample_ids))
|
|
165
|
+
checks.append(
|
|
166
|
+
_identity(
|
|
167
|
+
"identity_input_end_indices_verbatim",
|
|
168
|
+
output.input_end_indices,
|
|
169
|
+
input_end_indices,
|
|
170
|
+
input_end_snapshot,
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
checks.append(
|
|
174
|
+
_identity(
|
|
175
|
+
"identity_target_fill_mask_verbatim",
|
|
176
|
+
output.target_fill_mask,
|
|
177
|
+
target_fill_mask,
|
|
178
|
+
target_fill_snapshot,
|
|
179
|
+
)
|
|
180
|
+
)
|
|
181
|
+
checks.append(_finiteness(output))
|
|
182
|
+
try:
|
|
183
|
+
model.eval()
|
|
184
|
+
with torch.no_grad():
|
|
185
|
+
again = model.forward(batch)
|
|
186
|
+
stable = (
|
|
187
|
+
isinstance(again, Forecast)
|
|
188
|
+
and torch.equal(output.prediction, again.prediction)
|
|
189
|
+
and (
|
|
190
|
+
(output.variance is None and again.variance is None)
|
|
191
|
+
or (
|
|
192
|
+
isinstance(output.variance, torch.Tensor)
|
|
193
|
+
and isinstance(again.variance, torch.Tensor)
|
|
194
|
+
and torch.equal(output.variance, again.variance)
|
|
195
|
+
)
|
|
196
|
+
)
|
|
197
|
+
)
|
|
198
|
+
detail = "adjacent eval forwards equal; cross-run/device determinism is documented, not enforced"
|
|
199
|
+
checks.append(ConformanceCheck("stable_across_two_eval_forwards", bool(stable), detail))
|
|
200
|
+
except Exception as exc:
|
|
201
|
+
checks.append(
|
|
202
|
+
ConformanceCheck("stable_across_two_eval_forwards", False, f"second eval forward raised {exc!r}")
|
|
203
|
+
)
|
|
204
|
+
checks.append(_trainability(model, batch))
|
|
205
|
+
return checks, output
|
|
206
|
+
finally:
|
|
207
|
+
model.train(was_training)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def assert_conforms(model: torch.nn.Module, batch: Batch) -> Forecast:
|
|
211
|
+
checks, forecast = check_conformance(model, batch)
|
|
212
|
+
for check in checks:
|
|
213
|
+
if not check.passed:
|
|
214
|
+
raise ConformanceError(f"{check.name}: {check.detail}")
|
|
215
|
+
if forecast is None:
|
|
216
|
+
raise ConformanceError("harness error: all checks passed without a forecast")
|
|
217
|
+
return forecast
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
__all__ = ["ConformanceCheck", "ConformanceError", "assert_conforms", "check_conformance"]
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""A scalar-input LSTM reference model."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from dataclasses import dataclass, fields
|
|
5
|
+
from numbers import Real
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import nn
|
|
9
|
+
|
|
10
|
+
from hcx.batch import Batch
|
|
11
|
+
from hcx.output import Forecast
|
|
12
|
+
from hcx.specifications import OutputSpecification
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class LSTMConfig:
|
|
17
|
+
hidden_size: int = 64
|
|
18
|
+
num_layers: int = 1
|
|
19
|
+
dropout: float = 0.0
|
|
20
|
+
decoder_hidden_size: int = 32
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _parse_config(values: dict[str, object]) -> LSTMConfig:
|
|
24
|
+
allowed = {field.name for field in fields(LSTMConfig)}
|
|
25
|
+
for key, value in values.items():
|
|
26
|
+
if key not in allowed:
|
|
27
|
+
raise ValueError(f"unknown model config key {key!r} with value {value!r}")
|
|
28
|
+
|
|
29
|
+
defaults = LSTMConfig()
|
|
30
|
+
for key in ("hidden_size", "num_layers", "decoder_hidden_size"):
|
|
31
|
+
value = values.get(key, getattr(defaults, key))
|
|
32
|
+
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
|
33
|
+
raise ValueError(f"{key} must be a positive integer; got {value!r}")
|
|
34
|
+
dropout_value = values.get("dropout", defaults.dropout)
|
|
35
|
+
if isinstance(dropout_value, bool) or not isinstance(dropout_value, Real):
|
|
36
|
+
raise ValueError(f"dropout must be a finite real number in [0, 1); got {dropout_value!r}")
|
|
37
|
+
dropout = float(dropout_value)
|
|
38
|
+
if not math.isfinite(dropout) or not 0 <= dropout < 1:
|
|
39
|
+
raise ValueError(f"dropout must be a finite real number in [0, 1); got {dropout_value!r}")
|
|
40
|
+
hidden_size = values.get("hidden_size", defaults.hidden_size)
|
|
41
|
+
num_layers = values.get("num_layers", defaults.num_layers)
|
|
42
|
+
decoder_hidden_size = values.get("decoder_hidden_size", defaults.decoder_hidden_size)
|
|
43
|
+
assert isinstance(hidden_size, int) and not isinstance(hidden_size, bool)
|
|
44
|
+
assert isinstance(num_layers, int) and not isinstance(num_layers, bool)
|
|
45
|
+
assert isinstance(decoder_hidden_size, int) and not isinstance(decoder_hidden_size, bool)
|
|
46
|
+
return LSTMConfig(
|
|
47
|
+
hidden_size=hidden_size,
|
|
48
|
+
num_layers=num_layers,
|
|
49
|
+
dropout=dropout,
|
|
50
|
+
decoder_hidden_size=decoder_hidden_size,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _validate_size(key: str, value: object, *, allow_zero: bool = False) -> None:
|
|
55
|
+
valid_range = value >= 0 if isinstance(value, int) else False
|
|
56
|
+
if isinstance(value, bool) or not isinstance(value, int) or not valid_range or (not allow_zero and value == 0):
|
|
57
|
+
requirement = "a nonnegative integer" if allow_zero else "a positive integer"
|
|
58
|
+
raise ValueError(f"{key} must be {requirement}; got {value!r}")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _validate_names(key: str, names: list[str], size: int) -> tuple[str, ...]:
|
|
62
|
+
if len(names) != size:
|
|
63
|
+
raise ValueError(f"{key} count must equal its supplied size {size}; got {len(names)}")
|
|
64
|
+
for name in names:
|
|
65
|
+
if not isinstance(name, str):
|
|
66
|
+
raise ValueError(f"{key} names must be strings; got {name!r}")
|
|
67
|
+
if len(set(names)) != len(names):
|
|
68
|
+
raise ValueError(f"{key} names must be unique; got {names!r}")
|
|
69
|
+
return tuple(names)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ScalarLSTM(nn.Module):
|
|
73
|
+
def __init__(
|
|
74
|
+
self,
|
|
75
|
+
config: LSTMConfig,
|
|
76
|
+
input_size: int,
|
|
77
|
+
static_size: int,
|
|
78
|
+
output_size: int,
|
|
79
|
+
dynamic_inputs: tuple[str, ...],
|
|
80
|
+
static_inputs: tuple[str, ...],
|
|
81
|
+
output_specification: OutputSpecification[object],
|
|
82
|
+
) -> None:
|
|
83
|
+
super().__init__()
|
|
84
|
+
self.config = config
|
|
85
|
+
self.input_size = input_size
|
|
86
|
+
self.static_size = static_size
|
|
87
|
+
self.output_size = output_size
|
|
88
|
+
self.dynamic_inputs = dynamic_inputs
|
|
89
|
+
self.static_inputs = static_inputs
|
|
90
|
+
self.output_specification = output_specification
|
|
91
|
+
self.consumed_quadrants = ("scalar_dynamic",) if static_size == 0 else ("scalar_dynamic", "scalar_static")
|
|
92
|
+
|
|
93
|
+
self.embedding = nn.Linear(input_size + static_size, config.hidden_size)
|
|
94
|
+
self.lstm = nn.LSTM(
|
|
95
|
+
config.hidden_size,
|
|
96
|
+
config.hidden_size,
|
|
97
|
+
config.num_layers,
|
|
98
|
+
dropout=config.dropout if config.num_layers > 1 else 0,
|
|
99
|
+
batch_first=True,
|
|
100
|
+
)
|
|
101
|
+
self.decoder = nn.Sequential(
|
|
102
|
+
nn.Linear(config.hidden_size, config.decoder_hidden_size),
|
|
103
|
+
nn.ReLU(),
|
|
104
|
+
nn.Dropout(config.dropout),
|
|
105
|
+
nn.Linear(config.decoder_hidden_size, output_size * output_specification.raw_head_width),
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def forward(self, batch: Batch) -> Forecast:
|
|
109
|
+
x = batch.scalar_dynamic
|
|
110
|
+
if x is None:
|
|
111
|
+
raise ValueError("scalar dynamic input is required")
|
|
112
|
+
if x.shape[-1] != self.input_size:
|
|
113
|
+
raise ValueError(f"scalar dynamic width must be {self.input_size}; got {x.shape[-1]}")
|
|
114
|
+
batch_size, input_length, _ = x.shape
|
|
115
|
+
|
|
116
|
+
if self.static_size > 0:
|
|
117
|
+
static = batch.scalar_static
|
|
118
|
+
if static is None:
|
|
119
|
+
raise ValueError(f"scalar static input of width {self.static_size} is required")
|
|
120
|
+
if static.shape[-1] != self.static_size:
|
|
121
|
+
raise ValueError(f"scalar static width must be {self.static_size}; got {static.shape[-1]}")
|
|
122
|
+
x = torch.cat([x, static.unsqueeze(1).expand(-1, input_length, -1)], dim=-1)
|
|
123
|
+
elif batch.scalar_static is not None:
|
|
124
|
+
raise ValueError("scalar static input was supplied when static_size is zero")
|
|
125
|
+
|
|
126
|
+
embedded = self.embedding(x)
|
|
127
|
+
recurrent, _ = self.lstm(embedded)
|
|
128
|
+
decoded = self.decoder(recurrent[:, -1, :])
|
|
129
|
+
raw = decoded.view(batch_size, self.output_size, self.output_specification.raw_head_width)
|
|
130
|
+
parameters = self.output_specification.parameterize(raw)
|
|
131
|
+
return self.output_specification.populate_forecast(
|
|
132
|
+
parameters,
|
|
133
|
+
sample_ids=batch.metadata.sample_ids,
|
|
134
|
+
input_end_indices=batch.metadata.input_end_indices,
|
|
135
|
+
target_fill_mask=batch.metadata.target_fill_mask,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def factory(
|
|
140
|
+
model_config: dict[str, object],
|
|
141
|
+
*,
|
|
142
|
+
dynamic_inputs: list[str],
|
|
143
|
+
static_inputs: list[str],
|
|
144
|
+
input_size: int,
|
|
145
|
+
static_size: int,
|
|
146
|
+
output_size: int,
|
|
147
|
+
output_specification: OutputSpecification[object],
|
|
148
|
+
) -> torch.nn.Module:
|
|
149
|
+
_validate_size("input_size", input_size)
|
|
150
|
+
_validate_size("static_size", static_size, allow_zero=True)
|
|
151
|
+
_validate_size("output_size", output_size)
|
|
152
|
+
dynamic_names = _validate_names("dynamic_inputs", dynamic_inputs, input_size)
|
|
153
|
+
static_names = _validate_names("static_inputs", static_inputs, static_size)
|
|
154
|
+
return ScalarLSTM(
|
|
155
|
+
_parse_config(model_config),
|
|
156
|
+
input_size,
|
|
157
|
+
static_size,
|
|
158
|
+
output_size,
|
|
159
|
+
dynamic_names,
|
|
160
|
+
static_names,
|
|
161
|
+
output_specification,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
__all__ = ["LSTMConfig", "ScalarLSTM", "factory"]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
import torch
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class Forecast:
|
|
9
|
+
"""A forecast whose values are in target space."""
|
|
10
|
+
|
|
11
|
+
prediction: torch.Tensor
|
|
12
|
+
sample_ids: tuple[str, ...]
|
|
13
|
+
input_end_indices: np.ndarray
|
|
14
|
+
target_fill_mask: np.ndarray
|
|
15
|
+
variance: torch.Tensor | None = None
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from typing import Protocol, runtime_checkable
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
from hcx.batch import Batch
|
|
6
|
+
from hcx.output import Forecast
|
|
7
|
+
from hcx.specifications import OutputSpecification
|
|
8
|
+
|
|
9
|
+
MODEL_ENTRY_POINT_GROUP = "hcx.models"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@runtime_checkable
|
|
13
|
+
class ForecastModel(Protocol):
|
|
14
|
+
def forward(self, batch: Batch) -> Forecast: ...
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@runtime_checkable
|
|
18
|
+
class ModelFactory(Protocol):
|
|
19
|
+
def __call__(
|
|
20
|
+
self,
|
|
21
|
+
model_config: dict[str, object],
|
|
22
|
+
*,
|
|
23
|
+
dynamic_inputs: list[str],
|
|
24
|
+
static_inputs: list[str],
|
|
25
|
+
input_size: int,
|
|
26
|
+
static_size: int,
|
|
27
|
+
output_size: int,
|
|
28
|
+
output_specification: OutputSpecification[object],
|
|
29
|
+
) -> torch.nn.Module: ...
|
|
File without changes
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import ClassVar, Protocol, TypeVar, runtime_checkable
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import torch
|
|
6
|
+
import torch.nn.functional as functional
|
|
7
|
+
|
|
8
|
+
from hcx.output import Forecast
|
|
9
|
+
|
|
10
|
+
ParametersT = TypeVar("ParametersT")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@runtime_checkable
|
|
14
|
+
class OutputSpecification(Protocol[ParametersT]):
|
|
15
|
+
raw_head_width: int
|
|
16
|
+
|
|
17
|
+
def parameterize(self, raw: torch.Tensor) -> ParametersT: ...
|
|
18
|
+
|
|
19
|
+
def populate_forecast(
|
|
20
|
+
self,
|
|
21
|
+
parameters: ParametersT,
|
|
22
|
+
*,
|
|
23
|
+
sample_ids: tuple[str, ...],
|
|
24
|
+
input_end_indices: np.ndarray,
|
|
25
|
+
target_fill_mask: np.ndarray,
|
|
26
|
+
) -> Forecast: ...
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _validate_raw_head(raw: torch.Tensor, expected_width: int) -> None:
|
|
30
|
+
expected = f"[B, T_out, {expected_width}]"
|
|
31
|
+
actual = tuple(raw.shape)
|
|
32
|
+
if raw.ndim != 3 or raw.shape[-1] != expected_width:
|
|
33
|
+
raise ValueError(f"expected shape {expected}; actual shape {actual}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class PointParameters:
|
|
38
|
+
prediction: torch.Tensor
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class GaussianParameters:
|
|
43
|
+
prediction: torch.Tensor
|
|
44
|
+
variance: torch.Tensor
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class Point:
|
|
49
|
+
raw_head_width: ClassVar[int] = 1
|
|
50
|
+
|
|
51
|
+
def parameterize(self, raw: torch.Tensor) -> PointParameters:
|
|
52
|
+
_validate_raw_head(raw, self.raw_head_width)
|
|
53
|
+
return PointParameters(prediction=raw[..., 0])
|
|
54
|
+
|
|
55
|
+
def populate_forecast(
|
|
56
|
+
self,
|
|
57
|
+
parameters: PointParameters,
|
|
58
|
+
*,
|
|
59
|
+
sample_ids: tuple[str, ...],
|
|
60
|
+
input_end_indices: np.ndarray,
|
|
61
|
+
target_fill_mask: np.ndarray,
|
|
62
|
+
) -> Forecast:
|
|
63
|
+
return Forecast(parameters.prediction, sample_ids, input_end_indices, target_fill_mask)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Gaussian:
|
|
68
|
+
raw_head_width: ClassVar[int] = 2
|
|
69
|
+
|
|
70
|
+
def parameterize(self, raw: torch.Tensor) -> GaussianParameters:
|
|
71
|
+
_validate_raw_head(raw, self.raw_head_width)
|
|
72
|
+
raw_variance = raw[..., 1]
|
|
73
|
+
return GaussianParameters(
|
|
74
|
+
prediction=raw[..., 0],
|
|
75
|
+
variance=functional.softplus(raw_variance) + torch.finfo(raw_variance.dtype).tiny,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def populate_forecast(
|
|
79
|
+
self,
|
|
80
|
+
parameters: GaussianParameters,
|
|
81
|
+
*,
|
|
82
|
+
sample_ids: tuple[str, ...],
|
|
83
|
+
input_end_indices: np.ndarray,
|
|
84
|
+
target_fill_mask: np.ndarray,
|
|
85
|
+
) -> Forecast:
|
|
86
|
+
return Forecast(
|
|
87
|
+
parameters.prediction,
|
|
88
|
+
sample_ids,
|
|
89
|
+
input_end_indices,
|
|
90
|
+
target_fill_mask,
|
|
91
|
+
parameters.variance,
|
|
92
|
+
)
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import torch
|
|
3
|
+
|
|
4
|
+
from hcx.batch import Batch, BatchMetadata, GriddedDynamic, GriddedStatic
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def make_synthetic_batch(
|
|
8
|
+
*,
|
|
9
|
+
batch_size: int = 4,
|
|
10
|
+
input_length: int = 6,
|
|
11
|
+
output_length: int = 2,
|
|
12
|
+
scalar_dynamic_features: int = 3,
|
|
13
|
+
scalar_static_features: int = 2,
|
|
14
|
+
grid_cells: int = 5,
|
|
15
|
+
gridded_dynamic_features: int = 2,
|
|
16
|
+
gridded_static_features: int = 3,
|
|
17
|
+
dtype: torch.dtype = torch.float32,
|
|
18
|
+
device: torch.device | str = "cpu",
|
|
19
|
+
seed: int = 1729,
|
|
20
|
+
include_scalar_dynamic: bool = True,
|
|
21
|
+
include_scalar_static: bool = True,
|
|
22
|
+
include_gridded_dynamic: bool = True,
|
|
23
|
+
include_gridded_static: bool = True,
|
|
24
|
+
) -> Batch:
|
|
25
|
+
dimensions = {
|
|
26
|
+
"batch_size": batch_size,
|
|
27
|
+
"input_length": input_length,
|
|
28
|
+
"output_length": output_length,
|
|
29
|
+
"grid_cells": grid_cells,
|
|
30
|
+
}
|
|
31
|
+
if invalid := [name for name, value in dimensions.items() if value <= 0]:
|
|
32
|
+
raise ValueError(f"dimensions must be positive: {invalid!r}")
|
|
33
|
+
counts = {
|
|
34
|
+
"scalar_dynamic": scalar_dynamic_features,
|
|
35
|
+
"scalar_static": scalar_static_features,
|
|
36
|
+
"gridded_dynamic": gridded_dynamic_features,
|
|
37
|
+
"gridded_static": gridded_static_features,
|
|
38
|
+
}
|
|
39
|
+
if invalid := [name for name, value in counts.items() if value < 0]:
|
|
40
|
+
raise ValueError(f"feature counts must be nonnegative: {invalid!r}")
|
|
41
|
+
included = {
|
|
42
|
+
"scalar_dynamic": include_scalar_dynamic,
|
|
43
|
+
"scalar_static": include_scalar_static,
|
|
44
|
+
"gridded_dynamic": include_gridded_dynamic,
|
|
45
|
+
"gridded_static": include_gridded_static,
|
|
46
|
+
}
|
|
47
|
+
if invalid := [name for name, enabled in included.items() if enabled and counts[name] == 0]:
|
|
48
|
+
raise ValueError(f"included quadrants must have features: {invalid!r}")
|
|
49
|
+
|
|
50
|
+
device = torch.device(device)
|
|
51
|
+
generator = torch.Generator(device=device)
|
|
52
|
+
generator.manual_seed(seed)
|
|
53
|
+
|
|
54
|
+
def randn(shape: tuple[int, ...]) -> torch.Tensor:
|
|
55
|
+
return torch.randn(shape, dtype=dtype, device=device, generator=generator)
|
|
56
|
+
|
|
57
|
+
def rand(shape: tuple[int, ...]) -> torch.Tensor:
|
|
58
|
+
return torch.rand(shape, dtype=dtype, device=device, generator=generator)
|
|
59
|
+
|
|
60
|
+
scalar_dynamic = randn((batch_size, input_length, scalar_dynamic_features)) if include_scalar_dynamic else None
|
|
61
|
+
scalar_static = randn((batch_size, scalar_static_features)) if include_scalar_static else None
|
|
62
|
+
|
|
63
|
+
gridded_dynamic: dict[str, GriddedDynamic] = {}
|
|
64
|
+
coordinates: torch.Tensor | None = None
|
|
65
|
+
padding_mask: torch.Tensor | None = None
|
|
66
|
+
if include_gridded_dynamic:
|
|
67
|
+
values = randn((batch_size, input_length, grid_cells, gridded_dynamic_features))
|
|
68
|
+
coordinates = rand((batch_size, grid_cells, 2))
|
|
69
|
+
padding_mask = torch.zeros((batch_size, grid_cells), dtype=torch.bool, device=device)
|
|
70
|
+
gridded_dynamic["meteorology"] = GriddedDynamic(values, coordinates, padding_mask)
|
|
71
|
+
|
|
72
|
+
gridded_static: dict[str, GriddedStatic] = {}
|
|
73
|
+
if include_gridded_static:
|
|
74
|
+
values = randn((batch_size, grid_cells, gridded_static_features))
|
|
75
|
+
if coordinates is None:
|
|
76
|
+
coordinates = rand((batch_size, grid_cells, 2))
|
|
77
|
+
padding_mask = torch.zeros((batch_size, grid_cells), dtype=torch.bool, device=device)
|
|
78
|
+
assert padding_mask is not None
|
|
79
|
+
gridded_static["physiography"] = GriddedStatic(values, coordinates, padding_mask)
|
|
80
|
+
|
|
81
|
+
target = randn((batch_size, output_length))
|
|
82
|
+
metadata = BatchMetadata(
|
|
83
|
+
sample_ids=tuple(f"sample-{index:03d}" for index in range(batch_size)),
|
|
84
|
+
input_end_indices=np.full((batch_size,), input_length - 1, dtype=np.int64),
|
|
85
|
+
target_fill_mask=np.zeros((batch_size, output_length), dtype=np.int8),
|
|
86
|
+
)
|
|
87
|
+
return Batch(scalar_dynamic, scalar_static, gridded_dynamic, gridded_static, target, metadata)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
__all__ = ["make_synthetic_batch"]
|