canvod-store 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- canvod_store-0.2.0/PKG-INFO +72 -0
- canvod_store-0.2.0/README.md +48 -0
- canvod_store-0.2.0/pyproject.toml +72 -0
- canvod_store-0.2.0/src/canvod/store/__init__.py +23 -0
- canvod_store-0.2.0/src/canvod/store/file_registry.py +35 -0
- canvod_store-0.2.0/src/canvod/store/grid_adapters/__init__.py +9 -0
- canvod_store-0.2.0/src/canvod/store/grid_adapters/complete_grid_vod_workflow.py +1021 -0
- canvod_store-0.2.0/src/canvod/store/grid_adapters/grid_storage.py +2109 -0
- canvod_store-0.2.0/src/canvod/store/grid_adapters/hemigrid_polars_storage_methods.py +30 -0
- canvod_store-0.2.0/src/canvod/store/manager.py +884 -0
- canvod_store-0.2.0/src/canvod/store/preprocessing.py +193 -0
- canvod_store-0.2.0/src/canvod/store/reader.py +743 -0
- canvod_store-0.2.0/src/canvod/store/store.py +3350 -0
- canvod_store-0.2.0/src/canvod/store/viewer.py +1061 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: canvod-store
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Icechunk storage for GNSS VOD data
|
|
5
|
+
Keywords: GNSS,VOD,storage,icechunk,zarr
|
|
6
|
+
Author: Nicolas François Bader
|
|
7
|
+
Author-email: Nicolas François Bader <nicolas.bader@geo.tuwien.ac.at>
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
13
|
+
Requires-Dist: canvod-grids>=0.1.0
|
|
14
|
+
Requires-Dist: icechunk>=1.1.5
|
|
15
|
+
Requires-Dist: zarr>=3.1.2
|
|
16
|
+
Requires-Dist: xarray>=2023.12.0
|
|
17
|
+
Requires-Dist: polars>=1.0.0
|
|
18
|
+
Requires-Dist: numpy>=1.24.0
|
|
19
|
+
Requires-Dist: networkx>=3.6.1
|
|
20
|
+
Maintainer: Climate and Environmental Remote Sensing (CLIMERS)
|
|
21
|
+
Maintainer-email: Climate and Environmental Remote Sensing (CLIMERS) <nicolas.bader@geo.tuwien.ac.at>
|
|
22
|
+
Requires-Python: >=3.14
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# canvod-store
|
|
26
|
+
|
|
27
|
+
Icechunk storage for GNSS VOD data.
|
|
28
|
+
|
|
29
|
+
Part of the [canVODpy](https://github.com/nfb2021/canvodpy) ecosystem.
|
|
30
|
+
|
|
31
|
+
## Overview
|
|
32
|
+
|
|
33
|
+
This package provides versioned storage for GNSS data using Icechunk, managing:
|
|
34
|
+
|
|
35
|
+
- **RINEX Store (Level 1)**: Augmented observations per receiver
|
|
36
|
+
- **VOD Store (Level 2)**: Analysis products comparing receiver pairs
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install canvod-store
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Quick Start
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from canvod.store import create_rinex_store, GnssResearchSite
|
|
48
|
+
from pathlib import Path
|
|
49
|
+
|
|
50
|
+
# Create stores
|
|
51
|
+
rinex_store = create_rinex_store(Path("./rinex_store"))
|
|
52
|
+
|
|
53
|
+
# Or use site manager
|
|
54
|
+
site = GnssResearchSite(site_name="Rosalia")
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Features
|
|
58
|
+
|
|
59
|
+
- Automatic repository creation/connection
|
|
60
|
+
- Group management with validation
|
|
61
|
+
- Session management with context managers
|
|
62
|
+
- Integrated logging and metadata tracking
|
|
63
|
+
- Configurable compression and chunking
|
|
64
|
+
- Deduplication support
|
|
65
|
+
|
|
66
|
+
## Documentation
|
|
67
|
+
|
|
68
|
+
[Centralized documentation](../../docs/packages/store/overview.md)
|
|
69
|
+
|
|
70
|
+
## License
|
|
71
|
+
|
|
72
|
+
Apache License 2.0 - see LICENSE file
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# canvod-store
|
|
2
|
+
|
|
3
|
+
Icechunk storage for GNSS VOD data.
|
|
4
|
+
|
|
5
|
+
Part of the [canVODpy](https://github.com/nfb2021/canvodpy) ecosystem.
|
|
6
|
+
|
|
7
|
+
## Overview
|
|
8
|
+
|
|
9
|
+
This package provides versioned storage for GNSS data using Icechunk, managing:
|
|
10
|
+
|
|
11
|
+
- **RINEX Store (Level 1)**: Augmented observations per receiver
|
|
12
|
+
- **VOD Store (Level 2)**: Analysis products comparing receiver pairs
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install canvod-store
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick Start
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from canvod.store import create_rinex_store, GnssResearchSite
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
# Create stores
|
|
27
|
+
rinex_store = create_rinex_store(Path("./rinex_store"))
|
|
28
|
+
|
|
29
|
+
# Or use site manager
|
|
30
|
+
site = GnssResearchSite(site_name="Rosalia")
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Features
|
|
34
|
+
|
|
35
|
+
- Automatic repository creation/connection
|
|
36
|
+
- Group management with validation
|
|
37
|
+
- Session management with context managers
|
|
38
|
+
- Integrated logging and metadata tracking
|
|
39
|
+
- Configurable compression and chunking
|
|
40
|
+
- Deduplication support
|
|
41
|
+
|
|
42
|
+
## Documentation
|
|
43
|
+
|
|
44
|
+
[Centralized documentation](../../docs/packages/store/overview.md)
|
|
45
|
+
|
|
46
|
+
## License
|
|
47
|
+
|
|
48
|
+
Apache License 2.0 - see LICENSE file
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "canvod-store"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Icechunk storage for GNSS VOD data"
|
|
5
|
+
authors = [
|
|
6
|
+
{name = "Nicolas François Bader", email = "nicolas.bader@geo.tuwien.ac.at"},
|
|
7
|
+
]
|
|
8
|
+
maintainers = [
|
|
9
|
+
{name = "Climate and Environmental Remote Sensing (CLIMERS)", email = "nicolas.bader@geo.tuwien.ac.at"},
|
|
10
|
+
]
|
|
11
|
+
readme = "README.md"
|
|
12
|
+
requires-python = ">=3.14"
|
|
13
|
+
keywords = ["GNSS", "VOD", "storage", "icechunk", "zarr"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"License :: OSI Approved :: Apache Software License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.14",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
dependencies = [
|
|
23
|
+
"canvod-grids>=0.1.0",
|
|
24
|
+
"icechunk>=1.1.5",
|
|
25
|
+
"zarr>=3.1.2",
|
|
26
|
+
"xarray>=2023.12.0",
|
|
27
|
+
"polars>=1.0.0",
|
|
28
|
+
"numpy>=1.24.0",
|
|
29
|
+
"networkx>=3.6.1",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[dependency-groups]
|
|
33
|
+
dev = [
|
|
34
|
+
"pytest>=8.0",
|
|
35
|
+
"pytest-cov>=5.0",
|
|
36
|
+
"ruff>=0.14",
|
|
37
|
+
"ty>=0.0.9",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[tool.uv.sources]
|
|
41
|
+
canvod-grids = { workspace = true }
|
|
42
|
+
|
|
43
|
+
[build-system]
|
|
44
|
+
requires = ["uv_build>=0.9.17,<0.10.0"]
|
|
45
|
+
build-backend = "uv_build"
|
|
46
|
+
|
|
47
|
+
[tool.uv.build-backend]
|
|
48
|
+
module-name = "canvod.store"
|
|
49
|
+
|
|
50
|
+
[tool.ruff]
|
|
51
|
+
line-length = 88
|
|
52
|
+
target-version = "py314"
|
|
53
|
+
|
|
54
|
+
[tool.ruff.lint]
|
|
55
|
+
select = ["E", "F", "I", "N", "W", "UP"]
|
|
56
|
+
ignore = []
|
|
57
|
+
|
|
58
|
+
[tool.pytest.ini_options]
|
|
59
|
+
testpaths = ["tests"]
|
|
60
|
+
python_files = ["test_*.py"]
|
|
61
|
+
python_classes = ["Test*"]
|
|
62
|
+
python_functions = ["test_*"]
|
|
63
|
+
addopts = [
|
|
64
|
+
"-v",
|
|
65
|
+
"--strict-markers",
|
|
66
|
+
"--tb=short",
|
|
67
|
+
"--disable-warnings",
|
|
68
|
+
]
|
|
69
|
+
markers = [
|
|
70
|
+
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
|
|
71
|
+
"integration: marks tests as integration tests",
|
|
72
|
+
]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""canvod-store: Icechunk storage for GNSS VOD data.
|
|
2
|
+
|
|
3
|
+
This package provides versioned storage for GNSS VOD data using Icechunk.
|
|
4
|
+
Manages both RINEX observation storage and VOD analysis results.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from canvod.store.manager import GnssResearchSite
|
|
8
|
+
from canvod.store.reader import IcechunkDataReader
|
|
9
|
+
from canvod.store.store import (
|
|
10
|
+
MyIcechunkStore,
|
|
11
|
+
create_rinex_store,
|
|
12
|
+
create_vod_store,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"GnssResearchSite",
|
|
19
|
+
"IcechunkDataReader",
|
|
20
|
+
"MyIcechunkStore",
|
|
21
|
+
"create_rinex_store",
|
|
22
|
+
"create_vod_store",
|
|
23
|
+
]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""File registry (ingest ledger) helpers for Icechunk-backed stores.
|
|
2
|
+
|
|
3
|
+
This module manages the per-file tracking table stored at
|
|
4
|
+
``{group}/metadata/table`` inside an Icechunk store. Each row records
|
|
5
|
+
a single ingested file: its hash, temporal range, filename, path, etc.
|
|
6
|
+
|
|
7
|
+
**Not to be confused with** ``canvod.store_metadata`` (the separate package),
|
|
8
|
+
which manages store-level provenance: identity, creator, environment,
|
|
9
|
+
software versions, STAC/DataCite/ACDD compliance, etc.
|
|
10
|
+
|
|
11
|
+
Terminology
|
|
12
|
+
-----------
|
|
13
|
+
- **File registry** (this module): *which files* went into a store group.
|
|
14
|
+
- **Store metadata** (``canvod.store_metadata``): *who/what/when/how* about
|
|
15
|
+
the store as a whole.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from canvodpy.logging import get_logger
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class FileRegistryManager:
|
|
24
|
+
"""Manage file registry table CRUD, backups, and deduplication for groups.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
logger : Any, optional
|
|
29
|
+
Logger-like object to use. Defaults to the configured context logger.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, logger: Any | None = None) -> None:
|
|
33
|
+
self._logger = logger or get_logger(__name__)
|
|
34
|
+
|
|
35
|
+
# TODO: Future refactor — extract metadata table methods from MyIcechunkStore.
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Grid adapters for hemisphere grid storage in Icechunk."""
|
|
2
|
+
|
|
3
|
+
# These will be enabled when canvod-grids is migrated
|
|
4
|
+
# from canvod.store.grid_adapters.complete_grid_vod_workflow import (
|
|
5
|
+
# store_grid_to_vod_store,
|
|
6
|
+
# load_grid_from_vod_store,
|
|
7
|
+
# )
|
|
8
|
+
|
|
9
|
+
__all__ = []
|