collekt 0.2.0__tar.gz

collekt-0.2.0/LICENSE

@@ -0,0 +1,28 @@
+3-Clause BSD License / New BSD License
+
+Copyright (c) 2023-2025 Simula Research Laboratory, Oslo, Norway
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

collekt-0.2.0/PKG-INFO

@@ -0,0 +1,175 @@
+Metadata-Version: 2.4
+Name: collekt
+Version: 0.2.0
+Summary: A library to facilitate spatio-temporal data collection across arbitrary datasources
+Keywords: data source,collection
+Author: Thomas Roehr, Ola Formo Kihle, Julien Vitay
+Author-email: Thomas Roehr <roehr@simula.no>, Ola Formo Kihle <kihle@simula.no>, Julien Vitay <julien.vitay@eodyn.com>
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Development Status :: 3 - Alpha
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: pydantic
+Requires-Dist: pydantic-settings
+Requires-Dist: pyyaml
+Requires-Dist: requests
+Requires-Dist: shapely
+Requires-Dist: geopy
+Requires-Dist: rich
+Requires-Dist: tqdm
+Requires-Dist: xarray>=2026.4.0
+Requires-Dist: scipy>=1.18.0
+Requires-Dist: cfgrib>=0.9.15.1
+Requires-Dist: eccodeslib>=2.47.3.21
+Requires-Dist: damast
+Requires-Dist: geopandas
+Requires-Dist: polars
+Requires-Dist: copernicusmarine>=2.4.1
+Requires-Dist: cdsapi>=0.7.7
+Requires-Dist: ecmwf-opendata>=0.3.30
+Requires-Dist: hozint-apiclient
+Requires-Dist: python-dotenv>=1.2.2
+Maintainer: Thomas Roehr
+Maintainer-email: Thomas Roehr <roehr@simula.no>
+Requires-Python: >=3.13
+Project-URL: homepage, https://ai4copsec.github.io/collekt
+Project-URL: documentation, https://ai4copsec.github.io/collekt
+Project-URL: repository, https://github.com/ai4copsec/collekt
+Description-Content-Type: text/markdown
+
+# collekt
+
+A library to facilitate spatio-temporal data collection across arbitrary
+datasources.
+
+A dataset config names the products to fetch, and a request supplies the region,
+time window, and metadata. The selected products are fanned out to
+pluggable source adapters.
+Each adapter fetches source-native files (NetCDF, GRIB2, Parquet, product
+archives) and records what it did in a machine-readable manifest. Gridded results
+can then be assembled onto a common grid and time axis.
+
+## Installation
+
+Requires Python 3.13+ and [`uv`](https://docs.astral.sh/uv/).
+
+```bash
+git clone https://github.com/ai4copsec/collekt.git
+cd collekt
+uv sync --all-groups  # or: just install
+```
+
+All provider clients and the gridded `Assembler` are installed by default, so
+every source works out of the box. Clients are imported lazily, so importing
+collekt stays fast.
+
+To build the documentation locally, you will also need the `quarto` binary ([quarto.org](https://quarto.org/)):
+
+```bash
+just docs
+# or :
+uv run quartodoc build --config docs/_quarto.yml
+quarto render docs
+```
+
+We recommend using `just` (<https://github.com/casey/just>) with the make-like commands available in `justfile`.
+
+## Usage
+
+### Python
+
+```python
+import collekt
+
+config = collekt.DatasetConfig(
+    collekt.CMEMS("cmems_glorys_my", variables=["uo", "vo"], depth=[1.0, 1.1]),
+    collekt.CMEMS("cmems_duacs_my", variables=["ugos", "vgos"]),
+    collekt.CMEMS("cmems_med_currents_my", variables=["uo", "vo"], depth=[1.0, 1.1]),
+)
+request = collekt.Request(
+    region=collekt.Region.from_bbox((-6.0, 20.0, 35.0, 45.0)),
+    start="2023-06-15",
+)
+
+fetcher = collekt.Fetcher(request=request, config=config, output_dir="data/collections")
+result = fetcher.download()
+print(result.manifest_path)
+
+# Assemble gridded sources onto a common grid and time axis.
+assembler = collekt.Assembler(result)
+dataset = assembler.to_xarray(grid="lowest_resolution", time="lowest_resolution")
+```
+
+`Region` can also be built from a point and radius
+(`Region.from_point_radius(lat, lon, radius_km)`) or a GeoJSON file
+(`Region.from_geojson(path)`).
+
+### Command line
+
+```bash
+collekt config show       # inspect the bundled catalog
+collekt doctor            # check the environment and configuration
+
+cat > datasets.yaml <<'YAML'
+datasets:
+  - provider: cmems
+    key: cmems_glorys_my
+    variables: [uo, vo]
+    depth: [1.0, 1.1]
+  - provider: cmems
+    key: cmems_duacs_my
+    variables: [ugos, vgos]
+  - provider: cmems
+    key: cmems_med_currents_my
+    variables: [uo, vo]
+    depth: [1.0, 1.1]
+YAML
+
+collekt fetch --bbox -6 20 35 45 --start 2023-06-15 \
+  --dataset-config datasets.yaml --output-dir data/collections
+```
+
+Use `--dry-run` to plan provider requests without downloading, and `--strict` to
+fail if any requested source is skipped.
+
+### Sources and configuration
+
+collekt ships a **curated catalog** of datasets (CMEMS global + Mediterranean,
+ECMWF Open Data, ERA5, Skytruth, Copernicus Data Space, eOdyn). Select concrete
+dataset keys with `DatasetConfig` or its YAML form, and put provider parameters
+such as variable names and depth ranges directly on each selected dataset. See
+[Products](https://ai4copsec.github.io/collekt/products.html) and
+[Credentials](https://ai4copsec.github.io/collekt/credentials.html).
+
+## Development
+
+```bash
+just lint    # format + lint with ruff
+just test    # run the test suite
+just docs    # build the Quarto documentation
+```
+
+The definition of done for any change: `just lint` and `just test` pass, with a
+test for any behavior changed.
+
+## Documentation
+
+Built with [Quarto](https://quarto.org) and
+[quartodoc](https://machow.github.io/quartodoc/) from `docs/`; the rendered site
+is published to GitHub Pages.
+
+## License
+
+[BSD-3-Clause](LICENSE).
+
+## Copyright
+
+Copyright (c) 2023-2026 [Simula Research Laboratory, Oslo, Norway](https://www.simula.no).
+
+## Acknowledgments
+
+Part of the EU project [AI4COPSEC](https://ai4copsec.eu), funded by the Horizon
+Europe framework programme under Grant Agreement N. 101190021.

collekt-0.2.0/README.md

@@ -0,0 +1,133 @@
+# collekt
+
+A library to facilitate spatio-temporal data collection across arbitrary
+datasources.
+
+A dataset config names the products to fetch, and a request supplies the region,
+time window, and metadata. The selected products are fanned out to
+pluggable source adapters.
+Each adapter fetches source-native files (NetCDF, GRIB2, Parquet, product
+archives) and records what it did in a machine-readable manifest. Gridded results
+can then be assembled onto a common grid and time axis.
+
+## Installation
+
+Requires Python 3.13+ and [`uv`](https://docs.astral.sh/uv/).
+
+```bash
+git clone https://github.com/ai4copsec/collekt.git
+cd collekt
+uv sync --all-groups  # or: just install
+```
+
+All provider clients and the gridded `Assembler` are installed by default, so
+every source works out of the box. Clients are imported lazily, so importing
+collekt stays fast.
+
+To build the documentation locally, you will also need the `quarto` binary ([quarto.org](https://quarto.org/)):
+
+```bash
+just docs
+# or :
+uv run quartodoc build --config docs/_quarto.yml
+quarto render docs
+```
+
+We recommend using `just` (<https://github.com/casey/just>) with the make-like commands available in `justfile`.
+
+## Usage
+
+### Python
+
+```python
+import collekt
+
+config = collekt.DatasetConfig(
+    collekt.CMEMS("cmems_glorys_my", variables=["uo", "vo"], depth=[1.0, 1.1]),
+    collekt.CMEMS("cmems_duacs_my", variables=["ugos", "vgos"]),
+    collekt.CMEMS("cmems_med_currents_my", variables=["uo", "vo"], depth=[1.0, 1.1]),
+)
+request = collekt.Request(
+    region=collekt.Region.from_bbox((-6.0, 20.0, 35.0, 45.0)),
+    start="2023-06-15",
+)
+
+fetcher = collekt.Fetcher(request=request, config=config, output_dir="data/collections")
+result = fetcher.download()
+print(result.manifest_path)
+
+# Assemble gridded sources onto a common grid and time axis.
+assembler = collekt.Assembler(result)
+dataset = assembler.to_xarray(grid="lowest_resolution", time="lowest_resolution")
+```
+
+`Region` can also be built from a point and radius
+(`Region.from_point_radius(lat, lon, radius_km)`) or a GeoJSON file
+(`Region.from_geojson(path)`).
+
+### Command line
+
+```bash
+collekt config show       # inspect the bundled catalog
+collekt doctor            # check the environment and configuration
+
+cat > datasets.yaml <<'YAML'
+datasets:
+  - provider: cmems
+    key: cmems_glorys_my
+    variables: [uo, vo]
+    depth: [1.0, 1.1]
+  - provider: cmems
+    key: cmems_duacs_my
+    variables: [ugos, vgos]
+  - provider: cmems
+    key: cmems_med_currents_my
+    variables: [uo, vo]
+    depth: [1.0, 1.1]
+YAML
+
+collekt fetch --bbox -6 20 35 45 --start 2023-06-15 \
+  --dataset-config datasets.yaml --output-dir data/collections
+```
+
+Use `--dry-run` to plan provider requests without downloading, and `--strict` to
+fail if any requested source is skipped.
+
+### Sources and configuration
+
+collekt ships a **curated catalog** of datasets (CMEMS global + Mediterranean,
+ECMWF Open Data, ERA5, Skytruth, Copernicus Data Space, eOdyn). Select concrete
+dataset keys with `DatasetConfig` or its YAML form, and put provider parameters
+such as variable names and depth ranges directly on each selected dataset. See
+[Products](https://ai4copsec.github.io/collekt/products.html) and
+[Credentials](https://ai4copsec.github.io/collekt/credentials.html).
+
+## Development
+
+```bash
+just lint    # format + lint with ruff
+just test    # run the test suite
+just docs    # build the Quarto documentation
+```
+
+The definition of done for any change: `just lint` and `just test` pass, with a
+test for any behavior changed.
+
+## Documentation
+
+Built with [Quarto](https://quarto.org) and
+[quartodoc](https://machow.github.io/quartodoc/) from `docs/`; the rendered site
+is published to GitHub Pages.
+
+## License
+
+[BSD-3-Clause](LICENSE).
+
+## Copyright
+
+Copyright (c) 2023-2026 [Simula Research Laboratory, Oslo, Norway](https://www.simula.no).
+
+## Acknowledgments
+
+Part of the EU project [AI4COPSEC](https://ai4copsec.eu), funded by the Horizon
+Europe framework programme under Grant Agreement N. 101190021.

collekt-0.2.0/pyproject.toml

@@ -0,0 +1,129 @@
+[project]
+name = "collekt"
+version = "0.2.0"
+description = "A library to facilitate spatio-temporal data collection across arbitrary datasources"
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
+readme = "README.md"
+keywords = [
+    "data source",
+    "collection",
+]
+requires-python = ">=3.13"
+classifiers = [
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Development Status :: 3 - Alpha",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+dependencies = [
+    "pydantic",
+    "pydantic-settings",
+    "pyyaml",
+    "requests",
+    "shapely",
+    "geopy",
+    "rich",
+    "tqdm",
+    "xarray>=2026.4.0",
+    "scipy>=1.18.0",
+    "cfgrib>=0.9.15.1",
+    "eccodeslib>=2.47.3.21",
+    "damast",
+    "geopandas",
+    "polars",
+    "copernicusmarine>=2.4.1",
+    "cdsapi>=0.7.7",
+    "ecmwf-opendata>=0.3.30",
+    "hozint-apiclient",
+    "python-dotenv>=1.2.2",
+]
+
+[[project.authors]]
+name = "Thomas Roehr"
+email = "roehr@simula.no"
+
+[[project.authors]]
+name = "Ola Formo Kihle"
+email = "kihle@simula.no"
+
+[[project.authors]]
+name = "Julien Vitay"
+email = "julien.vitay@eodyn.com"
+
+[[project.maintainers]]
+name = "Thomas Roehr"
+email = "roehr@simula.no"
+
+[project.urls]
+homepage = "https://ai4copsec.github.io/collekt"
+documentation = "https://ai4copsec.github.io/collekt"
+repository = "https://github.com/ai4copsec/collekt"
+
+[project.scripts]
+collekt = "collekt.cli.main:run"
+
+[build-system]
+requires = ["uv_build>=0.11.2,<0.12.0"]
+build-backend = "uv_build"
+
+[tool.ruff]
+target-version = "py313"
+line-length = 120
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "W",
+    "I",
+    "UP",
+    "B",
+    "SIM",
+]
+ignore = [
+    "E501",
+    "E741",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.commitizen]
+version = "0.2.0"
+name = "cz_conventional_commits"
+version_files = [
+    "pyproject.toml:^version = ",
+    "src/collekt/version.py:^__version__ = ",
+]
+tag_format = "$version"
+annotated_tag = true
+major_version_zero = true
+update_changelog_on_bump = false
+
+[dependency-groups]
+test = [
+    "jsonschema>=4.26.0",
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "pytest-timeout>=2.3",
+]
+docs = ["quartodoc>=0.11"]
+dev = [
+    "ruff>=0.4",
+    "commitizen>=3.0",
+    { include-group = "test" },
+]
+notebook = [
+    "ipykernel>=7.3.0",
+    "ipywidgets>=8.1.0",
+    "jupyterlab>=4.6.0",
+    "matplotlib>=3.11.0",
+]

collekt-0.2.0/pyproject.toml.orig

@@ -0,0 +1,122 @@
+[project]
+name = "collekt"
+version = "0.2.0"
+description = "A library to facilitate spatio-temporal data collection across arbitrary datasources"
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
+readme = "README.md"
+keywords = ["data source", "collection"]
+authors = [
+    { name = "Thomas Roehr", email = "roehr@simula.no" },
+    { name = "Ola Formo Kihle", email = "kihle@simula.no" },
+    { name = "Julien Vitay", email = "julien.vitay@eodyn.com" },
+]
+maintainers = [
+    { name = "Thomas Roehr", email = "roehr@simula.no" },
+]
+requires-python = ">=3.13"
+classifiers = [
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Development Status :: 3 - Alpha",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+]
+
+# collekt installs everything by default: most users want all sources and the
+# gridded Assembler. Provider clients are still imported lazily, so importing
+# collekt stays fast and degrades gracefully if a dependency is removed.
+dependencies = [
+    # engine
+    "pydantic",
+    "pydantic-settings",
+    "pyyaml",
+    "requests",
+    "shapely",
+    "geopy",
+    "rich",
+    "tqdm",
+    # gridded assembler + GRIB reading
+    "xarray>=2026.4.0",
+    "scipy>=1.18.0",
+    "cfgrib>=0.9.15.1",
+    "eccodeslib>=2.47.3.21",
+    # tabular feature backend (Skytruth)
+    "damast",
+    "geopandas",
+    "polars",
+    # provider clients
+    "copernicusmarine>=2.4.1",
+    "cdsapi>=0.7.7",
+    "ecmwf-opendata>=0.3.30",
+    "hozint-apiclient",
+    "python-dotenv>=1.2.2",
+]
+
+[project.urls]
+homepage = "https://ai4copsec.github.io/collekt"
+documentation = "https://ai4copsec.github.io/collekt"
+repository = "https://github.com/ai4copsec/collekt"
+
+[project.scripts]
+collekt = "collekt.cli.main:run"
+
+[build-system]
+requires = ["uv_build>=0.11.2,<0.12.0"]
+build-backend = "uv_build"
+
+[tool.ruff]
+target-version = "py313"
+line-length = 120
+
+[tool.ruff.format]
+docstring-code-format = true  # also format code blocks inside docstrings
+
+[tool.ruff.lint]
+# E/W pycodestyle · F pyflakes · I import sorting · UP pyupgrade
+# · B bugbear (likely bugs) · SIM simplify
+select = ["E", "F", "W", "I", "UP", "B", "SIM"]
+ignore = [
+    "E501",  # line length: the formatter already wraps code; don't double-flag long strings/URLs
+    "E741",  # allow short math-style names (l, I, O), common in research code
+]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]  # imports in a package __init__ are re-exports, not dead code
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.commitizen]
+version = "0.2.0"
+name = "cz_conventional_commits"
+version_files = [
+    "pyproject.toml:^version = ",
+    "src/collekt/version.py:^__version__ = ",
+]
+tag_format = "$version"
+annotated_tag = true
+major_version_zero = true
+update_changelog_on_bump = false
+
+# Dev-only tooling. uv installs the `dev` group automatically on `uv sync`
+# and `uv run`, so `just test` / `just lint` work without extra flags.
+[dependency-groups]
+test = [
+    "jsonschema>=4.26.0",
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "pytest-timeout>=2.3",
+]
+docs = ["quartodoc>=0.11"]
+dev = [
+    "ruff>=0.4",
+    "commitizen>=3.0",
+    { include-group = "test" },
+]
+notebook = [
+    "ipykernel>=7.3.0",
+    "ipywidgets>=8.1.0",
+    "jupyterlab>=4.6.0",
+    "matplotlib>=3.11.0",
+]

collekt-0.2.0/src/collekt/__init__.py

@@ -0,0 +1,54 @@
+"""collekt: a library to facilitate spatio-temporal data collection.
+
+Public API.
+"""
+
+from dotenv import find_dotenv, load_dotenv
+
+# Importing the sources package registers the built-in source adapters.
+from collekt import sources  # noqa: F401
+from collekt.assemble import Assembler
+from collekt.core.catalog import available_datasets, describe_datasets
+from collekt.core.datasets import (
+    CMEMS,
+    ERA5,
+    CopernicusDataSpace,
+    DatasetConfig,
+    ECMWFOpenData,
+    Hozint,
+    SkyTruth,
+    eOdyn,
+)
+from collekt.core.doctor import DoctorCheck, run_doctor
+from collekt.core.fetcher import Fetcher
+from collekt.core.request import Region, Request
+from collekt.core.result import Result
+
+from .version import __version__, __version_info__
+
+# Load a project-local .env (searched upward from the working directory) so
+# credentials placed there reach every provider adapter: the provider tools read
+# the OS environment, which this fills. Existing environment variables win.
+load_dotenv(find_dotenv(usecwd=True))
+
+__all__ = [
+    "Assembler",
+    "CMEMS",
+    "CopernicusDataSpace",
+    "DatasetConfig",
+    "DoctorCheck",
+    "ECMWFOpenData",
+    "ERA5",
+    "Fetcher",
+    "Hozint",
+    "Region",
+    "Request",
+    "Result",
+    "SkyTruth",
+    "available_datasets",
+    "describe_datasets",
+    "eOdyn",
+    "run_doctor",
+    "__version__",
+    "__version_info__",
+]

collekt-0.2.0/src/collekt/assemble/__init__.py

@@ -0,0 +1,10 @@
+"""Analysis-ready assembly of downloaded collections.
+
+`Assembler` builds gridded outputs (xarray / NetCDF / Zarr / NumPy) from a
+`collekt.Result`. Feature (tabular) sources are written directly by their
+adapters and are not assembled here.
+"""
+
+from collekt.assemble.gridded import Assembler
+
+__all__ = ["Assembler"]