cloudnet-api-client 0.9.2__tar.gz → 0.11.0__tar.gz

cloudnet_api_client-0.11.0/.github/dataportal.env

@@ -0,0 +1,29 @@
+NODE_ENV=test
+SS_MODE=remote
+TYPEORM_HOST=db
+TYPEORM_USERNAME=dataportal
+TYPEORM_PASSWORD=dev
+TYPEORM_DATABASE=dataportal
+TYPEORM_PORT=5432
+TYPEORM_SYNCHRONIZE=false
+TYPEORM_MIGRATIONS_RUN=true
+TYPEORM_LOGGING=false
+TYPEORM_ENTITIES=build/entity/*.js
+TYPEORM_MIGRATIONS=build/migration/*.js
+DP_SS_URL=http://storage-service:5900
+DP_SS_USER=test
+DP_SS_PASSWORD=test
+DP_BACKEND_URL=http://localhost:3000/api
+DP_FRONTEND_URL=http://localhost:8080
+GEOLITE2_COUNTRY_PATH=tests/data/GeoLite2-Country-Test.mmdb
+CITATION_SERVICE_URL=http://citation-service
+DATACITE_API_URL=http://localhost:5802
+DATACITE_API_USERNAME=XXX
+DATACITE_API_PASSWORD=XXX
+DATACITE_API_TIMEOUT_MS=2000
+DATACITE_DOI_SERVER=http://handle.datacite.test
+DATACITE_DOI_PREFIX=XXX
+LABELLING_URL=http://localhost:5803
+HANDLE_API_URL=http://localhost:5804
+DVAS_URL=https://dvas.test
+DC_URL=https://dc.test

cloudnet_api_client-0.11.0/.github/db.env

@@ -0,0 +1,4 @@
+TZ="Europe/Helsinki"
+POSTGRES_USER=admin
+POSTGRES_PASSWORD=admin
+PGUSER=admin

cloudnet_api_client-0.11.0/.github/docker-compose.yml

@@ -0,0 +1,45 @@
+services:
+  dataportal-backend:
+    image: ghcr.io/actris-cloudnet/dataportal-backend
+    ports:
+      - "3000:3000"
+    depends_on:
+      db:
+        condition: service_healthy
+    volumes:
+      - ../dataportal-fixtures:/dataportal-fixtures
+      - ../backend-fixtures:/backend-fixtures
+    env_file:
+      - dataportal.env
+    command:
+      [
+        "sh",
+        "-c",
+        "node build/fixtures.js /backend-fixtures/backend/fixtures TRUNCATE && node build/fixtures.js /dataportal-fixtures APPEND && npm run start",
+      ]
+  db:
+    image: "postgres:16"
+    volumes:
+      - ./initdb.d:/docker-entrypoint-initdb.d
+    ports:
+      - "54321:54321"
+    env_file:
+      - db.env
+    healthcheck:
+      test: ["CMD", "psql", "-c", "select 1"]
+      interval: 1s
+      retries: 120
+  moto-server:
+    image: "motoserver/moto:3.0.1"
+  storage-service:
+    image: ghcr.io/actris-cloudnet/storage-service
+    ports:
+      - "5900:5900"
+    depends_on:
+      db:
+        condition: service_healthy
+      moto-server:
+        condition: service_started
+    env_file:
+      - ss.env
+    command: ["sh", "-c", "node build/init.js && npm start"]

cloudnet_api_client-0.11.0/.github/initdb.d/init-dbs.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE USER ss WITH PASSWORD 'dev';
+    CREATE DATABASE ss OWNER ss;
+    CREATE USER dataportal WITH PASSWORD 'dev';
+    CREATE DATABASE dataportal OWNER dataportal;
+EOSQL

cloudnet_api_client-0.11.0/.github/ss.env

@@ -0,0 +1,12 @@
+NODE_ENV=test
+SS_USER=test
+SS_PWHASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
+SS_MAXOBJECTSPERBUCKET=10000
+PGHOST=db
+PGDATABASE=ss
+PGUSER=ss
+PGPASSWORD=dev
+S3_ENDPOINT=http://moto-server:5000
+S3_ACCESSKEYID=dev
+S3_SECRETACCESSKEY=dev
+S3_FORCE_PATH_STYLE=true

cloudnet_api_client-0.11.0/.github/workflows/test.yml

@@ -0,0 +1,64 @@
+name: Test and lint
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    timeout-minutes: 5
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Cache Python dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ matrix.python-version }}-
+            ${{ runner.os }}-pip-
+
+      - name: Checkout fixtures
+        uses: actions/checkout@v4
+        with:
+          repository: actris-cloudnet/dataportal
+          sparse-checkout: backend/fixtures/
+          sparse-checkout-cone-mode: false
+          path: backend-fixtures
+
+      - name: Checkout production fixtures
+        uses: actions/checkout@v4
+        with:
+          repository: actris-cloudnet/dataportal-fixtures
+          path: dataportal-fixtures
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Start dataportal
+        run: docker compose -f .github/docker-compose.yml up -d --wait
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install .[dev,test]
+
+      - name: Run pre-commit checks
+        if: matrix.python-version == '3.13'
+        run: pre-commit run --all-files --show-diff-on-failure
+
+      - name: Run tests
+        run: pytest -s -vv
+
+      - name: Shutdown backend
+        if: always()
+        run: docker compose -f .github/docker-compose.yml down

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/.gitignore

@@ -1,3 +1,4 @@
 __pycache__
 *~
 .venv
+*.ipynb*

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 exclude: ^tests/data/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-case-conflict
       - id: check-executables-have-shebangs
@@ -13,7 +13,7 @@ repos:
         args: ["--fix", "lf"]
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.0
+    rev: v0.12.8
     hooks:
       - id: ruff
         args: ["--fix"]

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/CHANGELOG.md

@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.11.0 – 2025-08-16
+
+- Adjust routes and responses
+- Improve tests
+
+## 0.10.0 – 2025-08-13
+
+- Add `volatile` to metadata response
+- Run CI tests against true dataportal backend
+
 ## 0.9.2 – 2025-08-04
 
 - Use updated Cloudnet API

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cloudnet-api-client
-Version: 0.9.2
+Version: 0.11.0
 Summary: Cloudnet API client
 Author-email: Simo Tukiainen <simo.tukiainen@fmi.fi>
 License-File: LICENSE
@@ -22,7 +22,9 @@ Requires-Dist: types-requests; extra == 'dev'
 Requires-Dist: types-tqdm; extra == 'dev'
 Provides-Extra: test
 Requires-Dist: mypy; extra == 'test'
+Requires-Dist: netcdf4; extra == 'test'
 Requires-Dist: pytest; extra == 'test'
+Requires-Dist: pytest-asyncio; extra == 'test'
 Description-Content-Type: text/markdown
 
 [![CI](https://github.com/actris-cloudnet/cloudnet-api-client/actions/workflows/test.yml/badge.svg)](https://github.com/actris-cloudnet/cloudnet-api-client/actions/workflows/test.yml)
@@ -80,6 +82,7 @@ Parameters:
 | updated_at_to       | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
 | instrument_id       | `str` or `list[str]`        | `None`  | "rpg-fmcw-94"                                        |
 | instrument_pid      | `str` or `list[str]`        | `None`  | "https://hdl.handle.net/21.12132/3.191564170f8a4686" |
+| model_id            | `str` or `list[str]`        | `None`  | "gdas1"                                              |
 | product\*           | `str` or `list[str]`        | `None`  | "classification"                                     |
 | show_legacy\*       | `bool`                      | `False` |                                                      |
 | filename_prefix\*\* | `str` or `list[str]`        | `None`  | "stare"                                              |
@@ -127,13 +130,33 @@ Parameters:
 
 \* = only with `RawMetadata`
 
+### `APIClient().file()` &rarr; `ProductMetadata`
+
+Fetch metadata of a single file.
+
+Parameters:
+
+| name | type                 |
+| ---- | -------------------- |
+| uuid | `str` or `uuid.UUID` |
+
+### `APIClient().versions()` &rarr; `list[VersionMetadata]`
+
+Fetch information of all versions of a file.
+
+Parameters:
+
+| name | type                 |
+| ---- | -------------------- |
+| uuid | `str` or `uuid.UUID` |
+
 ### `APIClient().sites()` &rarr; `list[Site]`
 
 Fetch cloudnet sites.
 
 Parameters:
 
-| name    | type                 | Choices                                   | default |
+| name    | type                 | choices                                   | default |
 | ------- | -------------------- | ----------------------------------------- | ------- |
 | site_id | `str`                |                                           | `None`  |
 | type    | `str` or `list[str]` | "cloudnet", "campaign", "model", "hidden" | `None`  |
@@ -144,7 +167,7 @@ Fetch cloudnet products.
 
 Parameters:
 
-| name | type                 | Choices                                   | default |
+| name | type                 | choices                                   | default |
 | ---- | -------------------- | ----------------------------------------- | ------- |
 | type | `str` or `list[str]` | "instrument", "geophysical", "evaluation" | `None`  |
 

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/README.md

@@ -53,6 +53,7 @@ Parameters:
 | updated_at_to       | `str`, `date` or `datetime` | `None`  | "2025-01-01T12:00:00"                                |
 | instrument_id       | `str` or `list[str]`        | `None`  | "rpg-fmcw-94"                                        |
 | instrument_pid      | `str` or `list[str]`        | `None`  | "https://hdl.handle.net/21.12132/3.191564170f8a4686" |
+| model_id            | `str` or `list[str]`        | `None`  | "gdas1"                                              |
 | product\*           | `str` or `list[str]`        | `None`  | "classification"                                     |
 | show_legacy\*       | `bool`                      | `False` |                                                      |
 | filename_prefix\*\* | `str` or `list[str]`        | `None`  | "stare"                                              |
@@ -100,13 +101,33 @@ Parameters:
 
 \* = only with `RawMetadata`
 
+### `APIClient().file()` → `ProductMetadata`
+
+Fetch metadata of a single file.
+
+Parameters:
+
+| name | type                 |
+| ---- | -------------------- |
+| uuid | `str` or `uuid.UUID` |
+
+### `APIClient().versions()` → `list[VersionMetadata]`
+
+Fetch information of all versions of a file.
+
+Parameters:
+
+| name | type                 |
+| ---- | -------------------- |
+| uuid | `str` or `uuid.UUID` |
+
 ### `APIClient().sites()` → `list[Site]`
 
 Fetch cloudnet sites.
 
 Parameters:
 
-| name    | type                 | Choices                                   | default |
+| name    | type                 | choices                                   | default |
 | ------- | -------------------- | ----------------------------------------- | ------- |
 | site_id | `str`                |                                           | `None`  |
 | type    | `str` or `list[str]` | "cloudnet", "campaign", "model", "hidden" | `None`  |
@@ -117,7 +138,7 @@ Fetch cloudnet products.
 
 Parameters:
 
-| name | type                 | Choices                                   | default |
+| name | type                 | choices                                   | default |
 | ---- | -------------------- | ----------------------------------------- | ------- |
 | type | `str` or `list[str]` | "instrument", "geophysical", "evaluation" | `None`  |
 

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/cloudnet_api_client/client.py

@@ -3,12 +3,12 @@ import calendar
 import datetime
 import os
 import re
-import uuid
 from dataclasses import fields, is_dataclass
 from os import PathLike
 from pathlib import Path
 from typing import TypeVar, cast
 from urllib.parse import urljoin
+from uuid import UUID
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -25,6 +25,7 @@ from cloudnet_api_client.containers import (
     RawMetadata,
     RawModelMetadata,
     Site,
+    VersionMetadata,
 )
 from cloudnet_api_client.dl import download_files
 
@@ -42,6 +43,8 @@ class APIClient:
         base_url: str = "https://cloudnet.fmi.fi/api/",
         session: requests.Session | None = None,
     ) -> None:
+        if not base_url.endswith("/"):
+            base_url += "/"
         self.base_url = base_url
         self.session = session or _make_session()
 
@@ -74,7 +77,7 @@ class APIClient:
                 instrument_id=obj["instrument"]["id"],
                 model=obj["model"],
                 type=obj["type"],
-                uuid=uuid.UUID(obj["uuid"]),
+                uuid=UUID(obj["uuid"]),
                 pid=obj["pid"],
                 owners=obj["owners"],
                 serial_number=obj["serialNumber"],
@@ -83,6 +86,31 @@ class APIClient:
             for obj in res
         ]
 
+    def file(
+        self,
+        uuid: str | UUID,
+    ) -> ProductMetadata:
+        res = self._get_response(f"files/{uuid}")
+        return _build_meta_objects(res)[0]
+
+    def versions(self, uuid: str | UUID) -> list[VersionMetadata]:
+        res = self._get_response(
+            f"files/{uuid}/versions",
+            {"properties": ["pid", "dvasId", "legacy", "size", "checksum"]},
+        )
+        return [
+            VersionMetadata(
+                uuid=UUID(obj["uuid"]),
+                created_at=_parse_datetime(obj["createdAt"]),
+                pid=obj["pid"],
+                dvas_id=obj["dvasId"],
+                legacy=obj["legacy"],
+                size=int(obj["size"]),
+                checksum=obj["checksum"],
+            )
+            for obj in res
+        ]
+
     def metadata(
         self,
         site_id: QueryParam = None,
@@ -105,6 +133,10 @@ class APIClient:
             "product": product,
             "showLegacy": show_legacy,
         }
+        if show_legacy is not True:
+            # API shows legacy files with any value (even <False>)
+            del params["showLegacy"]
+
         _add_date_params(
             params, date, date_from, date_to, updated_at, updated_at_from, updated_at_to
         )
@@ -125,7 +157,8 @@ class APIClient:
             or (model_id is not None and (product is None or "model" in product))
         ):
             for key in ("showLegacy", "product", "instrument", "instrumentPid"):
-                del params[key]
+                if key in params:
+                    del params[key]
             params["model"] = model_id
             files_res += self._get_response("model-files", params)
 
@@ -418,7 +451,7 @@ def _build_meta_objects(res: list[dict]) -> list[ProductMetadata]:
             created_at=_parse_datetime(obj["createdAt"]),
             updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
-            uuid=uuid.UUID(obj["uuid"]),
+            uuid=UUID(obj["uuid"]),
             site=_create_site_object(obj["site"]),
         )
         for obj in res
@@ -437,7 +470,7 @@ def _build_raw_meta_objects(res: list[dict]) -> list[RawMetadata]:
             created_at=_parse_datetime(obj["createdAt"]),
             updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
-            uuid=uuid.UUID(obj["uuid"]),
+            uuid=UUID(obj["uuid"]),
             site=_create_site_object(obj["site"]),
         )
         for obj in res
@@ -456,7 +489,7 @@ def _build_raw_model_meta_objects(res: list[dict]) -> list[RawModelMetadata]:
             created_at=_parse_datetime(obj["createdAt"]),
             updated_at=_parse_datetime(obj["updatedAt"]),
             size=int(obj["size"]),
-            uuid=uuid.UUID(obj["uuid"]),
+            uuid=UUID(obj["uuid"]),
             site=_create_site_object(obj["site"]),
         )
         for obj in res
@@ -498,10 +531,10 @@ def _create_site_object(metadata: dict) -> Site:
 
 def _create_instrument_object(metadata: dict) -> Instrument:
     return Instrument(
-        instrument_id=metadata["instrumentId"],
+        instrument_id=metadata.get("instrumentId"),  # not in api/files/:uuid
         model=metadata["model"],
         type=metadata["type"],
-        uuid=uuid.UUID(metadata["uuid"]),
+        uuid=UUID(metadata["uuid"]),
         pid=metadata["pid"],
         owners=metadata["owners"],
         serial_number=metadata["serialNumber"],

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/cloudnet_api_client/containers.py

@@ -35,7 +35,7 @@ class Product:
 
 @dataclass(frozen=True, slots=True)
 class Instrument:
-    instrument_id: str  # CLU internal identifier, e.g. "rpg-fmcw-94"
+    instrument_id: str | None  # CLU internal identifier, e.g. "rpg-fmcw-94"
     model: str  # From ACTRIS Vocabulary, e.g. "RPG-FMCW-94 DP"
     type: str  # From ACTRIS Vocabulary, e.g. "Doppler non-scanning cloud radar"
     name: str  # e.g. "FMI RPG-FMCW-94 (Pallas)"
@@ -86,3 +86,24 @@ class ProductMetadata(Metadata):
     product: Product
     instrument: Instrument | None
     model: Model | None
+    volatile: bool
+    legacy: bool
+    pid: str
+    dvas_id: str | None
+    error_level: str | None
+    coverage: float
+    timeliness: str
+    format: str
+    start_time: datetime.datetime | None
+    stop_time: datetime.datetime | None
+
+
+@dataclass(frozen=True, slots=True)
+class VersionMetadata:
+    uuid: uuid.UUID
+    created_at: datetime.datetime
+    pid: str
+    checksum: str
+    legacy: bool
+    size: int
+    dvas_id: str | None

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/cloudnet_api_client/utils.py

@@ -1,3 +1,4 @@
+import base64
 import hashlib
 from os import PathLike
 from typing import Literal
@@ -7,13 +8,17 @@ def sha256sum(filename: str | PathLike) -> str:
     return _calc_hash_sum(filename, "sha256")
 
 
-def md5sum(filename: str | PathLike) -> str:
-    return _calc_hash_sum(filename, "md5")
+def md5sum(filename: str | PathLike, is_base64: bool = False) -> str:
+    return _calc_hash_sum(filename, "md5", is_base64)
 
 
-def _calc_hash_sum(filename: str | PathLike, method: Literal["sha256", "md5"]) -> str:
+def _calc_hash_sum(
+    filename: str | PathLike, method: Literal["sha256", "md5"], is_base64: bool = False
+) -> str:
     hash_sum = getattr(hashlib, method)()
     with open(filename, "rb") as f:
         for byte_block in iter(lambda: f.read(4096), b""):
             hash_sum.update(byte_block)
+    if is_base64:
+        return base64.b64encode(hash_sum.digest()).decode("utf-8")
     return hash_sum.hexdigest()

cloudnet_api_client-0.11.0/cloudnet_api_client/version.py

@@ -0,0 +1 @@
+__version__ = "0.11.0"

{cloudnet_api_client-0.9.2 → cloudnet_api_client-0.11.0}/pyproject.toml

@@ -22,12 +22,15 @@ dependencies = ["aiohttp", "numpy", "requests", "tqdm"]
 dynamic = ["version"]
 
 [project.optional-dependencies]
-test = ["mypy", "pytest"]
+test = ["mypy", "netCDF4", "pytest", "pytest-asyncio"]
 dev = ["pre-commit", "release-version", "types-requests", "types-tqdm"]
 
 [tool.hatch.version]
 path = "cloudnet_api_client/version.py"
 
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
 [tool.release-version]
 filename = "cloudnet_api_client/version.py"
 pattern = ["__version__ = \"(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)\""]