polytope-python 2.1.3__tar.gz → 2.1.4__tar.gz

{polytope_python-2.1.3 → polytope_python-2.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytope-python
-Version: 2.1.3
+Version: 2.1.4
 Summary: Polytope datacube feature extraction library
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 Maintainer-email: James Hawkes <James.Hawkes@ecmwf.int>, Mathilde Leuridan <Mathilde.Leuridan@ecmwf.int>
@@ -45,6 +45,9 @@ Provides-Extra: unstructured
 Requires-Dist: eckit; extra == "unstructured"
 Provides-Extra: catalogue
 Requires-Dist: qubed; extra == "catalogue"
+Provides-Extra: switching-grids
+Requires-Dist: eccodes; extra == "switching-grids"
+Requires-Dist: pyfdb; extra == "switching-grids"
 Dynamic: license-file
 
 

{polytope_python-2.1.3 → polytope_python-2.1.4}/polytope_feature/_version.py

@@ -1,2 +1,2 @@
 # Do not change! Do not track in version control!
-__version__ = "2.1.3"
+__version__ = "2.1.4"

polytope_python-2.1.4/polytope_feature/datacube/switching_grid_helper.py

@@ -0,0 +1,215 @@
+import json
+import math
+import os
+import tempfile
+
+import eccodes
+import pyfdb
+
+# from polytope_feature.options import MapperConfig
+
+
+def get_first_grib_message(req):
+    fdb = pyfdb.FDB()
+
+    # Make sure that we are accessing a single georef so that the grid is consistent
+    assert "georef" in req.keys()
+
+    first_field = next(fdb.list(req, keys=True))["keys"]
+
+    field = fdb.retrieve(first_field)
+
+    # Normalize the retrieve() result into a plain `bytes` object
+    if hasattr(field, "read"):
+        # file-like object: read the contents
+        data = field.read()
+    else:
+        data = field
+
+    # Convert common buffer types to bytes
+    if isinstance(data, bytes):
+        msg_bytes = data
+    elif isinstance(data, bytearray):
+        msg_bytes = bytes(data)
+    elif isinstance(data, memoryview):
+        msg_bytes = data.tobytes()
+    else:
+        # last resort: try to construct bytes (may raise)
+        try:
+            msg_bytes = bytes(data)
+        except Exception as e:
+            raise TypeError(f"Unsupported GRIB message type: {type(data)!r}") from e
+
+    gid = eccodes.codes_new_from_message(msg_bytes)
+    return gid
+
+
+def get_gridspec_lamebert_conformal(gid):
+    # Lambert lam grid
+
+    to_rad = math.pi / 180
+
+    md5hash = eccodes.codes_get(gid, "md5GridSection")
+
+    earth_round = (eccodes.codes_get(gid, "shapeOfTheEarth") == 0) or (
+        eccodes.codes_get(gid, "shapeOfTheEarth") == 6
+    )
+
+    if earth_round:
+        if eccodes.codes_get(gid, "shapeOfTheEarth") == 6:
+            radius = 6371229
+        elif eccodes.codes_get(gid, "shapeOfTheEarth") == 0:
+            radius = 6367470
+    else:
+        # TODO: set the earth major and minor axis accordingly
+        pass
+
+    nv = eccodes.codes_get(gid, "NV")
+    nx = eccodes.codes_get(gid, "Nx")
+    ny = eccodes.codes_get(gid, "Ny")
+    LoVInDegrees = eccodes.codes_get(gid, "LoV") / 1000000
+    Dx = eccodes.codes_get(gid, "Dx")
+    Dy = eccodes.codes_get(gid, "Dy")
+    latFirstInRadians = (
+        eccodes.codes_get(gid, "latitudeOfFirstGridPoint") / 1000000 * to_rad
+    )
+    lonFirstInRadians = (
+        eccodes.codes_get(gid, "longitudeOfFirstGridPoint") / 1000000 * to_rad
+    )
+    LoVInRadians = eccodes.codes_get(gid, "LoV") / 1000000 * to_rad
+    Latin1InRadians = eccodes.codes_get(gid, "Latin1") / 1000000 * to_rad
+    Latin2InRadians = eccodes.codes_get(gid, "Latin2") / 1000000 * to_rad
+    LaDInRadians = eccodes.codes_get(gid, "LaD") / 1000000 * to_rad
+
+    gridspec = {
+        "type": "lambert_conformal",
+        "earth_round": earth_round,
+        "radius": radius,
+        "nv": nv,
+        "nx": nx,
+        "ny": ny,
+        "LoVInDegrees": LoVInDegrees,
+        "Dx": Dx,
+        "Dy": Dy,
+        "latFirstInRadians": latFirstInRadians,
+        "lonFirstInRadians": lonFirstInRadians,
+        "LoVInRadians": LoVInRadians,
+        "Latin1InRadians": Latin1InRadians,
+        "Latin2InRadians": Latin2InRadians,
+        "LaDInRadians": LaDInRadians,
+    }
+    return (gridspec, md5hash)
+
+
+def get_gridspec_icon(gid):
+    # ICON
+    # TODO: Need the following:
+    # uuid: Optional[str] = None
+    md5hash = eccodes.codes_get(gid, "md5GridSection")
+    gridspec = {}
+    return (gridspec, md5hash)
+
+
+def get_gridspec_and_hash(gid):
+    grid_type = eccodes.codes_get(gid, "gridType")
+    if grid_type == "lambert_lam":
+        return get_gridspec_lamebert_conformal(gid)
+    elif grid_type == "icon":
+        return get_gridspec_icon(gid)
+    else:
+        raise ValueError(f"Unsupported grid type: {grid_type}")
+
+
+# TODO: extract the right info and then write it to file, one for the grid hash and one for the actual config
+
+
+def lookup_grid_config(req):
+    gid = get_first_grib_message(req)
+    req_georef = req["georef"]
+
+    # Cache file stored alongside this module
+    GRID_CACHE_FILE = os.path.join(os.path.dirname(__file__), "grid_cache.json")
+
+    def _load_cache():
+        try:
+            with open(GRID_CACHE_FILE, "r", encoding="utf-8") as fh:
+                return json.load(fh)
+        except FileNotFoundError:
+            return {}
+        except Exception:
+            return {}
+
+    def _save_cache(cache):
+        dirpath = os.path.dirname(GRID_CACHE_FILE)
+        os.makedirs(dirpath, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(dir=dirpath, prefix=".grid_cache.")
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as fh:
+                json.dump(cache, fh, indent=2, sort_keys=True)
+            os.replace(tmp, GRID_CACHE_FILE)
+        finally:
+            if os.path.exists(tmp):
+                try:
+                    os.remove(tmp)
+                except Exception:
+                    pass
+
+    # Use a stable serialization of the georef as the cache key
+    try:
+        cache_key = json.dumps(req_georef, sort_keys=True, default=str)
+    except Exception:
+        cache_key = str(req_georef)
+
+    cache = _load_cache()
+
+    try:
+        if cache_key in cache:
+            entry = cache[cache_key]
+            return (entry.get("gridspec"), entry.get("md5hash"))
+
+        gridspec, md5hash = get_gridspec_and_hash(gid)
+        cache[cache_key] = {"gridspec": gridspec, "md5hash": md5hash}
+        try:
+            _save_cache(cache)
+        except Exception:
+            # Swallow cache write errors but continue to return computed value
+            pass
+        return (gridspec, md5hash)
+    finally:
+        eccodes.codes_release(gid)
+
+
+# def gridspec_to_grid_config(gridspec, md5hash):
+#     if gridspec.get("type") == "lambert_conformal":
+#         mc = MapperConfig(
+#             name="mapper",
+#             type="lambert_conformal",
+#             md5_hash=md5hash,
+#             is_spherical=gridspec.get("earth_round"),
+#             radius=gridspec.get("radius"),
+#             nv=gridspec.get("nv"),
+#             nx=gridspec.get("nx"),
+#             ny=gridspec.get("ny"),
+#             LoVInDegrees=gridspec.get("LoVInDegrees"),
+#             Dx=gridspec.get("Dx"),
+#             Dy=gridspec.get("Dy"),
+#             latFirstInRadians=gridspec.get("latFirstInRadians"),
+#             lonFirstInRadians=gridspec.get("lonFirstInRadians"),
+#             LoVInRadians=gridspec.get("LoVInRadians"),
+#             Latin1InRadians=gridspec.get("Latin1InRadians"),
+#             Latin2InRadians=gridspec.get("Latin2InRadians"),
+#             LaDInRadians=gridspec.get("LaDInRadians"),
+#         )
+#         return mc
+#     return None
+
+# def replace_grid_config_in_options(options, req):
+#     gridspec, md5hash = lookup_grid_config(req)
+#     grid_config = gridspec_to_grid_config(gridspec, md5hash)
+#     if grid_config is not None:
+#         for axis_conf in options.axis_config:
+#             for idx, transformation in enumerate(axis_conf.transformations):
+#                 if getattr(transformation, "name", None) == "mapper":
+#                     axis_conf.transformations[idx] = grid_config
+#                     return True
+#     return False

{polytope_python-2.1.3 → polytope_python-2.1.4}/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py

@@ -191,30 +191,66 @@ class TypeChangeSubHourlyTimeSteps(DatacubeAxisTypeChange):
             return pd.Timedelta(hours=int(value))
 
         if isinstance(value, str):
-            # Extract hours and minutes using regex
+            # # Extract hours and minutes using regex
+            # h_match = re.search(r"(\d+)\s*h", value)
+            # m_match = re.search(r"(\d+)\s*m(?:in)?", value)
+
+            # hours = int(h_match.group(1)) if h_match else 0
+            # minutes = int(m_match.group(1)) if m_match else 0
+
+            # return pd.Timedelta(hours=hours, minutes=minutes)
+            # Extract days, hours, minutes and seconds using regex
+            d_match = re.search(r"(\d+)\s*d", value)
             h_match = re.search(r"(\d+)\s*h", value)
             m_match = re.search(r"(\d+)\s*m(?:in)?", value)
+            s_match = re.search(r"(\d+)\s*s(?:ec)?", value)
 
+            days = int(d_match.group(1)) if d_match else 0
             hours = int(h_match.group(1)) if h_match else 0
             minutes = int(m_match.group(1)) if m_match else 0
+            seconds = int(s_match.group(1)) if s_match else 0
 
-            return pd.Timedelta(hours=hours, minutes=minutes)
+            return pd.Timedelta(
+                days=days, hours=hours, minutes=minutes, seconds=seconds
+            )
 
         raise ValueError(f"Unsupported timestep format: {value}")
 
     def make_str(self, value):
+        return_vals = []
         for val in value:
-            total_minutes = int(val.total_seconds() // 60)
-            hours, minutes = divmod(total_minutes, 60)
-
-            if hours == 0 and minutes == 0:
-                return "0"
+            total_seconds = int(val.total_seconds())
+            days, rem = divmod(total_seconds, 86400)
+            hours, rem = divmod(rem, 3600)
+            minutes, seconds = divmod(rem, 60)
+
+            if days == 0 and hours == 0 and minutes == 0 and seconds == 0:
+                return_vals.append("0")
+            # Prefer compact forms when possible to remain backwards compatible
+            elif days > 0:
+                parts = [f"{days}d"]
+                if hours > 0:
+                    parts.append(f"{hours}h")
+                if minutes > 0:
+                    parts.append(f"{minutes}m")
+                if seconds > 0:
+                    parts.append(f"{seconds}s")
+                return_vals.append("".join(parts))
             elif hours == 0:
-                return f"{minutes}m"
-            elif minutes == 0:
-                return f"{hours}"
+                if minutes == 0:
+                    return_vals.append(f"{seconds}s")
+                elif seconds == 0:
+                    return_vals.append(f"{minutes}m")
+                else:
+                    return_vals.append(f"{minutes}m{seconds}s")
+            # hours > 0
+            elif minutes == 0 and seconds == 0:
+                return_vals.append(f"{hours}")
+            elif seconds == 0:
+                return_vals.append(f"{hours}h{minutes}m")
             else:
-                return f"{hours}h{minutes}m"
+                return_vals.append(f"{hours}h{minutes}m{seconds}s")
+        return return_vals
 
 
 _type_to_datacube_type_change_lookup = {

{polytope_python-2.1.3 → polytope_python-2.1.4}/polytope_feature/options.py

@@ -5,6 +5,8 @@ from typing import Dict, List, Literal, Optional, Tuple, Union
 from conflator import ConfigModel, Conflator
 from pydantic import ConfigDict
 
+from polytope_feature.datacube.switching_grid_helper import lookup_grid_config
+
 
 class TransformationConfig(ConfigModel):
     model_config = ConfigDict(extra="forbid")
@@ -86,6 +88,7 @@ class Config(ConfigModel):
     alternative_axes: Optional[List[GribJumpAxesConfig]] = []
     use_catalogue: Optional[bool] = False
     engine_options: Optional[Dict[str, str]] = {}
+    dynamic_grid: Optional[bool] = False
 
 
 class PolytopeOptions(ABC):
@@ -100,9 +103,21 @@ class PolytopeOptions(ABC):
         axis_config = config_options.axis_config
         compressed_axes_config = config_options.compressed_axes_config
         pre_path = config_options.pre_path
+        dynamic_grid = config_options.dynamic_grid
         alternative_axes = config_options.alternative_axes
         use_catalogue = config_options.use_catalogue
         engine_options = config_options.engine_options
+
+        if dynamic_grid:
+            # TODO: look at the pre-path and query the eccodes function to get the new grid option
+            # TODO: then change the grid option inside of the axis_config
+            try:
+                replaced = replace_grid_config_in_options(config_options, pre_path)
+                if replaced:
+                    axis_config = config_options.axis_config
+            except Exception:
+                # Fail silently and continue with original config
+                pass
         return (
             axis_config,
             compressed_axes_config,
@@ -111,3 +126,41 @@ class PolytopeOptions(ABC):
             use_catalogue,
             engine_options,
         )
+
+
+def gridspec_to_grid_config(gridspec, md5hash):
+    if gridspec.get("type") == "lambert_conformal":
+        mc = MapperConfig(
+            name="mapper",
+            type="lambert_conformal",
+            md5_hash=md5hash,
+            is_spherical=gridspec.get("earth_round"),
+            radius=gridspec.get("radius"),
+            nv=gridspec.get("nv"),
+            nx=gridspec.get("nx"),
+            ny=gridspec.get("ny"),
+            LoVInDegrees=gridspec.get("LoVInDegrees"),
+            Dx=gridspec.get("Dx"),
+            Dy=gridspec.get("Dy"),
+            latFirstInRadians=gridspec.get("latFirstInRadians"),
+            lonFirstInRadians=gridspec.get("lonFirstInRadians"),
+            LoVInRadians=gridspec.get("LoVInRadians"),
+            Latin1InRadians=gridspec.get("Latin1InRadians"),
+            Latin2InRadians=gridspec.get("Latin2InRadians"),
+            LaDInRadians=gridspec.get("LaDInRadians"),
+            axes=["latitude", "longitude"],
+        )
+        return mc
+    return None
+
+
+def replace_grid_config_in_options(options, req):
+    gridspec, md5hash = lookup_grid_config(req)
+    grid_config = gridspec_to_grid_config(gridspec, md5hash)
+    if grid_config is not None:
+        for axis_conf in options.axis_config:
+            for idx, transformation in enumerate(axis_conf.transformations):
+                if getattr(transformation, "name", None) == "mapper":
+                    axis_conf.transformations[idx] = grid_config
+                    return True
+    return False

{polytope_python-2.1.3 → polytope_python-2.1.4}/polytope_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytope-python
-Version: 2.1.3
+Version: 2.1.4
 Summary: Polytope datacube feature extraction library
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 Maintainer-email: James Hawkes <James.Hawkes@ecmwf.int>, Mathilde Leuridan <Mathilde.Leuridan@ecmwf.int>
@@ -45,6 +45,9 @@ Provides-Extra: unstructured
 Requires-Dist: eckit; extra == "unstructured"
 Provides-Extra: catalogue
 Requires-Dist: qubed; extra == "catalogue"
+Provides-Extra: switching-grids
+Requires-Dist: eccodes; extra == "switching-grids"
+Requires-Dist: pyfdb; extra == "switching-grids"
 Dynamic: license-file
 
 

{polytope_python-2.1.3 → polytope_python-2.1.4}/polytope_python.egg-info/SOURCES.txt

@@ -136,6 +136,7 @@ polytope_feature/datacube/__init__.py
 polytope_feature/datacube/datacube_axis.py
 polytope_feature/datacube/index_tree.proto
 polytope_feature/datacube/index_tree_pb2.py
+polytope_feature/datacube/switching_grid_helper.py
 polytope_feature/datacube/tensor_index_tree.py
 polytope_feature/datacube/tree_encoding.py
 polytope_feature/datacube/backends/__init__.py

{polytope_python-2.1.3 → polytope_python-2.1.4}/polytope_python.egg-info/requires.txt

@@ -10,6 +10,10 @@ protobuf
 [catalogue]
 qubed
 
+[switching_grids]
+eccodes
+pyfdb
+
 [tests]
 pytest
 pytest-cov

{polytope_python-2.1.3 → polytope_python-2.1.4}/pyproject.toml

@@ -62,6 +62,11 @@ catalogue = [
     "qubed"
 ]
 
+switching_grids = [
+    "eccodes",
+    "pyfdb"
+]
+
 [project.urls]
 repository = "https://github.com/ecmwf/polytope"
 documentation = "https://polytope.readthedocs.io/en/latest/"

{polytope_python-2.1.3 → polytope_python-2.1.4}/tests/test_type_change_transformation.py

@@ -59,14 +59,18 @@ class TestIntTypeChangeTransformation:
         assert type_change_transform.transform_type("1h15m") == pd.Timedelta(
             hours=1, minutes=15
         )
-
-        assert (
-            type_change_transform.make_str([pd.Timedelta(hours=1, minutes=15)])
-            == "1h15m"
+        assert type_change_transform.transform_type("1d2h30m15s") == pd.Timedelta(
+            days=1, hours=2, minutes=30, seconds=15
         )
-        assert type_change_transform.make_str([pd.Timedelta(minutes=20)]) == "20m"
-        assert type_change_transform.make_str([pd.Timedelta(hours=2)]) == "2"
-        assert type_change_transform.make_str([pd.Timedelta(hours=0)]) == "0"
+
+        assert type_change_transform.make_str([pd.Timedelta(hours=1, minutes=15)]) == [
+            "1h15m"
+        ]
+        assert type_change_transform.make_str([pd.Timedelta(minutes=20)]) == ["20m"]
+        assert type_change_transform.make_str([pd.Timedelta(hours=2)]) == ["2"]
+        assert type_change_transform.make_str([pd.Timedelta(hours=0)]) == ["0"]
+        assert type_change_transform.make_str([pd.Timedelta(seconds=30)]) == ["30s"]
+        assert type_change_transform.make_str([pd.Timedelta(days=1)]) == ["1d"]
 
     def test_subhourly_step_compact_type_change_axis(self):
         type_change_transform = TypeChangeSubHourlyTimeStepsCompact(