api-24sea 2.2.1__tar.gz → 2.3.0__tar.gz

{api_24sea-2.2.1 → api_24sea-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: api-24sea
-Version: 2.2.1
+Version: 2.3.0
 Summary: Facilitate the users' interaction with the 24SEA API (https://api/24sea/eu) by providing pandas interfaces to the API endpoints.
 Author-email: Pietro D'Antuono <pietro.dantuono@24sea.eu>
 Maintainer-email: Pietro D'Antuono <pietro.dantuono@24sea.eu>

{api_24sea-2.2.1 → api_24sea-2.3.0}/api_24sea/datasignals/core.py

@@ -57,13 +57,62 @@ def _collapse_timestamp_rows(
     return collapsed.sort_index()
 
 
+def merge_data_frames(
+    data_frames: List[pd.DataFrame],
+    outer_join_on_timestamp: bool,
+) -> pd.DataFrame:
+    """Merge split datasignals responses in deterministic query order.
+
+    Parameters
+    ----------
+    data_frames : list of pandas.DataFrame
+        Response frames from location and/or timestamp-window requests.
+    outer_join_on_timestamp : bool
+        Whether to return one row per timestamp without site and location
+        columns.
+
+    Returns
+    -------
+    pandas.DataFrame
+        Merged response ordered by timestamp, or by site, location, and
+        timestamp when location columns are retained.
+    """
+    if not data_frames:
+        return pd.DataFrame()
+
+    merged = pd.concat(data_frames, ignore_index=True, sort=False)
+    if outer_join_on_timestamp:
+        return _collapse_timestamp_rows(merged)
+
+    sort_columns = [
+        column
+        for column in ["site", "location", "timestamp"]
+        if column in merged.columns
+    ]
+    if sort_columns:
+        merged = merged.sort_values(sort_columns, kind="stable")
+    return merged.reset_index(drop=True)
+
+
 class API(AuthABC):
     """Accessor for working with data signals coming from the 24SEA API."""
 
+    @property
+    def base_url(self) -> str:
+        """Return the normalized datasignals API base URL."""
+        return self._base_url
+
+    @base_url.setter
+    def base_url(self, value: str) -> None:
+        """Accept either the version root or datasignals application URL."""
+        normalized = value.rstrip("/")
+        if not normalized.endswith("/datasignals"):
+            normalized = f"{normalized}/datasignals"
+        self._base_url = f"{normalized}/"
+
     def __init__(self):
         super().__init__()
         self._metrics_overview = self._permissions_overview
-        self.base_url: str = str(f"{self.base_url}datasignals/")
         self._selected_metrics: Optional[pd.DataFrame] = None
 
     @property
@@ -362,56 +411,107 @@ class API(AuthABC):
         )
         grouped_metrics = query.group_metrics_by_site(self._metrics_overview)
 
+        split_plan = U.estimate_chunk_size(
+            tasks=list(grouped_metrics),
+            start_timestamp=query.start_timestamp,  # type: ignore[arg-type]
+            end_timestamp=query.end_timestamp,  # type: ignore[arg-type]
+            grouped_metrics=grouped_metrics,
+            selected_metrics=self._selected_metrics,
+        )
+        time_windows = U.split_timestamp_range(
+            query.start_timestamp,  # type: ignore[arg-type]
+            query.end_timestamp,  # type: ignore[arg-type]
+            split_plan.get("time_splits", 1),
+        )
+
         data_frames = []
         import concurrent.futures
 
-        # fmt: off
+        request_items: List[Dict[str, Any]] = []
+        for site, group in grouped_metrics:
+            site_name = _group_site_name(site)
+            group_locations = _get_group_locations(group)
+            location_batches = (
+                [[loc] for loc in group_locations]
+                if split_plan.get("split_by_location", False)
+                else [group_locations]
+            )
+            for location_batch in location_batches:
+                if not location_batch:
+                    continue
+                group_for_request = group[
+                    group["location"].isin(location_batch)
+                ].copy()
+                if group_for_request.empty:
+                    # Fallback to original group when no explicit location
+                    # column is available for filtering.
+                    group_for_request = group
+                for start_ts, end_ts in time_windows:
+                    request_items.append(
+                        {
+                            "site": site_name,
+                            "locations": location_batch,
+                            "start_timestamp": start_ts,
+                            "end_timestamp": end_ts,
+                            "group": group_for_request,
+                        }
+                    )
+
+        if split_plan.get("split_by_location", False):
+            logging.info(
+                "\033[32;1m⏳ Splitting data query into "
+                f"{len(request_items)} request(s) "
+                f"[{split_plan.get('location_splits', 1)} location split(s), "
+                f"{split_plan.get('time_splits', 1)} time window(s)].\033[0m"
+            )
+
         try:
             with concurrent.futures.ThreadPoolExecutor(
                 max_workers=threads, thread_name_prefix="24SEA"
             ) as executor:
-                future_to_data = {
+                futures = [
                     executor.submit(
                         U.fetch_data_sync,
                         f"{self.base_url}data/",
-                        _group_site_name(site),
-                        _get_group_locations(group),
-                        query.start_timestamp,
-                        query.end_timestamp,
+                        item["site"],
+                        item["locations"],
+                        item["start_timestamp"],
+                        item["end_timestamp"],
                         query.headers,
-                        group,
+                        item["group"],
                         self._auth,
                         timeout,
                         force_cache_miss=query.force_cache_miss,
                         method=query.method,
-                    ): site
-                    for site, group in grouped_metrics
-                }
-                for future in concurrent.futures.as_completed(future_to_data):
+                    )
+                    for item in request_items
+                ]
+                for future in futures:
                     data_frames.append(pd.DataFrame(future.result()))
         except RuntimeError:
-            for site, group in grouped_metrics:
-                data_frames.append(pd.DataFrame(U.fetch_data_sync(
-                    f"{self.base_url}data/", _group_site_name(site),
-                    _get_group_locations(group),
-                    query.start_timestamp, query.end_timestamp, query.headers,
-                    group, self._auth, timeout,
-                    force_cache_miss=query.force_cache_miss,
-                    method=query.method))
+            for item in request_items:
+                data_frames.append(
+                    pd.DataFrame(
+                        U.fetch_data_sync(
+                            f"{self.base_url}data/",
+                            item["site"],
+                            item["locations"],
+                            item["start_timestamp"],
+                            item["end_timestamp"],
+                            query.headers,
+                            item["group"],
+                            self._auth,
+                            timeout,
+                            force_cache_miss=query.force_cache_miss,
+                            method=query.method,
+                        )
+                    )
                 )
-        # fmt: on
         # data_frames.append(pd.DataFrame(r_.json()))
 
         # if outer_join_on_timestamp is True, lose the location and site columns
         # and join on timestamp
-        if outer_join_on_timestamp:
-            for i, df in enumerate(data_frames):
-                if df.empty:
-                    continue
-                data_frames[i] = _collapse_timestamp_rows(df)
-            data_ = pd.concat([data_] + data_frames, axis=1, join="outer")
-        else:
-            data_ = pd.concat([data_] + data_frames, ignore_index=True)
+        data_ = merge_data_frames(data_frames, outer_join_on_timestamp)
 
         logging.info("\033[32;1m✔️ Data successfully retrieved.\033[0m")
         # fmt: off
@@ -845,6 +945,7 @@ class API(AuthABC):
         df = _collapse_timestamp_rows(df, index_column="bucket_start")
         df.index.name = "timestamp"
         df = U.parse_timestamp(df)
+        df = df.apply(pd.to_numeric, errors="coerce")
         # Convert NaN to 0
         df = df.fillna(0)
         if as_dict:
@@ -1138,38 +1239,82 @@ class AsyncAPI(API):
         self._selected_metrics = query.get_selected_metrics(metrics_overview)
         grouped_metrics = query.group_metrics_by_site(metrics_overview)
 
-        # Split tasks into chunks of 5 to avoid firing tens of requests together
-
-        # fmt: off
-        tasks = [U.fetch_data_async(f"{self.base_url}data/", _group_site_name(site),
-                        _get_group_locations(group),
-                        query.start_timestamp, query.end_timestamp,
-                        query.headers, group, self._auth, timeout,
-                        max_retries,
-                        force_cache_miss=query.force_cache_miss,
-                        method=query.method)
-             for site, group in grouped_metrics]
-        chunk_size_dict = U.estimate_chunk_size(
-            tasks,
-            query.start_timestamp,   # type: ignore
-            query.end_timestamp,     # type: ignore
-            grouped_metrics,
-            self._selected_metrics
+        split_plan = U.estimate_chunk_size(
+            tasks=list(grouped_metrics),
+            start_timestamp=query.start_timestamp,  # type: ignore[arg-type]
+            end_timestamp=query.end_timestamp,  # type: ignore[arg-type]
+            grouped_metrics=grouped_metrics,
+            selected_metrics=self._selected_metrics,
         )
-        # fmt: on
-        data_frames = []
-        data_frames = await U.gather_in_chunks(
-            tasks, chunk_size=chunk_size_dict["chunk_size"], timeout=timeout  # type: ignore  # pylint: disable=C0301  # noqa:E501
+        time_windows = U.split_timestamp_range(
+            query.start_timestamp,  # type: ignore[arg-type]
+            query.end_timestamp,  # type: ignore[arg-type]
+            split_plan.get("time_splits", 1),
         )
 
-        if outer_join_on_timestamp:
-            for i, df in enumerate(data_frames):
-                if df.empty:
+        request_items: List[Dict[str, Any]] = []
+        for site, group in grouped_metrics:
+            site_name = _group_site_name(site)
+            group_locations = _get_group_locations(group)
+            location_batches = (
+                [[loc] for loc in group_locations]
+                if split_plan.get("split_by_location", False)
+                else [group_locations]
+            )
+            for location_batch in location_batches:
+                if not location_batch:
                     continue
-                data_frames[i] = _collapse_timestamp_rows(df)
-            data_ = pd.concat([data_] + data_frames, axis=1, join="outer")
-        else:
-            data_ = pd.concat([data_] + data_frames, ignore_index=True)
+                group_for_request = group[
+                    group["location"].isin(location_batch)
+                ].copy()
+                if group_for_request.empty:
+                    # Fallback to original group when no explicit location
+                    # column is available for filtering.
+                    group_for_request = group
+                for start_ts, end_ts in time_windows:
+                    request_items.append(
+                        {
+                            "site": site_name,
+                            "locations": location_batch,
+                            "start_timestamp": start_ts,
+                            "end_timestamp": end_ts,
+                            "group": group_for_request,
+                        }
+                    )
+
+        if split_plan.get("split_by_location", False):
+            logging.info(
+                "\033[32;1m⏳ Splitting data query into "
+                f"{len(request_items)} request(s) "
+                f"[{split_plan.get('location_splits', 1)} location split(s), "
+                f"{split_plan.get('time_splits', 1)} time window(s)].\033[0m"
+            )
+
+        tasks = [
+            U.fetch_data_async(
+                f"{self.base_url}data/",
+                item["site"],
+                item["locations"],
+                item["start_timestamp"],
+                item["end_timestamp"],
+                query.headers,
+                item["group"],
+                self._auth,
+                timeout,
+                max_retries,
+                force_cache_miss=query.force_cache_miss,
+                method=query.method,
+            )
+            for item in request_items
+        ]
+
+        chunk_size = split_plan["chunk_size"] or len(tasks)
+        chunk_size = max(1, min(chunk_size, len(tasks))) if tasks else 1
+        data_frames = await U.gather_in_chunks(
+            tasks, chunk_size=chunk_size, timeout=timeout
+        )
+
+        data_ = merge_data_frames(data_frames, outer_join_on_timestamp)
 
         if all(
             getattr(r, "status_code", 200) == 200
@@ -1605,9 +1750,10 @@ class AsyncAPI(API):
             logging.warning("\033[33;1mAvailability response missing "
                             "'bucket_start' column.\033[0m")
             # fmt: on
-            return df
+            return df.apply(pd.to_numeric, errors="coerce")
         df = _collapse_timestamp_rows(df, index_column="bucket_start")
         df.index.name = "timestamp"
+        df = df.apply(pd.to_numeric, errors="coerce")
         # Convert NaN to 0
         df = df.fillna(0)
         if as_dict:
@@ -1762,7 +1908,7 @@ def to_category_value(
                 categorized["metric"]
                 .str.extract(r"(_TP_)|(_TW_)|(_MP_)")
                 .bfill(axis=1)
-                .infer_objects(copy=False)[0]  # type: ignore
+                .infer_objects()[0]  # type: ignore
                 .str.replace("_", "")
             )
         else:
@@ -2025,9 +2171,9 @@ def to_star_schema(
     )
     # Convert object columns to string in fact_pivot_metrics
     if convert_object_columns_to_string:
-        object_columns = fact_pivot_metrics.select_dtypes(
-            include=["object"]
-        ).columns  # noqa: E501  # pylint: disable=C0301
+        object_columns = fact_pivot_metrics.columns[
+            fact_pivot_metrics.dtypes == "object"
+        ]
         for col in object_columns:
             fact_pivot_metrics = U.column_to_type(fact_pivot_metrics, col, str)
 

{api_24sea-2.2.1 → api_24sea-2.3.0}/api_24sea/utils.py

@@ -3,19 +3,21 @@
 import asyncio
 import datetime
 import logging
+import math
 import multiprocessing
 import sys
 import time
 import warnings
 from collections import defaultdict
-from types import CoroutineType
 from typing import (
     Any,
+    Awaitable,
     DefaultDict,
     Dict,
     Iterable,
     List,
     Optional,
+    Sequence,
     Tuple,
     Union,
 )
@@ -576,10 +578,13 @@ def estimate_chunk_size(
     logging.debug(f"Number of tasks: {len(tasks)}")
     logging.debug(f"Start timestamp: {start_timestamp}")
     logging.debug(f"End timestamp: {end_timestamp}")
-    logging.debug(f"Number of grouped metrics: {len(list(grouped_metrics))}")
-    logging.debug(f"Grouped metrics: {list(grouped_metrics)}")
+    grouped_metrics_list = list(grouped_metrics)
+    logging.debug(f"Number of grouped metrics: {len(grouped_metrics_list)}")
+    logging.debug(f"Grouped metrics: {grouped_metrics_list}")
     logging.debug(f"Selected metrics:\n{selected_metrics}")
 
+    max_request_mb = 100.0
+
     def parse_dt(dt):
         if isinstance(dt, str):
             try:
@@ -588,7 +593,9 @@ def estimate_chunk_size(
                 _dt = parse_shorthand_datetime(dt)
                 if _dt is not None:
                     dt = _dt.replace(tzinfo=None)
-        return dt
+                    return pd.to_datetime(dt)
+                return pd.to_datetime(dt)
+        return pd.to_datetime(dt)
 
     start_dt = parse_dt(start_timestamp)
     end_dt = parse_dt(end_timestamp)
@@ -610,43 +617,155 @@ def estimate_chunk_size(
                 bytes_per_metric[metric] *= 7
             if "_flt_" in metric.lower():
                 bytes_per_metric[metric] *= 35
+    total_locations = 0
     total_bytes = 0
-    for _, group in grouped_metrics:
+    for _, group in grouped_metrics_list:
         if isinstance(group, pd.DataFrame):
             group_met = normalize_group_values(group[target].tolist())
+            if "location" in group.columns:
+                location_count = max(1, group["location"].nunique())
+            else:
+                location_count = 1
         else:
             group_met = [group[target]] if hasattr(group, target) else []
+            location_count = 1
         # If group_met contains only "all", use all selected metrics
         if group_met == ["all"] and selected_metrics is not None:
             group_met = selected_metrics[target].tolist()
         group_bytes = sum(bytes_per_metric.get(m, 8) for m in group_met)
         total_bytes += n_points * group_bytes
+        total_locations += location_count
+
+    if grouped_metrics_list and total_locations == 0:
+        total_locations = 1
+
+    # If location detail is unavailable in grouped metrics, keep a conservative
+    # fallback so tests and callers without location data remain stable.
+    if not grouped_metrics_list:
+        total_locations = max(n_tasks, 1)
 
     # Check for negative or zero values (sanity check)
     if total_bytes <= 0 or n_points <= 0:
         total_bytes = 0
     total_mb = total_bytes / (1024 * 1024)
 
+    per_location_mb = (
+        total_mb / max(total_locations, 1) if total_locations > 0 else total_mb
+    )
+    time_splits = (
+        max(1, math.ceil(per_location_mb / max_request_mb))
+        if per_location_mb > 0
+        else 1
+    )
+    needs_split = total_mb > max_request_mb
+    planned_tasks = max(n_tasks, max(1, total_locations) * max(1, time_splits))
+
     # Determine chunk_size
-    if total_mb < 40:
-        chunk_size = n_tasks
-    elif total_mb < 80:
-        chunk_size = max(1, n_tasks // 2)
-    elif total_mb < 160:
-        chunk_size = max(1, n_tasks // 4)
+    request_size = (10, 20, 30, 40, 50, 60, 70, 80, 90, 100)  # in MB
+    threshold = request_size[-1]
+    for threshold in request_size:
+        if total_mb < threshold:
+            break
+
+    if n_tasks == 0:
+        chunk_size = 0
     else:
-        chunk_size = max(1, n_tasks // 8)
+        chunk_size = max(
+            1,
+            planned_tasks // (request_size.index(threshold) + 1),
+        )
 
     logging.info(f"Estimated request size: {total_mb:.2f} MB")
     return {
         "total_mb": total_mb,
         "n_tasks": n_tasks,
         "chunk_size": chunk_size,
+        "split_by_location": needs_split,
+        "location_splits": max(total_locations, 1),
+        "time_splits": time_splits,
+        "planned_tasks": planned_tasks,
+        "max_request_mb": max_request_mb,
     }
 
 
+def split_timestamp_range(
+    start_timestamp: Union[str, datetime.datetime],
+    end_timestamp: Union[str, datetime.datetime],
+    n_splits: int,
+) -> List[Tuple[str, str]]:
+    """Split a timestamp range into contiguous half-open windows.
+
+    The returned windows are suitable for repeated API calls that should be
+    merged back into one DataFrame without gaps or duplicate boundary
+    timestamps.
+
+    Parameters
+    ----------
+    start_timestamp : str or datetime.datetime
+        Inclusive start of the full query window.
+    end_timestamp : str or datetime.datetime
+        Exclusive end of the full query window.
+    n_splits : int
+        Requested number of windows.
+
+    Returns
+    -------
+    list of tuple of str
+        Ordered ``(start, end)`` timestamp pairs. Each window end is the next
+        window start because the datasignals SQL queries use half-open ranges.
+    """
+
+    def parse_timestamp(
+        dt: Union[str, datetime.datetime],
+    ) -> pd.Timestamp:
+        if isinstance(dt, str):
+            try:
+                return pd.to_datetime(dt)
+            except pd._libs.tslibs.parsing.DateParseError:  # type: ignore
+                _dt = parse_shorthand_datetime(dt)
+                if _dt is not None:
+                    return pd.to_datetime(_dt.replace(tzinfo=None))
+                return pd.to_datetime(dt)
+        return pd.to_datetime(dt)
+
+    start_dt = parse_timestamp(start_timestamp)
+    end_dt = parse_timestamp(end_timestamp)
+
+    def format_timestamp(ts: pd.Timestamp) -> str:
+        if ts.tzinfo is not None:
+            return str(ts.tz_convert("UTC").strftime("%Y-%m-%dT%H:%M:%SZ"))
+        return str(ts.strftime("%Y-%m-%dT%H:%M:%S"))
+
+    if n_splits <= 1 or end_dt <= start_dt:
+        return [(format_timestamp(start_dt), format_timestamp(end_dt))]
+
+    total_seconds = int((end_dt - start_dt).total_seconds())
+    if total_seconds <= 0:
+        return [(format_timestamp(start_dt), format_timestamp(end_dt))]
+
+    effective_splits = min(n_splits, total_seconds)
+    windows: List[Tuple[str, str]] = []
+    for idx in range(effective_splits):
+        start_seconds = (total_seconds * idx) // effective_splits
+        end_seconds = (total_seconds * (idx + 1)) // effective_splits
+        window_start = start_dt + pd.Timedelta(seconds=start_seconds)
+        window_end = start_dt + pd.Timedelta(seconds=end_seconds)
+        windows.append(
+            (
+                format_timestamp(window_start),
+                format_timestamp(window_end),
+            )
+        )
+
+    last_start, _ = windows[-1]
+    windows[-1] = (last_start, format_timestamp(end_dt))
+    return windows
+
+
 async def gather_in_chunks(
-    tasks: List[CoroutineType], chunk_size: int = 5, timeout: int = 3600
+    tasks: Sequence[Awaitable[Any]],
+    chunk_size: int = 5,
+    timeout: int = 3600,
 ) -> List:
     results = []
     chunk_results = []

{api_24sea-2.2.1 → api_24sea-2.3.0}/api_24sea/version.py

@@ -19,7 +19,7 @@ Attributes:
 import re
 from typing import NamedTuple, Optional
 
-__version__: str = "2.2.1"
+__version__: str = "2.3.0"
 
 _REGEX = "".join(
     [

{api_24sea-2.2.1 → api_24sea-2.3.0}/pyproject.toml

@@ -240,12 +240,12 @@ norecursedirs = [".bzr", ".direnv", ".eggs", ".git", ".git-rewrite", ".hg",
                  "logs"]
 minversion = "6.0"
 addopts = "-ra -q"
+asyncio_default_fixture_loop_scope = "function"
 testpaths = [
     "test",
     "tests",
 ]
 filterwarnings = [
-    'ignore:configuration option "asyncio_default_fixture_loop_scope" is unset',
     'ignore:was never awaited',
 ]
 
@@ -256,7 +256,7 @@ filterwarnings = [
 # directories from being committed.
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "3.13.0"
+version = "2.3.0"
 bump_message = "bump(version): {current_version} → {new_version}"
 update_changelog_on_bump = true
 style = [