vortexa-claude-skills 1.0.0

package/lib/timeseries.py

@@ -0,0 +1,271 @@
+"""CargoTimeSeries queries and breakdown parsing."""
+
+import re
+import pandas as pd
+
+from lib.utils import _freq_norm
+
+
+def cargo_timeseries_split(
+    time_min, time_max, activity, split_property,
+    products=None, origins=None, destinations=None,
+    storage_locations=None, vessels=None,
+    unit="b", frequency="day",
+    intra_movements="exclude_intra_country",
+):
+    """Query CargoTimeSeries with split property, return pivoted wide DataFrame."""
+    from vortexasdk import CargoTimeSeries
+
+    kwargs = {
+        "filter_activity": activity,
+        "filter_time_min": time_min,
+        "filter_time_max": time_max,
+        "timeseries_frequency": frequency,
+        "timeseries_property": split_property,
+        "timeseries_unit": unit,
+        "intra_movements": intra_movements,
+    }
+    if products:
+        kwargs["filter_products"] = products
+    if origins:
+        kwargs["filter_origins"] = origins
+    if destinations:
+        kwargs["filter_destinations"] = destinations
+    if storage_locations:
+        kwargs["filter_storage_locations"] = storage_locations
+    if vessels:
+        kwargs["filter_vessels"] = vessels
+
+    result = CargoTimeSeries().search(**kwargs).to_list()
+    if not result:
+        return pd.DataFrame()
+
+    return _parse_split_result(result)
+
+
+def _parse_split_result(result):
+    """Parse .to_list() CTS result with breakdown into wide DataFrame."""
+    rows = []
+    for item in result:
+        d = pd.to_datetime(getattr(item, "key"))
+        total = float(getattr(item, "value", 0.0) or 0.0)
+        breakdown = getattr(item, "breakdown", None) or []
+
+        if not breakdown:
+            rows.append((d, "Other", total))
+            continue
+
+        summed = 0.0
+        for b in breakdown:
+            label = (
+                getattr(b, "label", None)
+                or getattr(b, "name", None)
+                or getattr(b, "id", None)
+                or "Unknown"
+            )
+            val = float(getattr(b, "value", 0.0) or 0.0)
+            rows.append((d, label, val))
+            summed += val
+
+        residual = max(total - summed, 0.0)
+        if residual > 0:
+            rows.append((d, "Other", residual))
+
+    df_long = pd.DataFrame(rows, columns=["date", "series", "value"])
+    df_long["value"] = pd.to_numeric(df_long["value"], errors="coerce").fillna(0.0)
+
+    wide = (
+        df_long.groupby(["date", "series"], as_index=False)["value"]
+        .sum()
+        .pivot(index="date", columns="series", values="value")
+        .fillna(0.0)
+        .sort_index()
+    )
+    wide["Total"] = wide.sum(axis=1)
+    wide.columns.name = None
+    return wide
+
+
+def _parse_timeseries_breakdown(df_all, show_top_x=None):
+    """Flatten CargoTimeSeries 'columns=all' breakdown.N.* fields into wide Date x Category table."""
+    df = df_all.copy()
+    if "key" not in df.columns:
+        raise ValueError("Unexpected TimeSeries frame: missing 'key'.")
+
+    date = pd.to_datetime(df["key"], utc=True, errors="coerce")
+    if date.isna().all():
+        raise ValueError("TimeSeries: 'key' could not be parsed to datetime.")
+
+    pat_label = re.compile(r"^breakdown\.(\d+)\.label$")
+    labels = [c for c in df.columns if pat_label.match(c)]
+
+    if not labels:
+        out = pd.DataFrame({
+            "Date": date.dt.floor("D"),
+            "Total": pd.to_numeric(df.get("value", 0), errors="coerce").fillna(0.0),
+        })
+        return out.groupby("Date", as_index=False)["Total"].sum().sort_values("Date")
+
+    idxs = sorted({int(pat_label.match(c).group(1)) for c in labels})
+    parts = []
+    for i in idxs:
+        lab_col = f"breakdown.{i}.label"
+        val_col = f"breakdown.{i}.value"
+        if lab_col in df.columns and val_col in df.columns:
+            parts.append(pd.DataFrame({
+                "Date": date,
+                "label": df[lab_col].fillna(""),
+                "value": pd.to_numeric(df[val_col], errors="coerce").fillna(0.0),
+            }))
+
+    if not parts:
+        return pd.DataFrame({"Date": pd.to_datetime([])})
+
+    long = pd.concat(parts, ignore_index=True)
+    long = long[long["label"] != ""].copy()
+    if long.empty:
+        return pd.DataFrame({"Date": pd.to_datetime([])})
+
+    long["Date"] = pd.to_datetime(long["Date"]).dt.floor("D")
+    agg = long.groupby(["Date", "label"], as_index=False)["value"].sum()
+    if agg.empty:
+        return pd.DataFrame({"Date": pd.to_datetime([])})
+
+    wide = agg.pivot(index="Date", columns="label", values="value").fillna(0.0)
+    wide.index = pd.to_datetime(wide.index)
+    wide.index.name = "Date"
+    wide = wide.sort_index()
+
+    if show_top_x is not None and wide.shape[1] > show_top_x:
+        totals = wide.sum(axis=0).sort_values(ascending=False)
+        keep = list(totals.index[:show_top_x])
+        other = list(totals.index[show_top_x:])
+        if other:
+            wide["Other"] = wide[other].sum(axis=1)
+        wide = wide[keep + (["Other"] if other else [])]
+
+    out = wide.reset_index()
+    out = out.rename_axis(columns=None)
+    return out
+
+
+def flows_time_series_split(
+    *,
+    time_min,
+    time_max,
+    activity,
+    split_property="shipping_region_v2",
+    frequency="week",
+    top_n=8,
+    unit="b",
+    products=None,
+    origins=None,
+    destinations=None,
+    exclude_origins=None,
+    exclude_destinations=None,
+    exclude_products=None,
+    storage_locations=None,
+    intra_movements="exclude_intra_country",
+):
+    """CargoTimeSeries split by a property with Top-N + Other and resampling."""
+    from vortexasdk import CargoTimeSeries
+
+    freq_norm = _freq_norm(frequency)
+
+    split_api = split_property
+    if split_property == "shipping_region_v2":
+        if activity in ("loading_state", "loading_end"):
+            split_api = "origin_shipping_region_v2"
+        elif activity in ("unloading_state", "unloading_start"):
+            split_api = "destination_shipping_region_v2"
+        elif activity == "storing_state":
+            split_api = "storage_location_shipping_region_v2"
+
+    kwargs = {
+        "filter_time_min": time_min,
+        "filter_time_max": time_max,
+        "filter_activity": activity,
+        "timeseries_frequency": "day",
+        "timeseries_property": split_api,
+        "timeseries_unit": unit,
+        "intra_movements": intra_movements,
+    }
+    if products:
+        kwargs["filter_products"] = products
+    if origins:
+        kwargs["filter_origins"] = origins
+    if destinations:
+        kwargs["filter_destinations"] = destinations
+    if exclude_origins:
+        kwargs["exclude_origins"] = exclude_origins
+    if exclude_destinations:
+        kwargs["exclude_destinations"] = exclude_destinations
+    if exclude_products:
+        kwargs["exclude_products"] = exclude_products
+    if storage_locations:
+        kwargs["filter_storage_locations"] = storage_locations
+
+    res = CargoTimeSeries().search(**kwargs).to_list()
+
+    if not res:
+        return pd.DataFrame(), pd.DataFrame()
+
+    rows = []
+    for it in res:
+        d = pd.to_datetime(getattr(it, "key"))
+        total = float(getattr(it, "value", 0.0) or 0.0)
+        breakdown = getattr(it, "breakdown", None) or []
+        if not breakdown:
+            rows.append((d, "Other", total))
+            continue
+        summed = 0.0
+        for b in breakdown:
+            label = (
+                getattr(b, "label", None)
+                or getattr(b, "name", None)
+                or getattr(b, "id", None)
+                or "Unknown"
+            )
+            val = float(getattr(b, "value", 0.0) or 0.0)
+            rows.append((d, label, val))
+            summed += val
+        residual = max(total - summed, 0.0)
+        if residual > 0:
+            rows.append((d, "Other", residual))
+
+    df_long = pd.DataFrame(rows, columns=["date", "series", "value"])
+    df_long["value"] = pd.to_numeric(df_long["value"], errors="coerce").fillna(0.0)
+
+    full_piv = (
+        df_long.groupby(["date", "series"], as_index=False)["value"]
+        .sum()
+        .pivot(index="date", columns="series", values="value")
+        .fillna(0.0)
+        .sort_index()
+    )
+
+    resample_map = {
+        "week": "W-SUN", "month": "MS", "quarter": "QS", "year": "YS",
+    }
+    if freq_norm in resample_map:
+        plot_piv_base = full_piv.resample(resample_map[freq_norm]).sum()
+    else:
+        plot_piv_base = full_piv.copy()
+
+    sums = (
+        df_long[df_long.series != "Other"]
+        .groupby("series")["value"]
+        .sum()
+        .sort_values(ascending=False)
+    )
+    keep = sums.head(top_n).index.tolist()
+
+    plot_piv = plot_piv_base.copy()
+    other_cols = [c for c in plot_piv.columns if c not in keep]
+    if keep:
+        plot_piv = plot_piv.reindex(columns=keep + other_cols)
+    if other_cols:
+        plot_piv["Other"] = plot_piv[other_cols].sum(axis=1)
+        plot_piv = plot_piv[keep + (["Other"] if "Other" in plot_piv.columns else [])]
+
+    return full_piv, plot_piv.fillna(0.0)

package/lib/utils.py

@@ -0,0 +1,120 @@
+"""Date helpers, DataFrame formatters, and shared utilities."""
+
+import pandas as pd
+from datetime import datetime
+
+
+def _to_dt(x):
+    """Convert to datetime, strip timezone."""
+    x = pd.to_datetime(x, errors="coerce")
+    if isinstance(x, (pd.Series, pd.Index)):
+        return x.dt.tz_localize(None)
+    if isinstance(x, (pd.Timestamp, datetime)):
+        try:
+            return x.tz_localize(None)
+        except Exception:
+            try:
+                return x.tz_convert(None)
+            except Exception:
+                return x
+    return x
+
+
+def _cols(df, names):
+    """Safe column selection -- returns only columns that exist in df."""
+    return [c for c in names if c in df.columns]
+
+
+def _freq_norm(x):
+    """Normalize user-friendly frequency string to API value."""
+    m = {
+        "d": "day", "day": "day", "daily": "day",
+        "w": "week", "week": "week", "weekly": "week",
+        "m": "month", "mon": "month", "month": "month", "monthly": "month",
+        "q": "quarter", "quarter": "quarter", "quarterly": "quarter",
+        "y": "year", "yr": "year", "year": "year", "annual": "year", "annually": "year",
+    }
+    x = (x or "").lower()
+    out = m.get(x, x)
+    if out not in {"day", "week", "month", "quarter", "year"}:
+        raise ValueError("frequency must map to day/week/month/quarter/year")
+    return out
+
+
+_CM_COLUMN_MAP = {
+    "cargo_movement_id": "Cargo ID",
+    "quantity": "Volume",
+    "discharge_quantity": "Discharge Volume",
+    "status": "Status",
+    "product.group.label": "Product Group",
+    "product.group_product.label": "Product Sub-Group",
+    "product.category.label": "Product Category",
+    "product.grade.label": "Product Grade",
+    "events.cargo_port_load_event.0.location.terminal.label": "Load Terminal",
+    "events.cargo_port_load_event.0.location.port.label": "Load Port",
+    "events.cargo_port_load_event.0.location.country.label": "Load Country",
+    "events.cargo_port_load_event.0.location.shipping_region_v2.label": "Load Shipping Region",
+    "events.cargo_port_load_event.0.start_timestamp": "Load Start",
+    "events.cargo_port_load_event.0.end_timestamp": "Load Date",
+    "events.cargo_port_unload_event.0.location.terminal.label": "Unload Terminal",
+    "events.cargo_port_unload_event.0.location.port.label": "Unload Port",
+    "events.cargo_port_unload_event.0.location.country.label": "Unload Country",
+    "events.cargo_port_unload_event.0.location.shipping_region_v2.label": "Unload Shipping Region",
+    "events.cargo_port_unload_event.0.start_timestamp": "Unload Date",
+    "events.cargo_port_unload_event.0.end_timestamp": "Unload End",
+    "vessels.0.name": "Vessel Name",
+    "vessels.0.imo": "Vessel IMO",
+    "vessels.0.vessel_class": "Vessel Class",
+    "vessels.0.corporate_entities.charterer.label": "Charterer",
+    "vessels.0.corporate_entities.effective_controller.label": "Effective Controller",
+    "vessels.0.corporate_entities.time_charterer.label": "Time Charterer",
+}
+
+_VSE_COLUMN_MAP = {
+    "START DATE": "Voyage Start",
+    "END DATE": "Voyage End",
+    "LATEST PRODUCT GROUP": "Product Group",
+    "LATEST PRODUCT GRADE": "Product Grade",
+    "ORIGIN COUNTRY": "Origin Country",
+    "DESTINATION COUNTRY": "Destination Country",
+    "ORIGIN PORT": "Origin Port",
+    "DESTINATION PORT": "Destination Port",
+}
+
+
+def rename_cm_columns(df):
+    """Rename CargoMovements DataFrame columns to human-readable names."""
+    return df.rename(columns={k: v for k, v in _CM_COLUMN_MAP.items() if k in df.columns})
+
+
+def rename_vse_columns(df):
+    """Rename VoyagesSearchEnriched DataFrame columns to human-readable names."""
+    return df.rename(columns={k: v for k, v in _VSE_COLUMN_MAP.items() if k in df.columns})
+
+
+def top_n_with_other(df, n=10, total_col="Total"):
+    """Keep top N columns by sum, aggregate the rest into 'Other'."""
+    out = df.copy()
+
+    # Separate total column if present
+    has_total = total_col in out.columns
+    if has_total:
+        total_series = out.pop(total_col)
+
+    # Rank columns by sum descending
+    ranked = out.sum().sort_values(ascending=False)
+    top_cols = ranked.index[:n].tolist()
+    rest_cols = ranked.index[n:].tolist()
+
+    # Build result with top N columns
+    result = out[top_cols].copy()
+
+    # Add Other only if there are leftover columns
+    if rest_cols:
+        result["Other"] = out[rest_cols].sum(axis=1)
+
+    # Restore total column
+    if has_total:
+        result[total_col] = total_series
+
+    return result

package/lib/vessels.py

@@ -0,0 +1,192 @@
+"""Vessel tracking -- IMO resolution, signals positions, fleet tracking."""
+
+import os
+
+import pandas as pd
+import requests
+
+
+signals_url = "https://api.vortexa.com/v6/signals/vessel-positions"
+
+
+def map_imos_to_vortexa_ids(imo_dict):
+    """Map {name: imo_number} dict to DataFrame with vessel IDs."""
+    from vortexasdk import Vessels
+
+    api_key = os.environ.get("VORTEXA_API_KEY")
+    if api_key is None:
+        raise RuntimeError("VORTEXA_API_KEY not set in environment.")
+
+    clean_imo_dict = {}
+    for name, imo in imo_dict.items():
+        if imo is None:
+            continue
+        try:
+            clean_imo_dict[name] = int(str(imo).strip())
+        except ValueError:
+            print(f"WARNING: Could not parse IMO '{imo}' for '{name}'. Skipping.")
+
+    imo_list = list(clean_imo_dict.values())
+    if not imo_list:
+        raise ValueError("No valid IMOs provided in imo_dict.")
+
+    vessels_df = Vessels().search().to_df(columns=["id", "name", "imo", "vessel_class"])
+    vessels_df["imo"] = pd.to_numeric(vessels_df["imo"], errors="coerce")
+    vessels_subset = vessels_df[vessels_df["imo"].isin(imo_list)]
+
+    mapping_records = []
+    for user_name, imo in clean_imo_dict.items():
+        matches = vessels_subset[vessels_subset["imo"] == imo]
+        if matches.empty:
+            print(f"WARNING: IMO {imo} for '{user_name}' not found.")
+            continue
+        row = matches.iloc[0]
+        mapping_records.append({
+            "user_vessel_name": user_name,
+            "imo": imo,
+            "vessel_id": row["id"],
+            "vortexa_name": row["name"],
+            "vessel_class": row.get("vessel_class"),
+        })
+
+    mapping_df = pd.DataFrame(mapping_records)
+    if mapping_df.empty:
+        raise ValueError("No IMOs could be matched to Vortexa vessels.")
+    return mapping_df
+
+
+def extract_vessel_positions(session, vessel_ids_short, start_timestamp, end_timestamp, interval="1d", timeout=45):
+    """Single batch signals API call for vessel positions."""
+    api_key = os.environ.get("VORTEXA_API_KEY")
+    params = {
+        "apikey": api_key,
+        "time_min": start_timestamp.isoformat(),
+        "time_max": end_timestamp.isoformat(),
+        "interval": interval,
+        "vessel_id": vessel_ids_short,
+    }
+    resp = session.get(signals_url, params=params, timeout=timeout)
+    resp.raise_for_status()
+    data = resp.json()
+    return pd.DataFrame(data.get("data", []))
+
+
+def fetch_signals_for_fleet(vessel_ids, start_timestamp, end_timestamp,
+                            interval="1d", group=20, verbose=True, timeout=45):
+    """Batch vessel position pull with rate limiting (groups of N)."""
+    all_positions = pd.DataFrame()
+    total = len(vessel_ids)
+
+    with requests.Session() as session:
+        for i in range(0, total, group):
+            batch_ids = vessel_ids[i:i + group]
+            if not batch_ids:
+                continue
+            batch_ids_short = [vid[:16] for vid in batch_ids]
+
+            df_batch = extract_vessel_positions(
+                session=session,
+                vessel_ids_short=batch_ids_short,
+                start_timestamp=start_timestamp,
+                end_timestamp=end_timestamp,
+                interval=interval,
+                timeout=timeout,
+            )
+            if not df_batch.empty:
+                all_positions = pd.concat([all_positions, df_batch], ignore_index=True)
+
+            if verbose:
+                done = min(i + group, total)
+                print(f"{done} vessels done out of {total}")
+
+    return all_positions
+
+
+def build_daily_positions_from_imos(imo_dict, time_min, time_max, interval="1d", group=20, verbose=True):
+    """Pull signals API positions for a fleet defined by IMO numbers."""
+    mapping_df = map_imos_to_vortexa_ids(imo_dict)
+    vessel_ids = mapping_df["vessel_id"].tolist()
+
+    positions = fetch_signals_for_fleet(
+        vessel_ids=vessel_ids,
+        start_timestamp=time_min,
+        end_timestamp=time_max,
+        interval=interval,
+        group=group,
+        verbose=verbose,
+    )
+
+    if positions.empty:
+        raise ValueError("Signals API returned no data for the selected vessels/time window.")
+
+    positions = positions.copy()
+    positions["timestamp"] = pd.to_datetime(positions["timestamp"], utc=True)
+    positions["date"] = positions["timestamp"].dt.floor("D")
+    positions["vessel_id_short"] = positions["vessel_id"].str[:16]
+
+    mapping_df = mapping_df.copy()
+    mapping_df.rename(columns={"vessel_id": "vortexa_vessel_id"}, inplace=True)
+    mapping_df["vessel_id_short"] = mapping_df["vortexa_vessel_id"].str[:16]
+
+    merged = positions.merge(
+        mapping_df[["user_vessel_name", "imo", "vortexa_vessel_id", "vessel_id_short"]],
+        on="vessel_id_short",
+        how="left",
+    )
+    merged = merged[merged["user_vessel_name"].notna()]
+
+    if merged.empty:
+        raise ValueError("Signals returned positions but none matched provided IMOs.")
+
+    merged_sorted = merged.sort_values(
+        ["user_vessel_name", "imo", "vessel_id_short", "date", "timestamp"]
+    )
+
+    daily = (
+        merged_sorted
+        .groupby(["user_vessel_name", "imo", "vortexa_vessel_id", "vessel_id_short", "date"], as_index=False)
+        .first()
+    )
+
+    lat_wide = daily.pivot(index="date", columns="user_vessel_name", values="lat").sort_index()
+    lon_wide = daily.pivot(index="date", columns="user_vessel_name", values="lon").sort_index()
+
+    daily["timestamp"] = daily["date"]
+    daily_long = daily[
+        ["date", "timestamp", "lat", "lon", "user_vessel_name", "imo",
+         "vortexa_vessel_id", "vessel_id_short"]
+    ]
+
+    return daily_long, lat_wide, lon_wide
+
+
+def build_latlon_timeseries_table(lat_wide, lon_wide):
+    """Combine lat/lon wide tables into single table with Date + per-vessel Lon/Lat."""
+    lat_named = lat_wide.copy()
+    lon_named = lon_wide.copy()
+    lat_named.columns.name = None
+    lon_named.columns.name = None
+
+    lat_named = lat_named.add_suffix("_lat")
+    lon_named = lon_named.add_suffix("_lon")
+
+    out = pd.concat([lat_named, lon_named], axis=1)
+    out = out.sort_index(axis=1)
+    out = out.reset_index().rename(columns={"date": "Date"})
+    return out
+
+
+def track_vessels_daily(
+    imo_dict, time_min, time_max,
+    interval="1d", group=20,
+):
+    """End-to-end: IMO dict -> signals positions -> daily data."""
+    daily_long, lat_wide, lon_wide = build_daily_positions_from_imos(
+        imo_dict=imo_dict,
+        time_min=time_min,
+        time_max=time_max,
+        interval=interval,
+        group=group,
+        verbose=True,
+    )
+    return {"daily_long": daily_long, "lat_wide": lat_wide, "lon_wide": lon_wide}