bb-integrations-library 3.0.11__py3-none-any.whl

bb_integrations_lib/pipelines/parsers/tank_reading_parser.py

@@ -0,0 +1,155 @@
+import json
+from typing import AsyncGenerator, Optional, cast
+from typing import override
+
+import dateutil
+from babel.numbers import parse_decimal, NumberFormatError
+from bb_integrations_lib.gravitate.sd_api import GravitateSDAPI
+from bb_integrations_lib.util.config.manager import GlobalConfigManager
+from dateutil.parser import parse
+from dateutil.tz import gettz
+from loguru import logger
+
+from bb_integrations_lib.mappers.rita_mapper import RitaMapper, AsyncMappingProvider
+from bb_integrations_lib.models.rita.issue import IssueCategory
+from bb_integrations_lib.models.rita.mapping import MappingType
+from bb_integrations_lib.protocols.flat_file import TankReading, TankMonitorType
+from bb_integrations_lib.protocols.pipelines import Parser
+from bb_integrations_lib.shared.model import MappingMode
+from bb_integrations_lib.util.utils import lookup
+
+tzmapping = {
+    'EST': gettz("US/Eastern"),
+    'EDT': gettz("US/Eastern"),
+    'CST': gettz("US/Central"),
+    'CDT': gettz("US/Central"),
+    'MST': gettz("US/Mountain"),
+    'MDT': gettz("US/Mountain"),
+    'PST': gettz("US/Pacific"),
+    'PDT': gettz("US/Pacific"),
+}
+
+
+class TankReadingParser(Parser):
+    def __init__(
+            self,
+            sd_client: GravitateSDAPI,
+            source_system: str | None = None,
+            mapping_provider: Optional[AsyncMappingProvider] = None,
+            included_payload: Optional[dict] = None,
+            verbose: bool = False,
+            *args,
+            **kwargs
+    ):
+        super().__init__(source_system, mapping_provider)
+        self.sd_client = sd_client
+        self.verbose = verbose
+        self.included_payload = included_payload or {}
+        self.mapper: Optional[RitaMapper] = None
+
+    @override
+    async def parse(self, data: list[dict], mapping_type: MappingMode | None = None) -> AsyncGenerator[
+        TankReading, None]:
+        if mapping_type is None:
+            logger.warning("TankReadingParser.parse mapping_type is None, defaulting to skip")
+            mapping_type = MappingMode.skip
+        sd_store_lkp = await self.get_store_lkp()
+        self.mapper = await self.load_mapper()
+        preparsed_records = self.preparse(data, mapping_type)
+
+        for rec in preparsed_records:
+            with logger.catch(message=f"Skipped record {rec} due to error"):
+                store_id = rec.get("site_id")
+                sd_store = sd_store_lkp.get(store_id , {})
+                store_tz = sd_store.get("timezone")
+                yield TankReading(
+                    store=store_id,
+                    date=rec.get("reading_time"),
+                    monitor_type=TankMonitorType.bbd,
+                    timezone=rec.get("timezone") or store_tz,
+                    volume=rec.get("volume"),
+                    tank=rec.get("tank_id"),
+                    payload=self.included_payload
+                )
+    async def get_store_lkp(self):
+        stores = await self.sd_client.all_stores()
+        return lookup(stores.json(), lambda x: x.get("store_number"))
+
+    def preparse(self, records: list[dict], mapping_type: MappingMode) -> list:
+        """Perform basic sanity checking on records and map tank and site ids, if applicable."""
+        parsed_records = []
+        mapping_failures = []
+        for translated in records:
+            try:
+                translated_volume = translated.get("volume")
+                if translated_volume == 'nan':
+                    if self.verbose:
+                        logger.warning(f"Skipped record {translated} due to NaN volume.")
+                    continue
+                try:
+                    if isinstance(translated_volume, str):
+                        trans_vol_decimal = float(parse_decimal(translated_volume, locale="en_US"))
+                    else:
+                        trans_vol_decimal = float(translated_volume)
+
+                    translated["volume"] = trans_vol_decimal
+                except NumberFormatError:
+                    if self.verbose:
+                        logger.warning(
+                            f"Skipped record {translated} due to invalid volume value '{translated_volume}'.")
+                    continue
+                except TypeError:
+                    if self.verbose:
+                        logger.warning(
+                            f"Skipped record {translated} due to invalid volume value '{translated_volume}'."
+                        )
+                    continue
+                if trans_vol_decimal < 0:
+                    if self.verbose:
+                        logger.warning(f"Skipped record {translated} due to negative volume.")
+                    continue
+                if translated.get("tank_id") == "nan":
+                    if self.verbose:
+                        logger.warning(f"Skipped record {translated} due to NaN tank.")
+                    continue
+                if not translated.get("reading_time"):
+                    logger.warning(f"Skipped record {translated} due to missing date")
+                    continue
+                try:
+                    date_parsed = dateutil.parser.parse(translated.get("reading_time"), tzinfos=tzmapping)
+                    translated["reading_time"] = date_parsed.isoformat()
+                except Exception as parse_error:
+                    logger.warning(f"Skipped record {translated} due to date parsing error: {parse_error}")
+                    continue
+
+                if mapping_type == MappingMode.skip:
+                    parsed_records.append(translated)
+                elif mapping_type == MappingMode.partial or mapping_type == MappingMode.full:
+                    try:
+                        site_id = translated["site_id"]
+                        tank_id = translated["tank_id"]
+                        mapped_site_ids = self.mapper.get_gravitate_parent_ids(site_id, MappingType.site)
+                        mapped_tank_ids = self.mapper.get_gravitate_child_ids(site_id, tank_id.strip(), MappingType.tank)
+                        for site_id in mapped_site_ids:
+                            for tank_id in mapped_tank_ids:
+                                translated["site_id"] = site_id
+                                translated["tank_id"] = tank_id
+                                parsed_records.append(translated)
+                    except (KeyError, ValueError) as e:
+                        if mapping_type == MappingMode.partial:
+                            parsed_records.append(translated)
+                        else:
+                            raise e
+
+            except (KeyError, ValueError) as e:
+                logger.warning(f"Skipped record {translated} due to error: {e}")
+                mapping_failures.append(translated)
+        if len(mapping_failures) > 0:
+            self.record_issue(
+                key_suffix="mapping_errors",
+                name=f"Mapping errors",
+                category=IssueCategory.TANK_READING,
+                problem_short=f"{len(mapping_failures)} rows failed to map",
+                problem_long=json.dumps(mapping_failures)
+            )
+        return parsed_records

bb_integrations_lib/pipelines/parsers/tank_sales_parser.py

@@ -0,0 +1,144 @@
+from zoneinfo import ZoneInfo
+
+import dateutil
+import json
+from babel.numbers import parse_decimal, NumberFormatError
+from dateutil.parser import parse
+from dateutil.tz import gettz
+from loguru import logger
+from pandas import DataFrame
+from typing import AsyncGenerator, Optional
+from typing import override
+
+from bb_integrations_lib.mappers.rita_mapper import RitaMapper, AsyncMappingProvider
+from bb_integrations_lib.models.rita.issue import IssueCategory
+from bb_integrations_lib.models.rita.mapping import MappingType
+from bb_integrations_lib.protocols.flat_file import TankSales
+from bb_integrations_lib.protocols.pipelines import Parser
+from bb_integrations_lib.shared.model import MappingMode
+
+tzmapping = {
+    'EST': gettz("US/Eastern"),
+    'EDT': gettz("US/Eastern"),
+    'CST': gettz("US/Central"),
+    'CDT': gettz("US/Central"),
+    'MST': gettz("US/Mountain"),
+    'MDT': gettz("US/Mountain"),
+    'PST': gettz("US/Pacific"),
+    'PDT': gettz("US/Pacific"),
+}
+
+
+class SalesParser(Parser):
+    def __init__(
+            self,
+            tenant_name: str,
+            source_system: str | None = None,
+            mapping_provider: Optional[AsyncMappingProvider] = None,
+            included_payload: Optional[dict] = None,
+            verbose: bool = False,
+            deduplicate: bool = True,
+    ):
+        super().__init__(tenant_name, source_system, mapping_provider)
+        self.verbose = verbose
+        self.included_payload = included_payload or {}
+        self.mapper: Optional[RitaMapper] = None
+        self.deduplicate = deduplicate
+
+    def dedupe_records(self, records: list[dict]) -> list[dict]:
+        df = DataFrame(records)
+        grouped_df = df.groupby(["site_id", "tank_id", "date"], as_index=False)["sales"].sum()
+        grouped_df = grouped_df[["site_id", "tank_id", "date", "sales"]]
+        return grouped_df.to_dict("records")
+
+    @override
+    async def parse(self, data: list[dict], mapping_type: MappingMode | None = None) -> AsyncGenerator[TankSales, None]:
+        if mapping_type is None:
+            logger.warning("TankSalesParser.parse mapping_type is None, defaulting to skip")
+            mapping_type = MappingMode.skip
+        self.mapper = await self.load_mapper()
+        preparsed_records = self.preparse(data, mapping_type)
+        if self.deduplicate:
+            preparsed_records = self.dedupe_records(preparsed_records)
+
+        for rec in preparsed_records:
+            with logger.catch(message=f"Skipped record {rec} due to error"):
+                ts = TankSales(
+                    store_number=rec["site_id"],
+                    tank_id=rec["tank_id"],
+                    sales=float(rec["sales"]),
+                    date=parse(rec['date']).replace(tzinfo=ZoneInfo("UTC")).isoformat(),
+                )
+            yield ts
+
+    def preparse(self, records: list[dict], mapping_type: MappingMode) -> list:
+        """Perform basic sanity checking on records and map tank and site ids, if applicable."""
+        parsed_records = []
+        mapping_failures = []
+        for translated in records:
+            try:
+                translated_volume = translated.get("sales")
+                if translated_volume == 'nan':
+                    if self.verbose:
+                        logger.warning(f"Skipped record {translated} due to NaN volume.")
+                    continue
+                try:
+                    trans_vol_decimal = float(parse_decimal(translated_volume, locale="en_US"))
+                    translated["sales"] = float(trans_vol_decimal)
+                except NumberFormatError:
+                    if self.verbose:
+                        logger.warning(
+                            f"Skipped record {translated} due to invalid volume value '{translated_volume}'.")
+                    continue
+                if trans_vol_decimal < 0:
+                    if self.verbose:
+                        logger.warning(f"Skipped record {translated} due to negative volume.")
+                    continue
+                if translated.get("tank_id") == "nan":
+                    if self.verbose:
+                        logger.warning(f"Skipped record {translated} due to NaN tank.")
+                    continue
+                if not translated.get("date"):
+                    logger.warning(f"Skipped record {translated} due to missing date")
+                    continue
+                try:
+                    date_parsed = dateutil.parser.parse(translated.get("date"), tzinfos=tzmapping)
+                    translated["date"] = date_parsed.isoformat()
+                except Exception as parse_error:
+                    logger.warning(f"Skipped record {translated} due to date parsing error: {parse_error}")
+                    continue
+
+                if mapping_type == MappingMode.skip:
+                    parsed_records.append(translated)
+                elif mapping_type == MappingMode.partial or mapping_type == MappingMode.full:
+                    try:
+                        site_id = translated["site_id"]
+                        tank_id = translated["tank_id"]
+                        mapped_site_ids = self.mapper.get_gravitate_parent_ids(site_id, MappingType.site)
+                        mapped_tank_ids = self.mapper.get_gravitate_child_ids(
+                            site_id, tank_id.strip(), MappingType.tank
+                        )
+                        for site_id in mapped_site_ids:
+                            for tank_id in mapped_tank_ids:
+                                translated["site_id"] = site_id
+                                translated["tank_id"] = tank_id
+                                parsed_records.append(translated)
+                    except (KeyError, ValueError) as e:
+                        if mapping_type == MappingMode.partial:
+                            parsed_records.append(translated)
+                        else:
+                            raise e
+
+            except (KeyError, ValueError) as e:
+                if self.verbose:
+                    logger.warning(f"Skipped record {translated} due to error: {e}")
+                mapping_failures.append(translated)
+        if len(mapping_failures) > 0:
+            self.record_issue(
+                key_suffix="mapping_errors",
+                name=f"Mapping errors",
+                category=IssueCategory.TANK_READING,
+                problem_short=f"{len(mapping_failures)} rows failed to map",
+                problem_long=json.dumps(mapping_failures)
+            )
+        return parsed_records

bb_integrations_lib/pipelines/shared/allocation_matching.py

@@ -0,0 +1,227 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from bb_integrations_lib.models.sd.bols_and_drops import (
+    AllocatedBOL,
+    BOL,
+    BOLDetail,
+    Drop as ExecutedDrop,
+)
+from bb_integrations_lib.models.sd.get_order import (
+    Drop as PlannedDrop,
+    DropDetail as PlannedDropDetail,
+    Load as PlannedLoad,
+    LoadDetail as PlannedLoadDetail,
+)
+
+
+class MatchedAllocation(BaseModel):
+    """Result of matching an allocated BOL to planned and executed data."""
+
+    # Executed
+    allocated_bol: AllocatedBOL
+    executed_bol: Optional[BOL] = None
+    executed_bol_detail: Optional[BOLDetail] = None
+    executed_drop: Optional[ExecutedDrop] = None
+
+    # Planned
+    planned_load: Optional[PlannedLoad] = None
+    planned_load_detail: Optional[PlannedLoadDetail] = None
+    planned_drop: Optional[PlannedDrop] = None
+    planned_drop_detail: Optional[PlannedDropDetail] = None
+
+    # Variance
+    planned_quantity: Optional[int] = None
+    actual_quantity: int
+    variance: Optional[int] = None
+    variance_pct: Optional[float] = None
+
+    # Match status
+    matched_to_planned_drop: bool = False
+    matched_to_planned_load: bool = False
+    matched_to_executed_bol: bool = False
+    matched_to_executed_drop: bool = False
+
+
+def calculate_volume_variance(
+    actual_qty: int,
+    planned_qty: Optional[int],
+) -> tuple[Optional[int], Optional[float]]:
+    """
+    Calculate variance between actual and planned quantities.
+
+    Returns:
+        tuple of (variance, variance_pct) - both None if planned_qty is None or zero
+    """
+    if planned_qty is None or planned_qty <= 0:
+        return None, None
+    variance = actual_qty - planned_qty
+    variance_pct = (variance / planned_qty) * 100
+    return variance, variance_pct
+
+
+def match_allocated_to_planned_drop(
+    allocated_bol: AllocatedBOL,
+    planned_drops: list[PlannedDrop],
+) -> tuple[Optional[PlannedDrop], Optional[PlannedDropDetail]]:
+    """
+    Match an allocated BOL to a planned drop detail.
+
+    Match keys:
+    - allocated_bol.location_id == planned_drop.location_id
+    - allocated_bol.store_product_id == planned_drop.detail.product_id
+    - allocated_bol.store_tank == planned_drop.detail.tank_id
+    """
+    for drop in planned_drops:
+        if drop.location_id != allocated_bol.location_id:
+            continue
+        for detail in drop.details:
+            if (
+                detail.product_id == allocated_bol.store_product_id
+                and detail.tank_id == allocated_bol.store_tank
+            ):
+                return drop, detail
+    return None, None
+
+
+def match_drop_to_load(
+    drop_detail: Optional[PlannedDropDetail],
+    planned_loads: list[PlannedLoad],
+) -> tuple[Optional[PlannedLoad], Optional[PlannedLoadDetail]]:
+    """
+    Match a planned drop detail to a planned load detail using compartment_index.
+
+    The bridge: drop_detail.sources[].compartment_index -> load_detail.compartment_index
+    """
+    if drop_detail is None:
+        return None, None
+
+    compartment_indexes = {s.compartment_index for s in drop_detail.sources}
+
+    for load in planned_loads:
+        for load_detail in load.details:
+            if load_detail.compartment_index in compartment_indexes:
+                return load, load_detail
+    return None, None
+
+
+def match_allocated_to_executed_bol(
+    allocated_bol: AllocatedBOL,
+    executed_bols: list[BOL],
+) -> tuple[Optional[BOL], Optional[BOLDetail]]:
+    """
+    Match an allocated BOL to an executed BOL detail.
+
+    Match keys:
+    - allocated_bol.bol_terminal_id == executed_bol.location_id
+    - allocated_bol.bol_product_id == executed_bol.detail.product_id
+    """
+    for bol in executed_bols:
+        if bol.location_id != allocated_bol.bol_terminal_id:
+            continue
+        for detail in bol.details:
+            if detail.product_id == allocated_bol.bol_product_id:
+                return bol, detail
+    return None, None
+
+
+def match_allocated_to_executed_drop(
+    allocated_bol: AllocatedBOL,
+    executed_drops: list[ExecutedDrop],
+) -> Optional[ExecutedDrop]:
+    """
+    Match an allocated BOL to an executed drop.
+
+    Match keys:
+    - allocated_bol.location_id == executed_drop.location_id
+    - allocated_bol.store_product_id == executed_drop.product_id
+    - allocated_bol.store_tank == executed_drop.tank_id
+    """
+    for drop in executed_drops:
+        if (
+            drop.location_id == allocated_bol.location_id
+            and drop.product_id == allocated_bol.store_product_id
+            and drop.tank_id == allocated_bol.store_tank
+        ):
+            return drop
+    return None
+
+
+def match_allocations(
+    allocated_bols: list[dict],
+    executed_bols: list[dict],
+    executed_drops: list[dict],
+    planned_loads: list[dict],
+    planned_drops: list[dict],
+) -> list[MatchedAllocation]:
+    """
+    Match allocated BOLs to planned and executed data.
+
+    Args:
+        allocated_bols: List of allocated_bols from bols_and_drops response
+        executed_bols: List of bols from bols_and_drops response
+        executed_drops: List of drops from bols_and_drops response
+        planned_loads: List of loads from get_orders response
+        planned_drops: List of drops from get_orders response
+
+    Returns:
+        List of MatchedAllocation results with variance calculations
+    """
+    # Parse inputs into Pydantic models
+    parsed_allocated = [AllocatedBOL.model_validate(ab) for ab in allocated_bols]
+    parsed_executed_bols = [BOL.model_validate(b) for b in executed_bols]
+    parsed_executed_drops = [ExecutedDrop.model_validate(d) for d in executed_drops]
+    parsed_planned_loads = [PlannedLoad.model_validate(l) for l in planned_loads]
+    parsed_planned_drops = [PlannedDrop.model_validate(d) for d in planned_drops]
+
+    results: list[MatchedAllocation] = []
+
+    for allocated_bol in parsed_allocated:
+        # Step 1: Match to planned drop
+        planned_drop, planned_drop_detail = match_allocated_to_planned_drop(
+            allocated_bol, parsed_planned_drops
+        )
+
+        # Step 2: Match planned drop to planned load via compartment
+        planned_load, planned_load_detail = match_drop_to_load(
+            planned_drop_detail, parsed_planned_loads
+        )
+
+        # Step 3: Match to executed BOL
+        executed_bol, executed_bol_detail = match_allocated_to_executed_bol(
+            allocated_bol, parsed_executed_bols
+        )
+
+        # Step 4: Match to executed drop
+        executed_drop = match_allocated_to_executed_drop(
+            allocated_bol, parsed_executed_drops
+        )
+
+        # Step 5: Calculate variance
+        actual_qty = allocated_bol.bol_gross_volume_allocated
+        planned_qty = planned_drop_detail.quantity if planned_drop_detail else None
+        variance, variance_pct = calculate_volume_variance(actual_qty, planned_qty)
+
+        results.append(
+            MatchedAllocation(
+                allocated_bol=allocated_bol,
+                executed_bol=executed_bol,
+                executed_bol_detail=executed_bol_detail,
+                executed_drop=executed_drop,
+                planned_load=planned_load,
+                planned_load_detail=planned_load_detail,
+                planned_drop=planned_drop,
+                planned_drop_detail=planned_drop_detail,
+                planned_quantity=planned_qty,
+                actual_quantity=actual_qty,
+                variance=variance,
+                variance_pct=variance_pct,
+                matched_to_planned_drop=planned_drop is not None,
+                matched_to_planned_load=planned_load is not None,
+                matched_to_executed_bol=executed_bol is not None,
+                matched_to_executed_drop=executed_drop is not None,
+            )
+        )
+
+    return results