bb-integrations-library 3.0.11__py3-none-any.whl

bb_integrations_lib/pipelines/steps/processing/bbd_upload_tankreading_step.py

@@ -0,0 +1,80 @@
+import json
+from math import ceil
+from time import sleep
+from typing import List
+
+from loguru import logger
+from more_itertools import chunked
+
+from bb_integrations_lib.gravitate.sd_api import GravitateSDAPI
+from bb_integrations_lib.models.pipeline_structs import BBDUploadResult
+from bb_integrations_lib.models.rita.issue import IssueBase, IssueCategory
+from bb_integrations_lib.protocols.flat_file import TankReading
+from bb_integrations_lib.protocols.pipelines import Step
+
+
+class BBDUploadTankReadingStep(Step):
+    """
+    Takes a list of TankReading and uploads them to S&D, breaking them into chunks as required.
+
+    :param sd_client: The GravitateSDAPI client to use for uploading.
+    :param sleep_between: The number of seconds to sleep between each chunk.
+    :param chunk_size: The maximum number of readings to upload in each chunk.
+    """
+
+    def __init__(self, sd_client: GravitateSDAPI, sleep_between: float = 0.5, chunk_size: int = 1000, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.sd_client = sd_client
+        self.sleep_between = sleep_between
+        self.chunk_size = chunk_size
+
+    def describe(self) -> str:
+        return "Upload TankReadings to BBD"
+
+    async def execute(self, i: List[TankReading]) -> BBDUploadResult:
+        try:
+            total_readings = len(i)
+            count = ceil(total_readings / self.chunk_size)
+            attempted = 0
+            succeeded = 0
+            included_set = set()
+            failed = []
+            for idx, group in enumerate(chunked(i, self.chunk_size)):
+                logger.info(f"Uploading readings to bestbuy {idx + 1} of {count}")
+                attempted += len(group)
+                group = [g.model_dump(mode="json") for g in group]
+                list({json.dumps(record, sort_keys=True): record for record in group}.values())
+                try:
+                    response = await self.sd_client.upload_readings(group, raise_error=False)
+                    response = response.json()
+                    succeeded += len(response["ids"])
+                    included_set = included_set.union(set(response["ids"]))
+                    if len(response["unable to upload"]) > 0:
+                        logger.error(f"Errors occurred while uploading data: {response}")
+                        failed = failed + response["unable to upload"]
+                    sleep(self.sleep_between)
+                except Exception as e:
+                    logger.error(f"Batch {idx} readings failed | {e}")
+                    failed = failed + [g["store"] for g in group]
+                    continue
+            logger.info(f"Successfully uploaded {succeeded} of {attempted} readings.")
+            included_set = {x.split(":")[0] for x in included_set}
+
+            if len(failed) > 0:
+                logger.info(f"Failed to upload {len(failed)} of {attempted} readings")
+                fc = self.pipeline_context.file_config
+                key = f"{fc.config_id}_failed_to_upload"
+                self.pipeline_context.issues.append(IssueBase(
+                        key=key,
+                        config_id=fc.config_id,
+                        name="Failed to upload TankReadings",
+                        category=IssueCategory.TANK_READING,
+                        problem_short=f"{len(failed)} readings did not upload",
+                        problem_long=json.dumps(failed)
+                    ))
+
+            return BBDUploadResult(succeeded=succeeded, failed=attempted - succeeded,
+                                   succeeded_items=list(included_set))
+        except Exception as e:
+            logger.exception(f"Unable to upload | {e}")
+            raise e

bb_integrations_lib/pipelines/steps/processing/convert_bbd_order_to_cargas_step.py

@@ -0,0 +1,226 @@
+from datetime import datetime
+from typing import AsyncGenerator
+
+import pandas as pd
+from loguru import logger
+
+from bb_integrations_lib.gravitate.rita_api import GravitateRitaAPI
+from bb_integrations_lib.gravitate.sd_api import GravitateSDAPI
+from bb_integrations_lib.mappers.rita_mapper import RitaMapperCore, RitaMapper, RitaAPIMappingProvider
+from bb_integrations_lib.models.rita.mapping import MappingType
+from bb_integrations_lib.protocols.pipelines import GeneratorStep
+from bb_integrations_lib.provider.api.cargas.model import CreateWholesaleTicketRequestBundle, \
+    CreateWholesaleTicketRequest, CreateWholesaleLineRequest
+from bb_integrations_lib.util.config.manager import GlobalConfigManager
+
+
+class ConvertBBDOrderToCargasWholesaleStep(GeneratorStep):
+    def __init__(self, rita_client: GravitateRitaAPI, sd_client: GravitateSDAPI, mapping_source_system: str = "Cargas", *args, **kwargs):
+        """
+        Convert a BBD order's drops to Cargas wholesale ticket request objects. Input should be an item returned by the
+        v1/bols_and_drops endpoint, and should only include orders with at least 1 drop at a wholesale store. Passing an
+        order with no wholesale drops will result in a ticket request being generated with no line items.
+
+        Mappings are best-effort; if a field that is not strictly required is not mapped, it will be left out.
+
+        :param rita_client: The GravitateRitaAPI instance to use to get mappings with.
+        :param sd_client: The GravitateSDAPI instance to use for API calls.
+        :param mapping_source_system: The source system to use for mapping. Defaults to "Cargas".
+        """
+        super().__init__(*args, **kwargs)
+        self.rita_client = rita_client
+        self.sd_client = sd_client
+        self.mapping_source_system = mapping_source_system
+
+        self.store_lkp = {}
+        self.driver_sched_lkp = {}
+        # Only optional because it will be late initialized by either _load_lookups or _set_test_lookups.
+        self.rita_mapper: RitaMapper | None = None
+
+    def describe(self) -> str:
+        return f"Convert drops on a BBD order to Cargas create wholesale ticket / ticket line requests"
+
+    async def _load_lookups(self, driver_schedules: set[str]):
+        self.gcm = GlobalConfigManager()
+
+        stores = await self.sd_client.get_all_stores()
+        stores.raise_for_status()
+        self.store_lkp = {store["_id"]: store for store in stores.json()}
+
+        drivers = await self.sd_client.get_driver_tracking(driver_schedule_ids=list(driver_schedules))
+        drivers.raise_for_status()
+        self.driver_sched_lkp = {driver["driver_schedule_id"]: driver for driver in drivers.json()}
+
+        self.rita_mapper = RitaMapper(
+            provider=RitaAPIMappingProvider(self.rita_client),
+            source_system=self.mapping_source_system
+        )
+        await self.rita_mapper.load_mappings_async()
+
+    def _set_test_lookups(self, store_lkp: dict, driver_sched_lkp: dict, rita_mapper: RitaMapperCore):
+        self.store_lkp = store_lkp
+        self.driver_sched_lkp = driver_sched_lkp
+        self.rita_mapper = rita_mapper
+
+    async def generator(self, orders: list[dict]) -> AsyncGenerator[CreateWholesaleTicketRequestBundle, None]:
+        driver_schedules = set([
+            abol["driver_schedule"] for order in orders for abol in order["allocated_bols"]
+        ])
+        await self._load_lookups(driver_schedules=driver_schedules)
+
+        assert self.rita_mapper is not None, "RITA mapper must be initialized before generator is called"
+
+        for order in orders:
+            for result in self.convert_order(order):
+                yield result
+
+    def convert_order(self, order: dict) -> list[CreateWholesaleTicketRequestBundle]:
+        """
+        Converts an S&D order dictionary into one or more Cargas ticket request bundle objects, including line items.
+        Line items produced by this function will have a blank DocumentID, which should be filled in with the
+        DocumentID that CreateWholesaleTicket returns.
+        """
+        order_number = order["order_number"]
+        allocated_bols = order["allocated_bols"]
+        if not allocated_bols:
+            raise Exception("Could not convert order: no allocated BOLs")
+
+        bols_df = pd.DataFrame.from_records(allocated_bols)
+        # Precompute destination counterparty - could be 2 different stores of same counterparty on a split load
+        bols_df["store_counterparty_name"] = bols_df.apply(
+            lambda x: self.store_lkp[x["store_id"]]["counterparty_name"],
+            axis="columns"
+        )
+        bols_df["driver_name"] = bols_df.apply(
+            lambda x: self.driver_sched_lkp.get(x["driver_schedule"], {}).get("driver_name", ""),
+            axis="columns"
+        )
+        bols_df["tractor"] = bols_df.apply(
+            lambda x: self.driver_sched_lkp.get(x["driver_schedule"], {}).get("driver_log", {}).get("tractor", ""),
+            axis="columns"
+        )
+
+        results = []
+        bol_gb = bols_df.groupby(by=["bol_number", "store_counterparty_name", "driver_name", "tractor"])
+        for (bol_number, store_cp_name, driver_name, tractor), group in bol_gb:
+            mcid_str = None
+            try:
+                mcid_str = self.rita_mapper.get_source_parent_id(store_cp_name, MappingType.counterparty)
+                mapped_customer_id = int(mcid_str)
+            except KeyError as e:
+                mapped_customer_id = None
+                logger.warning(f"Failed to map store counterparty {store_cp_name}: {e}, skipping drops")
+                continue
+            except ValueError as e:
+                logger.warning(f"Failed to convert mapped customer ID '{mcid_str}' to int, skipping drops")
+                continue
+
+            try:
+                mapped_driver_id = self.rita_mapper.get_source_parent_id(driver_name, MappingType.driver)
+            except KeyError:
+                logger.warning(f"Failed to map driver ID '{driver_name}', it will be blank on the ticket")
+                mapped_driver_id = None
+            try:
+                mapped_tractor_id = self.rita_mapper.get_source_parent_id(tractor, MappingType.tractor)
+            except KeyError:
+                logger.warning(f"Failed to map tractor ID '{tractor}', it will be blank on the ticket")
+                mapped_tractor_id = None
+
+            try:
+                ticket_request = CreateWholesaleTicketRequest(
+                    CustomerID=mapped_customer_id,
+                    DeliveryDate=datetime.fromisoformat(max(group["delivered_date"])),
+                    CustomerPONumber=bol_number,
+                    DriverID=mapped_driver_id,
+                    WholesaleTruckID=mapped_tractor_id,
+                    Message="",
+                    InvoiceNotes="",
+                    CostCenterID=None,
+                    SubTypeID=None,
+                    SalespersonID=None,
+                    AdditionalNotes="",
+                    UserName="Gravitate"
+                )
+
+                line_requests = []
+                for index, bol_item in group.iterrows():
+                    try:
+                        store_number = bol_item["store_number"]
+                        tank_number = str(bol_item["store_tank"])
+                        mtid_str = None
+                        try:
+                            mtid_str = self.rita_mapper.get_source_child_id(
+                                gravitate_parent_id=store_number, gravitate_child_id=tank_number,
+                                mapping_type=MappingType.tank
+                            )
+                            mapped_tank_id = int(mtid_str)
+                        except KeyError as e:
+                            logger.warning(f"Failed to map tank '{tank_number}', skipping BOL item: {e}")
+                            continue
+                        except ValueError as e:
+                            logger.warning(
+                                f"Failed to convert mapped tank '{tank_number}'->'{mtid_str}' to int, skipping BOL item: {e}")
+                            continue
+
+                        miid_str = None
+                        bol_product = bol_item["bol_product"]
+                        try:
+                            miid_str = self.rita_mapper.get_source_parent_id(
+                                gravitate_id=bol_product, mapping_type=MappingType.product
+                            )
+                            mapped_item_id = int(miid_str)
+                        except KeyError as e:
+                            logger.warning(f"Failed to map product '{bol_product}', skipping BOL item: {e}")
+                            continue
+                        except ValueError as e:
+                            logger.warning(
+                                f"Failed to convert mapped product '{bol_product}'->'{miid_str}', skipping BOL item: {e}")
+                            continue
+
+                        # We could theoretically be handed an order with no wholesale store drops, in which case we'll
+                        # upload a wholesale ticket with no line items. However, upstream should prefilter out any
+                        # orders with no wholesale store drops.
+                        if self.store_lkp[bol_item["store_id"]].get("extra_data", {}).get("type") != "Wholesale":
+                            logger.info(
+                                f"Skipping drop at non-wholesale store '{group['store_number']}' "
+                                f"(order {order_number}, BOL {bol_number})"
+                            )
+                            continue
+
+                        line_requests.append(
+                            CreateWholesaleLineRequest(
+                                DocumentID=-1,
+                                ItemID=mapped_item_id,
+                                TankID=mapped_tank_id,
+                                Quantity=bol_item["bol_net_volume_allocated"],
+                                QuantityGross=bol_item["bol_gross_volume_allocated"],
+                                UnitPrice=bol_item.get("price", {}).get("price"),
+                                FreightRateID=None,  # TODO: Retrieve? May not be necessary
+                                VendorLocationID=None,
+                                FreightAmount=None,  # TODO: Retrieve? May not be necessary
+                                SurchargeAmount=None,
+                                UnitCostOverride=None,
+                                CustomerPricingID=None,
+                                UserName="Gravitate"
+                            )
+                        )
+                    except Exception as e:
+                        logger.warning(f"Could not convert bol item #{index} on order {order_number}: {e}")
+                        continue
+
+                if not line_requests:
+                    logger.warning(
+                        f"No line requests for order {order_number}; will upload a ticket request but it will be empty")
+                results.append(CreateWholesaleTicketRequestBundle(
+                    ticket_request=ticket_request,
+                    line_requests=line_requests
+                ))
+
+            except Exception as e:
+                logger.warning(f"Could not convert order {order_number}, BOL {bol_number}: {e}")
+                continue
+
+        return results
+
+    def get_store_cp_name(self, store_id: str) -> str:
+        return self.store_lkp[store_id]["counterparty_name"]

bb_integrations_lib/pipelines/steps/processing/delete_sftp_step.py

@@ -0,0 +1,33 @@
+import os
+from typing import Dict, Any
+
+import loguru
+
+from bb_integrations_lib.protocols.pipelines import Step
+from bb_integrations_lib.provider.ftp.client import FTPIntegrationClient
+from bb_integrations_lib.shared.model import FileConfigRawData, RawData
+
+
+class DeleteSFTPStep(Step):
+    def __init__(self, ftp_client: FTPIntegrationClient, src_directory: str | None = None, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.ftp_client = ftp_client
+        self.src_directory = src_directory
+
+    def describe(self) -> str:
+        return "SFTP Delete Step"
+
+    async def execute(self, i: Any) -> Any:
+        if isinstance(i, FileConfigRawData):
+            filename = os.path.join(i.file_config.inbound_directory, i.file_name)
+        elif isinstance(i, RawData):
+            if self.src_directory is None:
+                raise RuntimeError("Attempted to delete a RawData object but src_directory was not set.")
+            filename = os.path.join(self.src_directory, i.file_name)
+        else:
+            raise NotImplementedError(f"Unsupported input type: {type(i)}")
+        try:
+            self.ftp_client.delete_file(filename)
+        except Exception as e:
+            loguru.logger.warning(f"Failed to delete: {e}")
+        return i

bb_integrations_lib/pipelines/steps/processing/dtn/__init__.py

@@ -0,0 +1,2 @@
+from .convert_dtn_invoice_to_sd_model import ConvertDTNInvoiceToSDModel
+from .parse_dtn_invoice_step import ParseDTNInvoiceStep

bb_integrations_lib/pipelines/steps/processing/dtn/convert_dtn_invoice_to_sd_model.py

@@ -0,0 +1,145 @@
+import itertools
+from datetime import datetime
+from operator import attrgetter
+from typing import Callable
+
+from bb_integrations_lib.models.dtn_supplier_invoice import DTNSupplierInvoice, Item, ItemTax, SummaryTax, \
+    DeferredTaxItem
+from bb_integrations_lib.protocols.pipelines import Step
+from bb_integrations_lib.shared.model import SDSupplierInvoiceCreateRequest, SDSupplierInvoiceDetail, \
+    SDSupplierInvoiceDetailType
+
+
+class ConvertDTNInvoiceToSDModel(Step):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def describe(self) -> str:
+        return "Convert DTN invoice data model to SD supplier invoice data model"
+
+    async def execute(self, i: DTNSupplierInvoice) -> SDSupplierInvoiceCreateRequest:
+        return self.convert_dtn_invoice(i)
+
+    def convert_dtn_invoice_deferred_tax_item(self, deferred_tax: DeferredTaxItem) -> SDSupplierInvoiceDetail:
+        return SDSupplierInvoiceDetail(
+            bol_number=None, # Not present for summary taxes, since they're invoice level
+            bol_date_local=None,
+            type=SDSupplierInvoiceDetailType.tax,
+            product=None,
+            tax_type=None,
+            tax_description=deferred_tax.description,
+            tax_authority=None,
+            tax_non_deferred=False,
+            rate=deferred_tax.amount,
+            amount=1,
+            total=deferred_tax.amount,
+            uom=None,
+            gross_volume=None,
+            net_volume=None,
+        )
+
+    def convert_dtn_invoice_summary_tax(self, summary_tax: SummaryTax) -> SDSupplierInvoiceDetail:
+        return SDSupplierInvoiceDetail(
+            bol_number=None, # Not present for summary taxes, since they're invoice level
+            bol_date_local=None,
+            type=SDSupplierInvoiceDetailType.tax,
+            product=None,
+            tax_type=summary_tax.tax_code,
+            tax_description=summary_tax.description,
+            tax_authority=None,
+            tax_non_deferred=not summary_tax.deferred,
+            rate=summary_tax.rate,
+            amount=summary_tax.quantity_billed,
+            total=summary_tax.line_total,
+            uom=summary_tax.unit_of_measure,
+            gross_volume=None,
+            net_volume=None,
+        )
+
+    def convert_dtn_invoice_item_tax(
+            self, item_tax: ItemTax, item_product: str, bol_date: datetime | None
+    ) -> SDSupplierInvoiceDetail:
+        return SDSupplierInvoiceDetail(
+            bol_number=item_tax.bol_number,
+            bol_date_local=bol_date,
+            type=SDSupplierInvoiceDetailType.tax,
+            product={"source_name": item_product},
+            tax_type=item_tax.tax_code,
+            tax_description=item_tax.description,
+            tax_authority=None,
+            tax_non_deferred=not item_tax.deferred,
+            rate=item_tax.rate,
+            amount=item_tax.quantity_billed,
+            total=item_tax.line_total,
+            uom=item_tax.unit_of_measure,
+            gross_volume=item_tax.quantity_billed,
+            net_volume=item_tax.quantity_billed,
+        )
+
+    def convert_dtn_invoice_item(self, invoice_detail: Item) -> list[SDSupplierInvoiceDetail]:
+        product = invoice_detail.description
+        bol_date = invoice_detail.ship_datetime
+        item_detail = SDSupplierInvoiceDetail(
+            bol_number=invoice_detail.bol_number,
+            bol_date_local=bol_date,
+            type=SDSupplierInvoiceDetailType.supply,
+            product={"source_name": product},
+            rate=invoice_detail.rate,
+            amount=invoice_detail.quantity_billed,
+            total=invoice_detail.line_total,
+            uom=invoice_detail.unit_of_measure,
+            gross_volume=invoice_detail.gross_quantity or invoice_detail.quantity_billed,
+            net_volume=invoice_detail.net_quantity or invoice_detail.quantity_billed
+        )
+        tax_details = [
+            self.convert_dtn_invoice_item_tax(tax, product, bol_date) for tax in invoice_detail.tax_records or []
+        ]
+        # Should we support RINS numbers here? How?
+        return [item_detail] + tax_details
+
+    @staticmethod
+    def resolve_field(invoice: DTNSupplierInvoice, attr_getter: Callable[[Item], str]) -> str | None:
+        """
+        Resolve a specific field from the invoice line items, if they are all the same.
+         :return: The return value retrieved by calling attr_getter on each invoice item, if all are the same (or if
+           only one item is present), or None if no items are present or if any items disagree with each other.
+        """
+        # Get a list of terminals from the invoice line items, where the terminal is not blank
+        item_fields = list(map(attr_getter, invoice.items))
+        if len(set(item_fields)) == 1:
+            return item_fields[0]
+        return None
+
+    def convert_dtn_invoice(self, invoice: DTNSupplierInvoice) -> SDSupplierInvoiceCreateRequest:
+        deferred_tax_details = []
+        summary_tax_details = []
+        if invoice.deferred_taxes is not None:
+            deferred_tax_details.extend([
+                self.convert_dtn_invoice_deferred_tax_item(detail_item)
+                for detail_item in invoice.deferred_taxes.items
+            ])
+        for summary_tax in invoice.summary_taxes:
+            summary_tax_details.append(
+                self.convert_dtn_invoice_summary_tax(summary_tax)
+            )
+        return SDSupplierInvoiceCreateRequest(
+            invoice_number=invoice.header.invoice_number,
+            source_name=invoice.header.sold_to_name,
+            supplier=invoice.header.seller_name,
+            terminal=self.resolve_field(invoice, attrgetter("ship_from_name")) or None,
+            due_date_local=invoice.header.invoice_due_date,
+            # Convert date to datetime at midnight
+            invoice_date_local=datetime.combine(invoice.header.invoice_date, datetime.min.time()),
+            # Get a flattened list, since each call to convert_dtn_invoice_item yields a list of 1 or more detail items
+            # Then tack on summary and deferred tax details, when applicable.
+            details=list(itertools.chain.from_iterable(
+                [self.convert_dtn_invoice_item(item) for item in invoice.items]
+            )) + summary_tax_details + deferred_tax_details,
+            extra_data={
+                "invoice_filename": invoice.original_filename
+            } if invoice.original_filename else {},
+            ship_to_city=self.resolve_field(invoice, attrgetter("ship_to_city")) or None,
+            ship_to_state=self.resolve_field(invoice, attrgetter("ship_to_state")) or None,
+            ship_from_city=self.resolve_field(invoice, attrgetter("ship_from_city")) or None,
+            ship_from_state=self.resolve_field(invoice, attrgetter("ship_from_state")) or None,
+        )

bb_integrations_lib/pipelines/steps/processing/dtn/parse_dtn_invoice_step.py

@@ -0,0 +1,38 @@
+from io import TextIOWrapper
+from loguru import logger
+from typing import Dict, AsyncIterable
+
+from bb_integrations_lib.models.dtn_supplier_invoice import DTNSupplierInvoice, Parser
+from bb_integrations_lib.protocols.pipelines import GeneratorStep
+from bb_integrations_lib.shared.model import RawData
+
+
+class ParseDTNInvoiceStep(GeneratorStep):
+    def __init__(self, skip_failed: bool = True, *args, **kwargs) -> None:
+        """
+        Parse a DTN invoice and convert it into one or more modeled invoice items.
+
+        :param skip_failed: Whether to skip any invoices in this file that fail to parse.
+        """
+        super().__init__(*args, **kwargs)
+        self.skip_failed = skip_failed
+
+    def describe(self) -> str:
+        return "Parse a DTN supplier invoice file into one or more invoice models"
+
+    async def generator(self, i: RawData) -> AsyncIterable[DTNSupplierInvoice]:
+        try:
+            p = Parser()
+            tio = TextIOWrapper(i.data, encoding="utf-8")
+            tio.seek(0)
+            res = p.parse(tio)
+            tio.detach() # Don't allow the CSV module to close the underlying BytesIO in case another step wants to use it
+            for invoice in res:
+                invoice.original_filename = i.file_name
+                yield invoice
+        except Exception as e:
+            if self.skip_failed:
+                logger.exception(f"Failed to process file {i.file_name}, skipping")
+                return
+            else:
+                raise e