bb-integrations-library 3.0.11__py3-none-any.whl

bb_integrations_lib/pipelines/steps/processing/bbd_format_tank_readings_step.py

@@ -0,0 +1,492 @@
+import json
+from datetime import datetime, UTC, timedelta
+from typing import Dict, Any, Tuple, Iterable
+
+import polars as pl
+import pytz
+from loguru import logger
+
+from bb_integrations_lib.models.pipeline_structs import StopPipeline
+from dateutil.parser import parse
+
+from bb_integrations_lib.protocols.pipelines import Step
+from bb_integrations_lib.shared.exceptions import StepInitializationError
+from bb_integrations_lib.shared.model import RawData, FileFormat
+
+
+class ReadingParser:
+    """
+    Parser for tank reading data that supports multiple output formats.
+
+    This class provides parsing capabilities for tank reading data into different
+    client-specific formats, including standard PDI-compatible output and various
+    Circle K formats for different integration systems.
+
+    The parser takes preprocessed DataFrame input (with standardized column names)
+    and transforms it according to the specified format requirements.
+
+    Supported formats:
+        - standard: PDI-compatible format with configurable disconnection tracking
+        - circlek: Circle K format with TelaPoint integration fields
+        - circlek2: Simplified Circle K format for Gravitate system integration
+    """
+
+    def __init__(self, file_format: FileFormat):
+        """
+        Initialize the parser with a specific output format.
+
+        Args:
+            file_format (FileFormat): The desired output format for parsed data
+        """
+        self.format = file_format
+
+    def parse(self, df: pl.DataFrame, disconnected_column: bool = False,
+              disconnected_only: bool = False, water_level_column: bool = False) -> pl.DataFrame:
+        """
+        Parse tank reading data according to the configured format.
+
+        Args:
+            df (DataFrame): Input DataFrame with standardized column names:
+                - Store Number: Store identifier
+                - Name: Store name
+                - Tank Id: Tank identifier
+                - Tank Product: Product type in tank
+                - Carrier: Carrier information
+                - Volume: Current volume measurement
+                - Ullage: Unfilled space in tank
+                - Read Time: Timestamp of reading
+                - Store Source Number: Store number assigned by client  (extra_data.site_source_number)
+                - Disconnected: Boolean disconnection status (optional)
+            disconnected_column (bool): Whether to include Disconnected column
+                in output (standard format only)
+            disconnected_only (bool): Whether to filter to only disconnected
+                tanks (standard format only)
+            water_level_column (bool): Whether to include Water Level column
+
+        Returns:
+            DataFrame: Parsed data in the specified format
+
+        Raises:
+            ValueError: If the configured format is not supported
+        """
+        if self.format == FileFormat.standard:
+            return self._parse_standard(df, disconnected_column, disconnected_only, water_level_column)
+        elif self.format == FileFormat.circlek:
+            return self._parse_circlek(df)
+        elif self.format == FileFormat.circlek2:
+            return self._parse_circlek2(df)
+        elif self.format == FileFormat.reduced:
+            return self._parse_reduced(df)
+        else:
+            raise ValueError(f"Unsupported format: {self.format}")
+
+    def _parse_standard(self, df: pl.DataFrame, disconnected_column: bool,
+                        disconnected_only: bool, water_level_column: bool) -> pl.DataFrame:
+        """
+        Parse data into standard PDI-compatible format.
+
+        Produces columns: Store Number, Name, Tank Id, Tank Product, Carrier,
+        Volume, Ullage, Read Time, and optionally Disconnected.
+        """
+        column_order = [
+            'Store Number', 'Name', 'Tank Id', 'Tank Product',
+            'Carrier', 'Volume', 'Ullage', 'Read Time'
+        ]
+
+        if water_level_column:
+            column_order.append('Water')
+
+        if disconnected_column:
+            column_order.append('Disconnected')
+
+        if disconnected_only:
+            df = df.filter(pl.col("Disconnected") == True)
+
+        return df.select(column_order)
+
+    def _parse_circlek(self, df: pl.DataFrame) -> pl.DataFrame:
+        """
+        Parse data into Circle K format with TelaPoint integration fields.
+
+        Transforms input data into Circle K's expected structure with TelaPoint
+        account/site numbers and formatted timestamps.
+        """
+
+        def parse_date(dt: str) -> str:
+            dt = parse(dt)
+            return dt.strftime("%m/%d/%Y %H:%M")
+
+        rows = []
+        records = df.to_dicts()
+
+        for record in records:
+            rows.append({
+                "ClientName": None,
+                "FacilityName": None,
+                "FacilityInternalID": None,
+                "FacilityState": None,
+                "VolumePercentage": None,
+                "TankStatus": None,
+                "TankNbr": None,
+                "TankInternalID": None,
+                "AtgTankNumber": record['Tank Id'],
+                "ATGTankLabel": None,
+                "Product": None,
+                "TankCapacity": None,
+                "Ullage": None,
+                "SafeUllage": None,
+                "Volume": record['Volume'],
+                "Height": None,
+                "Water": None,
+                "Temperature": None,
+                "InventoryDate": parse_date(record['Read Time']),
+                "SystemUnits": None,
+                "CollectionDateTimeUtc": None,
+                "TelaPointAccountNumber": 100814,
+                "TelaPointSiteNumber": record['Store Number'],
+            })
+
+        return pl.DataFrame(rows)
+
+    def _parse_circlek2(self, df: pl.DataFrame) -> pl.DataFrame:
+        """
+        Parse data into simplified Circle K format.
+        """
+
+        def parse_date(dt: str) -> str:
+            dt = parse(dt)
+            return dt.strftime("%m/%d/%Y %H:%M")
+
+        rows = []
+        records = df.to_dicts()
+
+        for record in records:
+            rows.append({
+                "storeNumber": record.get('Store Source Number'),
+                "timestamp": parse_date(record['Read Time']),
+                "tankLabel": record.get('Tank Product'),  # Product name assigned to tank
+                "volume": record['Volume'],
+                "tankNumber": record['Tank Id'],
+                "ullage": record.get('Ullage', 0),
+                "productLevel": 0,  # Can be set to 0 as specified
+                "waterLevel": 0,  # Can be set to 0 as specified
+                "temperature": 0  # Can be set to 0 as specified
+            })
+
+    def _parse_reduced(self, df: pl.DataFrame) -> pl.DataFrame:
+        """
+        Parse data into reduced format with minimal columns.
+        """
+        df = df.with_columns([
+            (pl.col('Read Time')
+            .str.replace(" UTC", "")
+            .str.to_datetime(format="%Y-%m-%d %H:%M:%S %z")
+            .dt.convert_time_zone("UTC")
+            .dt.strftime("%Y-%m-%dT%H:%M:%S.%3f")
+            .str.replace(r"\.(\d{3})\d*$", ".$1")
+            + pl.lit("Z"))
+            .alias('read_time')
+        ])
+
+        return df.select([
+            pl.col('Store Number').alias('store_number'),
+            pl.col('Tank Id').alias('tank_id'),
+            pl.col('read_time'),
+            pl.col('Volume').alias('volume')
+        ])
+
+    @classmethod
+    def create_parser(cls, file_format: FileFormat) -> 'ReadingParser':
+        """
+        Factory method to create a parser for the specified format.
+
+        Args:
+            file_format (FileFormat): The desired output format
+
+        Returns:
+            ReadingParser: Configured parser instance
+        """
+        return cls(file_format)
+
+
+class ParseTankReadingsStep(Step):
+    def __init__(self, file_format: FileFormat, timezone: str, include_water_level: bool = False,
+                 disconnected_column: bool = False, disconnected_only: bool = False,
+                 disconnected_hours_threshold: float | None = None,
+                 *args, **kwargs):
+        """
+        Parse tank readings from BBDExportReadingsStep and create a PDI-compatible file, either in the standard output
+        format (with some configuration options) or in a client-specific format.
+
+        See the ``FileFormat`` enum for currently supported formats.
+
+        :param file_format: The ``FileFormat`` to use for the output file. "standard" provides a PDI compatible file,
+          but additional formats may be implemented at client request.
+        :param timezone: Timezone to localize read times to. Must be a Pytz-known timezone name.
+        :param include_water_level: Whether to include water level in the output file. Defaults to False.
+        :param disconnected_column: Whether to include a "Disconnected" column in the output file. Independent of
+          disconnected_only. Requires ``disconnected_hours_threshold`` to be set.
+        :param disconnected_only: Whether to post-filter result rows to only disconnected site/tanks. Independent of
+          disconnected_column. Requires ``disconnected_hours_threshold`` to be set.
+        :param disconnected_hours_threshold: How long it may be since the last reading before a tank is considered
+          disconnected. Setting this value without ``disconnected_column`` or ``disconnected_only`` will have no effect.
+        """
+        super().__init__(*args, **kwargs)
+        self.file_format: FileFormat = file_format
+        self.timezone = timezone
+        self.step_created_time = datetime.now(UTC)
+        self.include_water_level = include_water_level
+        self.disconnected_column = disconnected_column
+        self.disconnected_only = disconnected_only
+        self.disconnected_hours_threshold = disconnected_hours_threshold
+        self.disconnected_threshold = timedelta(
+            hours=self.disconnected_hours_threshold) if self.disconnected_hours_threshold else None
+
+        # Initialize the reading parser for the configured format
+        self.reading_parser = ReadingParser(self.file_format)
+
+        if (self.disconnected_column or self.disconnected_only) and not self.disconnected_hours_threshold:
+            raise StepInitializationError(
+                "If disconnected_column or disconnected_only is True, disconnected_hours_threshold must be set")
+
+    def describe(self) -> str:
+        return f"Format tank readings step"
+
+    async def execute(self, data: Tuple[Dict, Dict, Iterable]) -> pl.DataFrame:
+        store_lkp, tank_lkp, readings = data
+        df = pl.LazyFrame(readings, schema={
+            "tank_agent_name": str,
+            "store_number": str,
+            "run_time": datetime,
+            "tank_id": str,
+            "read_time": datetime,
+            "product": str,
+            "monitor_type": str,
+            "volume": float,
+        })
+        return (await self.parse_data(data=df, tank_lkp=tank_lkp, store_lkp=store_lkp)).collect()
+
+    @staticmethod
+    def safe_expand_extra_data(df: pl.DataFrame, extra_data_col='extra_data') -> pl.DataFrame:
+        if extra_data_col not in df.columns:
+            logger.warning(f"Warning: {extra_data_col} column not found")
+            return df
+        if df[extra_data_col].is_null().all():
+            logger.warning(f"Warning: All {extra_data_col} values are null")
+            return df.drop(extra_data_col)
+        try:
+            col_dtype = df[extra_data_col].dtype
+            if str(col_dtype).startswith("Struct"):
+                df = df.unnest(extra_data_col)
+            else:
+                df = df.with_columns([
+                    pl.col(extra_data_col).map_elements(
+                        lambda x: x if isinstance(x, dict) else (json.loads(x) if isinstance(x, str) else {}),
+                        return_dtype=pl.Struct
+                    ).alias(f"{extra_data_col}_parsed")
+                ])
+                df = df.unnest(f"{extra_data_col}_parsed")
+                df = df.drop(extra_data_col)
+            return df
+        except Exception as e:
+            logger.error(f"Error expanding {extra_data_col}: {e}")
+            return df
+
+    def maybe_add_water_level(self, df: pl.DataFrame, columns_to_keep: list) -> pl.DataFrame:
+        if self.include_water_level:
+            columns_to_keep.append('water')
+            df = df.with_columns([
+                pl.col('water').fill_nan(None).alias('water')
+            ])
+        return df
+
+    def map_stores_to_gravitate_name(self, df: pl.DataFrame, store_lkp: Dict) -> pl.DataFrame:
+        store_name_map = {k: v.get('name') if v else None for k, v in store_lkp.items()}
+        store_source_number_map = {k: v.get('extra_data', {}).get('site_source_number') if v else None
+                                   for k, v in store_lkp.items()}
+
+        df = df.with_columns([
+            pl.col('store_number').replace(store_name_map, default=None).alias('name'),
+            pl.col('store_number').replace(store_source_number_map, default=None).alias('store_source_number')
+        ])
+        return df
+
+    def select_keep_columns(self, df: pl.DataFrame, columns_to_keep: list) -> pl.DataFrame:
+        return df.select(columns_to_keep)
+
+    def localize_timestamps(self, df: pl.DataFrame) -> pl.DataFrame:
+        df = df.with_columns([
+            pl.col('read_time')
+            .dt.replace_time_zone("UTC")
+            .dt.convert_time_zone(self.timezone)
+            .alias('read_time')
+        ])
+        return df
+
+    def maybe_calculate_disconnected_tanks(self, df: pl.DataFrame) -> pl.DataFrame:
+        """
+        Optionally calculate disconnected status for tanks.
+
+        Groups by (store_number, tank_id) and determines if tank is disconnected
+        based on reading timestamps and configured threshold.
+
+        Args:
+            df: Input DataFrame with read_time column
+
+        Returns:
+            DataFrame with optional 'disconnected' column added
+        """
+        if self.disconnected_column or self.disconnected_only:
+            disconnections = df.group_by(['store_number', 'tank_id']).agg([
+                pl.col('read_time').map_elements(
+                    lambda times: self.is_disconnected(times, self.disconnected_threshold),
+                    return_dtype=pl.Boolean
+                ).first().alias('disconnected')
+            ])
+            df = df.join(disconnections, on=['store_number', 'tank_id'], how='left')
+        return df
+
+    def format_timestamps(self, df: pl.DataFrame) -> pl.DataFrame:
+        """
+        Format read_time datetime to string using vectorized strftime.
+
+        Converts to "YYYY-MM-DD HH:MM:SS TZ±HHMM" format.
+
+        Args:
+            df: Input DataFrame with datetime read_time column
+
+        Returns:
+            DataFrame with string-formatted read_time
+        """
+        df = df.with_columns([
+            pl.col('read_time').dt.strftime("%Y-%m-%d %H:%M:%S %Z%z").alias('read_time')
+        ])
+        return df
+
+    def create_tank_lookup_key(self, df: pl.DataFrame) -> pl.DataFrame:
+        df = df.with_columns([
+            (pl.col('store_number').cast(pl.Utf8) + ':' + pl.col('tank_id').cast(pl.Utf8)).alias('key')
+        ])
+        return df
+
+    def map_tank_metadata(self, df: pl.DataFrame, tank_lkp: Dict) -> pl.DataFrame:
+        """
+        Map tank metadata (product, carrier, storage_max) using composite key.
+        """
+        tank_product_map = {k: v.get('product') if v else None for k, v in tank_lkp.items()}
+        tank_carrier_map = {k: v.get('carrier') if v else None for k, v in tank_lkp.items()}
+        tank_storage_max_map = {k: v.get('storage_max') if v else None for k, v in tank_lkp.items()}
+        df = df.with_columns([
+            pl.col('key').replace(tank_product_map, default=None).alias('tank_product'),
+            pl.col('key').replace(tank_carrier_map, default=None).alias('carrier'),
+            pl.col('key').replace(tank_storage_max_map, default=None).alias('storage_max')
+        ])
+        return df
+
+    def calculate_ullage(self, df: pl.DataFrame) -> pl.DataFrame:
+        """
+        Formula: ullage = storage_max - volume
+        """
+        df = df.with_columns([
+            (pl.col('storage_max').fill_null(0) - pl.col('volume').fill_null(0)).alias('ullage')
+        ])
+        return df
+
+    def map_tanks_to_gravitate_id(self, df: pl.DataFrame, tank_lkp: Dict) -> pl.DataFrame:
+        df = self.create_tank_lookup_key(df)
+        df = self.map_tank_metadata(df, tank_lkp)
+        df = self.calculate_ullage(df)
+        return df
+
+    def format_column_names(self, df: pl.DataFrame) -> pl.DataFrame:
+        column_mapping = {col: ParseTankReadingsStep.format_column_name(col) for col in df.columns}
+        return df.rename(column_mapping)
+
+    def add_reference_data(self, df: pl.DataFrame, tank_lkp: Dict, store_lkp: Dict) -> pl.DataFrame:
+        columns_to_keep = ['store_number', 'tank_id', 'read_time', 'volume', 'name', 'store_source_number']
+        df = self.maybe_add_water_level(df, columns_to_keep)
+        df = self.map_stores_to_gravitate_name(df, store_lkp)
+        df = self.select_keep_columns(df, columns_to_keep)
+        df = self.localize_timestamps(df)
+        df = self.maybe_calculate_disconnected_tanks(df)
+        df = self.format_timestamps(df)
+        df = self.map_tanks_to_gravitate_id(df, tank_lkp)
+        df = self.format_column_names(df)
+
+        return df
+
+    def file_parser(self, df: pl.DataFrame, tank_lkp: Dict, store_lkp: Dict) -> pl.DataFrame:
+        """
+        Parse tank reading data into the configured output format.
+
+        This method first enriches the data with reference information, then uses
+        the ReadingParser to transform it into the appropriate format.
+
+        Args:
+            df (DataFrame): Raw tank readings DataFrame
+            tank_lkp (Dict): Tank lookup dictionary
+            store_lkp (Dict): Store lookup dictionary
+
+        Returns:
+            DataFrame: Parsed DataFrame in the configured output format
+        """
+        df = self.add_reference_data(df, tank_lkp, store_lkp)
+        return self.reading_parser.parse(
+            df,
+            disconnected_column=self.disconnected_column,
+            disconnected_only=self.disconnected_only,
+            water_level_column=self.include_water_level
+        )
+
+    def localize(self, dt: datetime, timezone: str = None) -> datetime:
+
+        if timezone is None:
+            timezone = self.timezone
+        utc = pytz.timezone('UTC')
+        dt = utc.localize(dt)
+        dt = dt.astimezone(pytz.timezone(timezone))
+        return dt
+
+    def format_dt_col(self, dt: datetime) -> str:
+        return dt.strftime("%Y-%m-%d %H:%M:%S %Z%z")
+
+    def is_disconnected(self, reading_times: Iterable[datetime], threshold: timedelta) -> bool:
+        """
+        Determine if a tank is disconnected based on reading timestamps.
+
+        A tank is considered disconnected if it has no readings within the threshold
+        period from the current time, or if it has no readings at all.
+
+        Args:
+            reading_times (Iterable[datetime]): Collection of reading timestamps
+            threshold (timedelta): Time threshold for disconnection detection
+
+        Returns:
+            bool: True if tank is disconnected, False otherwise
+        """
+        # Skip future times, with a 15 minute grace period (maybe the clocks are just slightly desynced)
+        filtered = [t for t in reading_times if t <= self.step_created_time + timedelta(minutes=15)]
+        # No readings, or a reading older than now - threshold? Disconnected
+        return len(filtered) == 0 or max(filtered) < datetime.now(UTC) - threshold
+
+    @staticmethod
+    def format_column_name(col_name: str) -> str:
+        return ' '.join(word.capitalize() for word in col_name.split('_'))
+
+    async def parse_data(self, data: pl.DataFrame, store_lkp: Dict, tank_lkp: Dict) -> pl.DataFrame:
+        """
+        Parse tank reading data and return as RawData for pipeline output.
+
+        Args:
+            data (DataFrame): Raw tank readings DataFrame
+            store_lkp (Dict): Store lookup dictionary
+            tank_lkp (Dict): Tank lookup dictionary
+
+        Returns:
+            DataFrame: Parsed data output in the requested format. An additional step is required to export this data
+            to a RawData object.
+        """
+        df = self.file_parser(df=data, tank_lkp=tank_lkp, store_lkp=store_lkp)
+        # self.pipeline_context.included_files["parse pricing engine prices to PDI file step"] = json.dumps(
+        #     df.to_dicts())
+        return df

bb_integrations_lib/pipelines/steps/processing/bbd_upload_prices_step.py

@@ -0,0 +1,54 @@
+import json
+from math import ceil
+from time import sleep
+from typing import Iterable, Union
+
+from loguru import logger
+from more_itertools import chunked
+
+from bb_integrations_lib.gravitate.sd_api import GravitateSDAPI
+from bb_integrations_lib.protocols.flat_file import PriceRow
+from bb_integrations_lib.protocols.pipelines import Step
+from bb_integrations_lib.shared.model import SupplyPriceUpdateManyRequest
+
+
+class BBDUploadPricesStep(Step):
+    def __init__(self, sd_client: GravitateSDAPI, sleep_between: float = 0.5, chunk_size: int = 1000, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.sd_client = sd_client
+        self.sleep_between = sleep_between
+        self.chunk_size = chunk_size
+
+    def describe(self) -> str:
+        return "Upload prices to BBD"
+
+    async def execute(self, i: Union[Iterable[PriceRow], Iterable[SupplyPriceUpdateManyRequest]]) -> int:
+        total_prices = len(i)
+        count = ceil(total_prices / 1000)
+        attempted = 0
+        succeeded = 0
+        responses = []
+        price_dump = i.model_dump(mode="json")
+        list({json.dumps(record, sort_keys=True): record for record in price_dump}.values())
+        for idx, group in enumerate(chunked(i, self.price_dump)):
+
+            
+            logger.info(f"Uploading prices to bestbuy {idx + 1} of {count}")
+            sleep(self.sleep_between)
+            attempted += len(group)
+            group = [g.model_dump(mode="json") for g in group]
+            try:
+                successes, response = await self.sd_client.upload_prices(group)
+                succeeded += successes
+                responses.append(response)
+            except Exception as e:
+                logger.error(f"Batch {idx} prices failed | {e}")
+                continue
+        logger.info(f"Successfully uploaded {succeeded} prices to BBD.")
+        logs = {
+            "response": responses,
+            "attempted": attempted,
+            "succeeded": succeeded
+        }
+        self.pipeline_context.included_files["upload prices to sd"] = json.dumps(logs)
+        return succeeded

bb_integrations_lib/pipelines/steps/processing/bbd_upload_tank_sales_step.py

@@ -0,0 +1,124 @@
+import json
+from math import ceil
+from time import sleep
+from typing import Dict, List
+
+from loguru import logger
+from more_itertools import chunked
+
+from bb_integrations_lib.gravitate.sd_api import GravitateSDAPI
+from bb_integrations_lib.models.pipeline_structs import BBDUploadResult
+from bb_integrations_lib.models.rita.issue import IssueBase, IssueCategory
+from bb_integrations_lib.protocols.pipelines import Step
+from bb_integrations_lib.protocols.flat_file import TankSales
+from bb_integrations_lib.util.utils import CustomJSONEncoder
+from bb_integrations_lib.util.config.manager import GlobalConfigManager
+from bb_integrations_lib.util.config.model import GlobalConfig
+
+
+class BBDUploadTankSalesStep(Step):
+    """
+    Takes a list of TankSales and uploads them to Best Buy
+    """
+
+    def __init__(self, sd_client: GravitateSDAPI, sleep_between: float = 0.5, chunk_size: int = 1000, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.sd_client = sd_client
+        self.sleep_between = sleep_between
+        self.chunk_size = chunk_size
+
+    def describe(self) -> str:
+        return "Upload Tanksales to BBD"
+
+    async def execute(self, i: List[TankSales]) -> BBDUploadResult:
+        logs = {"requests": [], "responses": [], "errors": []}
+        try:
+            total_sales = len(i)
+            count = ceil(total_sales / self.chunk_size)
+            attempted = 0
+            succeeded = 0
+            failed_items = []
+            store_ids = []
+
+            for idx, group in enumerate(chunked(i, self.chunk_size)):
+                logger.info(f"Uploading sales to bestbuy {idx + 1} of {count} to: {self.sd_client.base_url}  ")
+                attempted += len(group)
+                serialized_group = [g.model_dump(mode="json") for g in group]
+                batch_store_ids = [g.get("store_number", "unknown") for g in serialized_group]
+                logs["requests"].append({
+                    "row_id": idx,
+                    "request": serialized_group,
+                    "store_ids": batch_store_ids
+                })
+
+                try:
+                    response: Dict = await self.sd_client.upload_tank_sales(serialized_group)
+                    response_data = response
+                    logs["responses"].append({
+                        "row_id": idx,
+                        "response": response_data,
+                    })
+                    created = response_data.get("created", 0)
+                    updated = response_data.get("updated", 0)
+                    failed = response_data.get("failed", [])
+                    current_succeeded = created + updated
+                    succeeded += current_succeeded
+                    if current_succeeded > 0:
+                        if current_succeeded == len(serialized_group) and not failed:
+                            store_ids.extend(batch_store_ids)
+                        else:
+                            failed_store_numbers = [f["record"]["store_number"] for f in failed if
+                                                    "record" in f and "store_number" in f["record"]]
+                            failed_set = set(failed_store_numbers)
+                            successful_ids = [
+                                store_id for store_id in batch_store_ids
+                                if store_id not in failed_set
+                            ]
+                            store_ids.extend(successful_ids)
+                    if failed:
+                        failed_items.extend(failed)
+                        logs["errors"].append({
+                            "row_id": idx,
+                            "failed_items": failed,
+                            "response": response_data
+                        })
+                        logger.error(f"Errors occurred while uploading data: {failed}")
+                    logger.info(f"Batch {idx + 1}: Created {created}, Updated {updated}, Failed {len(failed)}")
+                    sleep(self.sleep_between)
+                except Exception as e:
+                    error_msg = f"Batch {idx} sales failed | {e}"
+                    logger.error(error_msg)
+                    failed_items.extend(batch_store_ids)
+                    logs["errors"].append({
+                        "row_id": idx,
+                        "exception": str(e),
+                        "store_ids": batch_store_ids
+                    })
+                    continue
+
+            logger.info(f"Successfully uploaded {succeeded} of {attempted} sales.")
+            logger.info(f"Failed to upload {len(failed_items)} of {attempted} sales")
+            if failed_items and hasattr(self.pipeline_context,
+                                        'issue_report_config') and self.pipeline_context.issue_report_config:
+                irc = self.pipeline_context.issue_report_config
+                fc = self.pipeline_context.file_config
+                key = f"{irc.key_base}_{fc.config_id}_failed_to_upload"
+                self.pipeline_context.issues.append(IssueBase(
+                    key=key,
+                    config_id=fc.config_id,
+                    name="Failed to upload Tanksales",
+                    category=IssueCategory.TANK_READING,
+                    problem_short=f"{len(failed_items)} sales did not upload",
+                    problem_long=json.dumps(failed_items)
+                ))
+
+            self.pipeline_context.included_files["sales data upload"] = json.dumps(logs, cls=CustomJSONEncoder)
+            return BBDUploadResult(
+                succeeded=succeeded,
+                failed=len(failed_items),
+                succeeded_items=store_ids
+            )
+
+        except Exception as e:
+            logger.exception(f"Unable to upload | {e}")
+            raise e