bb-integrations-library 4.3.0__tar.gz → 4.3.2__tar.gz

{bb_integrations_library-4.3.0 → bb_integrations_library-4.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bb-integrations-library
-Version: 4.3.0
+Version: 4.3.2
 Summary: Provides common logic for all types of integration jobs.
 Author: Alejandro Jordan, Ben Allen, Nicholas De Nova, Kira Threlfall, David Skinner
 Author-email: Alejandro Jordan <ajordan@capspire.com>, Ben Allen <ben.allen@capspire.com>, Nicholas De Nova <nicholas.denova@gravitate.energy>, Kira Threlfall <kira.threlfall@gravitate.energy>, David Skinner <david.skinner@gravitate.energy>

{bb_integrations_library-4.3.0 → bb_integrations_library-4.3.2}/bb_integrations_lib/models/sd/directive_upsert.py

@@ -96,13 +96,13 @@ class SDContract(BaseModel):
             if isinstance(value, list):
                 return [serialize_value(v) for v in value]
             if isinstance(value, BaseModel):
-                return value.model_dump(exclude_none=True)
+                return value.model_dump()
             return value
 
         return {
             **{
                 key: serialize_value(value)
-                for key, value in self.model_dump(exclude_none=True).items()
+                for key, value in self.model_dump().items()
             },
             "min": self._min_directive_volume,
             "max": self._max_directive_volume,

bb_integrations_library-4.3.2/bb_integrations_lib/pipelines/steps/processing/sftp_archive_aged_files_step.py

@@ -0,0 +1,94 @@
+import posixpath
+import uuid
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+from loguru import logger
+
+from bb_integrations_lib.protocols.pipelines import Step
+from bb_integrations_lib.provider.ftp.client import FTPIntegrationClient
+
+
+class SFTPArchiveAgedFilesStep(Step):
+    """Move files older than ``max_age`` from ``src_directory`` into ``archive_directory``.
+
+    Designed as a pipeline ``final_step`` so the sweep runs every cadence regardless of
+    whether upstream steps yielded any files.
+    """
+
+    def __init__(
+        self,
+        ftp_client: FTPIntegrationClient,
+        src_directory: str,
+        archive_directory: str,
+        max_age: timedelta = timedelta(days=14),
+        ensure_archive_dir: bool = True,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.ftp_client = ftp_client
+        self.src_directory = src_directory
+        self.archive_directory = archive_directory
+        self.max_age = max_age
+        self.ensure_archive_dir = ensure_archive_dir
+
+    def describe(self) -> str:
+        return (
+            f"Archive files older than {self.max_age} from "
+            f"{self.src_directory} -> {self.archive_directory}"
+        )
+
+    def _archive_one(self, file_name: str) -> None:
+        old_path = posixpath.join(self.src_directory, file_name)
+        new_path = posixpath.join(self.archive_directory, file_name)
+        try:
+            logger.debug(f"Archiving {old_path} -> {new_path}")
+            self.ftp_client.rename_file(old_path, new_path)
+            return
+        except Exception:
+            logger.debug(f"Archive rename failed for {old_path}, retrying with unique name")
+        try:
+            unique = posixpath.join(
+                self.archive_directory, f"DUPLICATE_{uuid.uuid4()}_{file_name}"
+            )
+            self.ftp_client.rename_file(old_path, unique)
+            logger.debug(f"Archived (dedup) {old_path} -> {unique}")
+        except Exception:
+            logger.warning(
+                f"Archive backup also failed for {old_path}; deleting source to bound directory."
+            )
+            self.ftp_client.delete_file(old_path)
+
+    async def execute(self, i: Any = None) -> Any:
+        if self.ensure_archive_dir:
+            self.ftp_client.mkdir(self.archive_directory, exist_ok=True)
+
+        cutoff = datetime.now(tz=timezone.utc) - self.max_age
+        logger.info(
+            f"Scanning {self.src_directory} for files older than {self.max_age} "
+            f"(mtime <= {cutoff.isoformat()})"
+        )
+
+        file_names = list(self.ftp_client.list_files(self.src_directory))
+        archived = 0
+        skipped = 0
+        for file_name in file_names:
+            file_path = posixpath.join(self.src_directory, file_name)
+            info = self.ftp_client.get_file_info(file_path)
+            if not info.last_modification_time:
+                logger.debug(f"Skipping {file_name}: no server mtime available")
+                skipped += 1
+                continue
+            mtime = datetime.fromtimestamp(info.last_modification_time, tz=timezone.utc)
+            if mtime > cutoff:
+                skipped += 1
+                continue
+            self._archive_one(file_name)
+            archived += 1
+
+        logger.info(
+            f"Archive sweep complete: archived={archived}, kept={skipped}, "
+            f"scanned={len(file_names)}"
+        )
+        return i

{bb_integrations_library-4.3.0 → bb_integrations_library-4.3.2}/bb_integrations_lib/provider/api/dtn/client.py

@@ -4,6 +4,30 @@ from typing import Dict
 import httpx
 import pandas as pd
 from httpx import Response
+from loguru import logger
+from tenacity import AsyncRetrying, retry_if_exception, stop_after_attempt, wait_exponential
+
+
+ALLOCATIONS_MAX_ATTEMPTS = 5
+ALLOCATIONS_RETRY_STATUS = 421
+
+
+def _is_retriable_allocation_error(exc: BaseException) -> bool:
+    # DTN intermittently returns 421 (Misdirected Request); retry those.
+    return (
+            isinstance(exc, httpx.HTTPStatusError)
+            and exc.response.status_code == ALLOCATIONS_RETRY_STATUS
+    )
+
+
+def _log_allocation_retry(retry_state) -> None:
+    exc = retry_state.outcome.exception()
+    status = exc.response.status_code if isinstance(exc, httpx.HTTPStatusError) else "error"
+    wait = retry_state.next_action.sleep
+    logger.warning(
+        f"DTN /allocations returned {status}, retrying in {wait:.1f}s "
+        f"(attempt {retry_state.attempt_number} of {ALLOCATIONS_MAX_ATTEMPTS})"
+    )
 
 
 class DTNClient(httpx.AsyncClient):
@@ -21,7 +45,6 @@ class DTNClient(httpx.AsyncClient):
         self.api_key = api_key
         self.base_url = base_url.rstrip("/")
         self._base_url_str = base_url.rstrip("/")
-        self.client = httpx.AsyncClient(timeout=180)
         self.timeout = httpx.Timeout(timeout, connect=60.0)
 
     def __repr__(self):
@@ -38,8 +61,26 @@ class DTNClient(httpx.AsyncClient):
 
     async def get_allocations(self) -> Response:
         url = "/allocations"
-        res = await self.get(url=url, headers=self.custom_headers, timeout=self.timeout)
-        res.raise_for_status()
+        try:
+            async for attempt in AsyncRetrying(
+                stop=stop_after_attempt(ALLOCATIONS_MAX_ATTEMPTS),
+                wait=wait_exponential(multiplier=2, min=2, max=30),
+                retry=retry_if_exception(_is_retriable_allocation_error),
+                before_sleep=_log_allocation_retry,
+                reraise=True,
+            ):
+                with attempt:
+                    res = await self.get(
+                        url=url, headers=self.custom_headers, timeout=self.timeout
+                    )
+                    res.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            if exc.response.status_code == ALLOCATIONS_RETRY_STATUS:
+                logger.error(
+                    f"DTN /allocations still returning {ALLOCATIONS_RETRY_STATUS} "
+                    f"after {ALLOCATIONS_MAX_ATTEMPTS} attempts; giving up."
+                )
+            raise
         return res
 
     async def get_data_in_group(self, url: str) -> Response:

{bb_integrations_library-4.3.0 → bb_integrations_library-4.3.2}/bb_integrations_lib/provider/ftp/client.py

@@ -129,6 +129,16 @@ class FTPIntegrationClient:
         """
         return self.interface.get_file_info(path)
 
+    def mkdir(self, path: str, exist_ok: bool = True) -> None:
+        """
+        Create a directory on the remote server.
+        :param path: The remote path of the directory to create.
+        :param exist_ok: If True (default), do not raise if the directory already exists.
+        :raises FileExistsError: If the directory already exists and ``exist_ok`` is False.
+        :raises NotImplementedError: For non-SFTP connections.
+        """
+        self.interface.mkdir(path, exist_ok=exist_ok)
+
     def download_dir(
         self, directory: str, filt: Callable[[str], bool] = None
     ) -> Iterable[RawData]:

{bb_integrations_library-4.3.0 → bb_integrations_library-4.3.2}/bb_integrations_lib/provider/ftp/interface.py

@@ -48,6 +48,9 @@ class FTPClientInterface(ABC):
     def download_files(self, paths: Iterable[str]) -> Iterable[RawData]:
         raise NotImplementedError()
 
+    def mkdir(self, path: str, exist_ok: bool = True) -> None:
+        raise NotImplementedError()
+
     @staticmethod
     def reconnect_retry(func):
         def wrap(*args, **kwargs):
@@ -249,6 +252,17 @@ class SFTPClient(FTPClientInterface):
             last_modification_time=file_info.st_mtime,
         )
 
+    @FTPClientInterface.reconnect_retry
+    def mkdir(self, path: str, exist_ok: bool = True) -> None:
+        try:
+            self.sftp.stat(path)
+        except IOError:
+            logger.debug(f"Creating directory {path}")
+            self.sftp.mkdir(path)
+            return
+        if not exist_ok:
+            raise FileExistsError(f"Directory already exists: {path}")
+
 
 def get_ppk(path: str = "secrets/id_rsa") -> RSAKey:
     """

{bb_integrations_library-4.3.0 → bb_integrations_library-4.3.2}/pyproject.toml

@@ -8,7 +8,7 @@ module-root = ""
 
 [project]
 name = "bb-integrations-library"
-version = "4.3.0"
+version = "4.3.2"
 authors = [
     {name = "Alejandro Jordan", email = "ajordan@capspire.com"},
     {name = "Ben Allen", email = "ben.allen@capspire.com"},