flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/IcimsFolderCopy.py

@@ -0,0 +1,173 @@
+import os
+import shutil
+import asyncio
+from navconfig.logging import logging
+from asyncdb.exceptions import ProviderError
+from asyncdb.drivers.pg import pg
+from .flow import FlowComponent
+from ..exceptions import ComponentError, DataNotFound
+from querysource.conf import default_dsn, DB_TIMEOUT
+import pandas as pd
+
+
+class IcimsFolderCopy(FlowComponent):
+    """
+    IcimsFolderCopy.
+
+    Copies folders from one directory to another based on data retrieved from a PostgreSQL database.
+    Supports three modes: copying all folders,
+    copying based on the associate's name, or copying based on the associate's ID.
+
+    Properties:
+    +-------------------+-------------+-------------------------------------------------+
+    | Name              | Required    | Summary                                         |
+    +-------------------+-------------+-------------------------------------------------+
+    |  driver           |   Yes       | pg (default asyncdb PostgreSQL driver)          |
+    |  source_directory |   Yes       | Directory where folders are located             |
+    |  destination_dir  |   Yes       | Directory where folders will be copied to       |
+    |  by_name          |   No        | Person's name to filter by                      |
+    |  by_associate_id  |   No        | Associate ID to filter by                       |
+    +-------------------+-------------+-------------------------------------------------+
+    """
+
+    def __init__(self, loop: asyncio.AbstractEventLoop = None, job=None, stat=None, **kwargs):
+        self.source_directory = kwargs.get("source_directory")
+        self.destination_directory = kwargs.get("destination_directory")
+        self.driver = kwargs.get("driver", "pg")
+        self.by_name = kwargs.get("by_name", None)
+        self.by_associate_id = kwargs.get("by_associate_id", None)
+
+        # Automatically load environment-based credentials
+        self.dsn = default_dsn
+        self.timeout = DB_TIMEOUT
+
+        # Initialize FlowComponent
+        FlowComponent.__init__(self, loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs) -> bool:
+        """Prepare component for execution."""
+        await super(IcimsFolderCopy, self).start(**kwargs)
+        return True
+
+    async def run(self):
+        """Execute the folder copying process based on database results using Pandas."""
+        try:
+            db = pg(dsn=self.dsn, timeout=self.timeout)
+        except ProviderError as e:
+            raise ComponentError(f"Error connecting to database: {e}")
+
+        try:
+            # Establish connection and assign it to _connection
+            self._connection = await db.connection()
+
+            if not self._connection.is_connected():
+                raise ComponentError(f"DB Error: driver {self.driver} is not connected.")
+
+            query_to_run = ""
+
+            if self.by_name:
+                associate_query = (
+                    f"SELECT associate_id, full_name FROM icims.forms_list "
+                    f"WHERE full_name LIKE '%{self.by_name}%' "
+                    f"GROUP BY associate_id, full_name"
+                )
+                query_to_run = associate_query
+                logging.info(f"Running query: {query_to_run}")
+                associate_result = await self._connection.query(associate_query)
+
+            elif self.by_associate_id:
+                associate_query = (
+                    f"SELECT associate_id, full_name FROM icims.forms_list "
+                    f"WHERE associate_id = '{self.by_associate_id}' "
+                    f"GROUP BY associate_id, full_name"
+                )
+                query_to_run = associate_query
+                logging.info(f"Running query: {query_to_run}")
+                associate_result = await self._connection.query(associate_query)
+
+            elif hasattr(self, "query"):
+                # Custom query provided
+                query_to_run = self.query
+                logging.info(f"Running custom query: {query_to_run}")
+                associate_result = await self._connection.query(query_to_run)
+            else:
+                raise ComponentError("Either by_name, by_associate_id, or a custom query is required for this operation.")
+
+            # Flatten and filter out None
+            associate_result = [item for sublist in associate_result if sublist is not None for item in sublist]
+
+            # Dynamically extract field names and values
+            data = [{k: v for k, v in row.items()} for row in associate_result]
+
+            # Convert to DataFrame
+            df = pd.DataFrame(data)
+
+            # Handle missing values
+            df.dropna(subset=["associate_id"], inplace=True)
+
+            # Extract folder codes as a list
+            folder_codes = df["associate_id"].tolist()
+
+            # Copy folders
+            copied_count = self.copy_folders(folder_codes)
+
+            # Check if no folders were copied
+            if copied_count == 0:
+                raise ComponentError("No folders were copied. Please check the source or the query results.")
+
+        except ProviderError as e:
+            raise ComponentError(f"Error querying database: {e}") from e
+        except Exception as e:
+            self._logger.error(e)
+            raise
+        return True
+
+
+    def copy_folders(self, folder_codes):
+        """Copy folders and handle missing folders gracefully."""
+        copied_count = 0
+        failed_folders = []
+
+        for code in folder_codes:
+            src_folder = os.path.join(self.source_directory, code)
+            dest_folder = os.path.join(self.destination_directory, code)
+
+            if os.path.exists(src_folder):
+                if not os.path.exists(dest_folder):
+                    os.makedirs(dest_folder)
+
+                for root, dirs, files in os.walk(src_folder):
+                    for file in files:
+                        src_file = os.path.join(root, file)
+                        rel_path = os.path.relpath(src_file, src_folder)
+                        dest_file = os.path.join(dest_folder, rel_path)
+
+                        os.makedirs(os.path.dirname(dest_file), exist_ok=True)
+
+                        try:
+                            shutil.copy2(src_file, dest_file)
+                        except shutil.Error as e:
+                            self._logger.error(f"Error copying file {src_file}: {e}")
+                copied_count += 1
+            else:
+                failed_folders.append(code)
+                logging.warning(f"Folder {code} does not exist in the source directory.")
+
+        if failed_folders:
+            self._logger.error(f"Failed to find folders for the following associate_ids: {failed_folders}")
+
+        logging.info(f"Total folders copied or merged: {copied_count} out of {len(folder_codes)} requested.")
+        return copied_count
+
+    async def close(self, connection=None):
+        """Close the database connection."""
+        if not connection:
+            connection = getattr(self, '_connection', None)  # Safely get the connection
+        try:
+            if connection is not None:
+                await connection.close()
+                logging.info("Database connection closed successfully.")
+            else:
+                logging.warning("No active connection to close.")
+        except Exception as err:
+            self._logger.error(f"Error closing database connection: {err}")

flowtask/components/ImageFeatures/__init__.py

@@ -0,0 +1,5 @@
+from .process import ImageFeatures
+
+__all__ = (
+    "ImageFeatures",
+)

flowtask/components/ImageFeatures/process.py

@@ -0,0 +1,233 @@
+import asyncio
+from collections.abc import Callable
+from contextlib import AsyncExitStack
+from typing import Any
+from PIL import Image, UnidentifiedImageError
+import pyheif
+from pillow_heif import register_heif_opener
+from io import BytesIO
+import filetype
+from ..flow import FlowComponent
+from ...exceptions import (
+    ConfigError,
+    ComponentError,
+    FileError
+)
+# Parrot Image Processing plug-ins
+from parrot.interfaces.images.plugins import PLUGINS, ImagePlugin
+
+register_heif_opener()  # HEIF support
+
+class ImageFeatures(FlowComponent):
+    """
+    ImageFeatures is a component for extracting image features.
+    It extends the FlowComponent class and implements a Plugin system for various image processing tasks.
+
+    Args:
+        *args: Variable length argument list.
+        **kwargs: Arbitrary keyword arguments.
+
+    Attributes:
+        _model_name (str): The name of the model used for feature extraction.
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._plugins_list: list = kwargs.get("plugins")
+        self._plugins: list = []
+        self._semaphore = asyncio.Semaphore(8)   # limit GPU tasks
+        if not self._plugins_list:
+            raise ConfigError("Plugins list is required.")
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """
+        start.
+
+            Initialize Task.
+        """
+        if self.previous:
+            self.data = self.input
+        if self.data_column not in self.data.columns:
+            raise ValueError(
+                f'Data column {self.data_column} not found in data.'
+            )
+        # Check plugin Names:
+        plugins = []
+        for n in self._plugins_list:
+            # n is a dictionary with the plugin name and args
+            if isinstance(n, dict):
+                name = list(n.keys())[0].lower()
+                args = n[name]
+            plugin = PLUGINS.get(name)
+            if not plugin:
+                raise ConfigError(
+                    f'Plugin {n} not found in available plugins.'
+                )
+            if not issubclass(plugin, ImagePlugin):
+                raise ConfigError(
+                    f'Plugin {n} is not a subclass of ImagePlugin.'
+                )
+            if not args:
+                args = {}
+            if not isinstance(args, dict):
+                raise ConfigError(
+                    f'Plugin {n} args must be a dictionary.'
+                )
+            plugins.append(
+                {
+                    "plugin": plugin,
+                    "args": args
+                }
+            )
+        self._plugins = plugins
+
+    async def close(self):
+        pass
+
+    def get_bio(self, bio: Any) -> BytesIO:
+        """Return a BytesIO object for a payload."""
+        if isinstance(bio, Image.Image):
+            return bio
+        if isinstance(bio, bytes):
+            bio = BytesIO(bio)
+        if isinstance(bio, str):
+            bio = BytesIO(bio.encode('utf-8'))
+        if isinstance(bio, BytesIO):
+            return bio
+        if hasattr(bio, "read"):
+            bio = BytesIO(bio.read())
+        if hasattr(bio, "getvalue"):
+            bio = BytesIO(bio.getvalue())
+        else:
+            raise TypeError(
+                f"Expected bytes, str, or BytesIO, got {type(bio)}"
+            )
+        if not bio.readable():
+            raise TypeError("BytesIO is not readable.")
+        bio.seek(0)
+        return bio
+
+    async def _run_plugin(self, plugin, img: Image.Image, heif: Any = None, **kwargs):
+        """
+        Call plugin.analyze(); transparently await if it's an async def.
+        """
+        if asyncio.iscoroutinefunction(plugin.analyze):
+            return await plugin.analyze(img, heif=heif, **kwargs)
+        return plugin.analyze(img, heif)
+
+    async def run(self):
+        """
+        run.
+
+            Execute the plugin List to extract image features.
+        """
+        # Iterate over all plugins (create one single instance of each plugin):
+        _plugins = []
+        async with AsyncExitStack() as stack:
+            for spec in self._plugins:
+                cls, args = spec["plugin"], spec["args"]
+                if cls.column_name not in self.data.columns:
+                    # Create a new column in the DataFrame for the plugin's results
+                    self.data[cls.column_name] = None
+                plugin = cls(**args)
+                try:
+                    await plugin.start()
+                except Exception as e:
+                    raise ComponentError(
+                        f"Error starting plugin {plugin}: {str(e)}"
+                    ) from e
+                # If the plugin implements .open() returning an async‑context
+                if hasattr(plugin, "open"):
+                    plugin = await stack.enter_async_context(plugin)  # ⇦ one‑time open
+                _plugins.append(plugin)
+
+        # Iterate over all rows in the DataFrame:
+        # - Convert the image to a PIL Image
+        # - Call the plugin's analyze method
+        # - Store the result in the DataFrame
+        # - Use a semaphore to limit concurrent tasks
+        # - Use asyncio.gather to run the tasks concurrently
+        # - Use a memoryview to avoid copying the image data
+        # Convert BytesIO → bytes/PIL *one* time per row.
+        async def process_row(idx, row):
+            bio = row[self.data_column]
+            if not bio:
+                return
+            async with self._semaphore:
+                try:
+                    try:
+                        bio = self.get_bio(bio)
+                    except Exception as e:
+                        self._logger.error(
+                            f"Error getting BytesIO from {bio}: {e}"
+                        )
+                        return
+                    kind = filetype.guess(bio)
+                    heic = None
+                    if kind == 'image/heic':
+                        try:
+                            heic = pyheif.read_heif(bio)
+                        except Exception as e:
+                            self._logger.error(
+                                "Unable to parse Apple Heic Photo"
+                            )
+                            return
+                    if isinstance(bio, Image.Image):
+                        image = bio
+                    elif kind == 'image/heic':
+                        image = Image.frombytes(
+                            mode=heic.mode,
+                            size=heic.size,
+                            data=heic.data
+                        )
+                    else:
+                        # Decode the image once
+                        try:
+                            image = Image.open(bio)
+                        except UnidentifiedImageError:
+                            raise FileError(
+                                f"PIL cannot identify image file. MIME: {kind.mime}"
+                            )
+                    # Results from all plugins for this row
+                    for plugin in _plugins:
+                        result = await self._run_plugin(plugin, image, heic)
+                        self.data.at[idx, plugin.column_name] = result
+                except FileError as e:
+                    self._logger.error(
+                        f"Image Error on {row}: {e}"
+                    )
+                    return
+                except Exception as e:
+                    self._logger.error(
+                        f'Error processing image at index {idx}: {e}'
+                    )
+                    return
+        # Kick off tasks – DataFrame scanned exactly once
+        tasks = [
+            process_row(idx, row)
+            for idx, row in self.data.iterrows()
+        ]
+        try:
+            await asyncio.gather(*tasks)
+            self._print_data_(self.data, ':: Image Features ::')
+            self._result = self.data
+            return self._result
+        except Exception as e:
+            raise ComponentError(
+                f"Error in ImageFeatures run: {str(e)}"
+            ) from e
+        finally:
+            # Dispose of all plugins
+            for plugin in _plugins:
+                if hasattr(plugin, "dispose"):
+                    try:
+                        await plugin.dispose()
+                    except Exception as e:
+                        self._logger.error(
+                            f"Error disposing plugin {plugin}: {str(e)}"
+                        )

flowtask/components/IteratorBase.py

@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*-
+from typing import Any
+import threading
+from threading import Semaphore
+import asyncio
+from collections.abc import Callable
+from navconfig.logging import logging
+from asyncdb.exceptions import NoDataFound, ProviderError
+from ..exceptions import (
+    ComponentError,
+    NotSupported,
+    DataNotFound,
+    FileNotFound
+)
+
+from .flow import FlowComponent
+from ..interfaces.log import SkipErrors
+
+
+class ThreadJob(threading.Thread):
+    def __init__(self, job: Any, step_name: str, semaphore: Semaphore):
+        super().__init__()
+        self.step_name = step_name
+        self.job = job
+        self.exc = None
+        self.result = None
+        self.semaphore = semaphore
+
+    def run(self):
+        try:
+            asyncio.run(self.execute_job(self.job, self.step_name))
+        except Exception as ex:
+            self.exc = ex
+        finally:
+            # Release semaphore
+            self.semaphore.release()
+
+    async def execute_job(self, job: Any, step_name: str):
+        start = getattr(job, "start", None)
+        if callable(start):
+            try:
+                if asyncio.iscoroutinefunction(start):
+                    st = await job.start()
+                else:
+                    st = job.start()
+                self._logger.debug(f"STARTED: {st}")
+            except (NoDataFound, DataNotFound) as err:
+                raise DataNotFound(f"{err!s}") from err
+            except (ProviderError, ComponentError, NotSupported) as err:
+                raise ComponentError(
+                    f"Error running Start Function on {step_name}, error: {err}"
+                ) from err
+        else:
+            raise ComponentError(f"Error running Function on {step_name}")
+        try:
+            run = getattr(job, "run", None)
+            if asyncio.iscoroutinefunction(run):
+                self.result = await job.run()
+            else:
+                self.result = job.run()
+            return self.result
+        except (NoDataFound, DataNotFound) as err:
+            raise DataNotFound(f"{err!s}") from err
+        except (ProviderError, ComponentError, NotSupported) as err:
+            raise NotSupported(
+                f"Error running Component {step_name}, error: {err}"
+            ) from err
+        except Exception as err:
+            self._logger.exception(err, exc_info=True)
+            raise ComponentError(
+                f"Iterator Error on {step_name}, error: {err}"
+            ) from err
+        finally:
+            try:
+                close = getattr(job, "close", None)
+                if asyncio.iscoroutinefunction(close):
+                    await job.close()
+                else:
+                    job.close()
+            except Exception:
+                pass
+
+
+class IteratorBase(FlowComponent):
+    """
+    IteratorBase
+
+    Overview
+
+        The IteratorBase class is an abstract component for handling iterative tasks. It extends the FlowComponent class
+        and provides methods for starting tasks, retrieving steps, and executing jobs asynchronously.
+
+    .. table:: Properties
+    :widths: auto
+
+        +-----------------+----------+-------------------------------------------------------------------------------+
+        | Name            | Required | Description                                                                   |
+        +-----------------+----------+-------------------------------------------------------------------------------+
+        | iterate         |   No     | Boolean flag indicating if the component should                               |
+        |                 |          | iterate the components or return the list, defaults to False.                 |
+        +-----------------+----------+-------------------------------------------------------------------------------+
+
+        The methods in this class manage the execution of iterative tasks, including initialization, step retrieval,
+        job creation, and asynchronous execution.
+
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.iterate: bool = False
+        self._iterator: bool = True
+        self._conditions: dict = {}
+        super(IteratorBase, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """
+        start.
+
+            Initialize (if needed) a task
+        """
+        if self.previous:
+            self.data = self.input
+        return True
+
+    def get_step(self):
+        params = None
+        try:
+            if not self._TaskPile:
+                raise ComponentError("No Components in TaskPile")
+            step, idx = self._TaskPile.nextStep(self.StepName)
+            params = step.params()
+            try:
+                if params["conditions"]:
+                    self._conditions[step.name] = params["conditions"]
+            except KeyError:
+                pass
+            params["ENV"] = self._environment
+            # program
+            if hasattr(self, "_program"):
+                params["_program"] = self._program
+            # params
+            params["params"] = self._params
+            # parameters
+            params["parameters"] = self._parameters
+            # useful to change variables in set var components
+            params["_vars"] = self._vars
+            # variables dictionary
+            params["variables"] = self._variables
+            params["_args"] = self._args
+            # argument list for components (or tasks) that need argument lists
+            params["arguments"] = self._arguments
+            # for components with conditions, we can add more conditions
+            conditions = params.get("conditions", {})
+            step_conds = self._conditions.get(step.name, {})
+            if self.conditions is not None:
+                step_conds = {**self.conditions, **step_conds}
+            params["conditions"] = {**conditions, **step_conds}
+            # attributes only usable component-only
+            params["attributes"] = self._attributes
+            # the current Pile of components
+            params["TaskPile"] = self._TaskPile
+            # params['TaskName'] = step_name
+            params["debug"] = self._debug
+            params["argparser"] = self._argparser
+            # the current in-memory connector
+            params["memory"] = self._memory
+            target = step.component
+            # remove this element from tasks, doesn't need to run again
+            self._TaskPile.delStep(idx)
+            # return target and params
+            return [step, target, params]
+        finally:
+            pass
+
+    def get_job(self, target, **params):
+        job = None
+        try:
+            job = target(job=self, loop=self._loop, stat=self.stat, **params)
+            return job
+        except Exception as err:
+            raise ComponentError(
+                f"Generic Component Error on {target}, error: {err}"
+            ) from err
+
+    async def async_job(self, job, step_name):
+        start = getattr(job, "start", None)
+        if callable(start):
+            try:
+                if asyncio.iscoroutinefunction(start):
+                    st = await job.start()
+                else:
+                    st = job.start()
+                self._logger.debug(f"STARTED: {st}")
+            except (NoDataFound, DataNotFound) as err:
+                raise DataNotFound(f"{err!s}") from err
+            except (ProviderError, ComponentError, NotSupported) as err:
+                raise ComponentError(
+                    f"Error running Start Function on {step_name}, error: {err}"
+                ) from err
+        else:
+            raise ComponentError(
+                f"Error running Function on {step_name}"
+            )
+        try:
+            run = getattr(job, "run", None)
+            if asyncio.iscoroutinefunction(run):
+                result = await job.run()
+            else:
+                result = job.run()
+            self._result = result
+            return self._result
+        except (NoDataFound, DataNotFound, FileNotFound) as err:
+            try:
+                if job.skipError == SkipErrors.SKIP:
+                    self._logger.warning(
+                        f"Component {job!s} was Skipped, error: {err}"
+                    )
+                    self._result = self.data
+                    return self._result
+                elif job.skipError == SkipErrors.ENFORCE:
+                    # Enforcing to Raise Error:
+                    raise DataNotFound(f"{err!s}") from err
+                else:
+                    # Log Only
+                    self._logger.error(
+                        f"Component {job!s} was Skipped, error: {err}"
+                    )
+            except AttributeError:
+                raise DataNotFound(f"{err!s}") from err
+        except (ProviderError, ComponentError, NotSupported) as err:
+            raise NotSupported(
+                f"Error running Component {step_name}, error: {err}"
+            ) from err
+        except Exception as err:
+            self._logger.exception(err, exc_info=True)
+            raise ComponentError(
+                f"Iterator Error on {step_name}, error: {err}"
+            ) from err
+        finally:
+            try:
+                close = getattr(job, "close", None)
+                if asyncio.iscoroutinefunction(close):
+                    await job.close()
+                else:
+                    job.close()
+            except Exception:
+                pass

flowtask/components/LangchainLoader/__init__.py

@@ -0,0 +1,5 @@
+from .loader import LangchainLoader
+
+__all__ = (
+    'LangchainLoader',
+)