flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/DuplicatePhoto.py

@@ -0,0 +1,411 @@
+from typing import Callable, Dict, Any, Optional, List
+import re
+import asyncio
+import asyncpg
+from pgvector.asyncpg import register_vector
+from navigator.libs.json import JSONContent
+from .flow import FlowComponent
+from ..exceptions import ConfigError, ComponentError
+from ..conf import default_dsn
+
+IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+def qid(name: str) -> str:
+    """
+    Very small helper to quote SQL identifiers safely.
+    Raises if name contains anything but letters, digits or '_'.
+    """
+    if not IDENT_RE.match(name):
+        raise ValueError(
+            f"illegal identifier: {name!r}"
+        )
+    return '"' + name + '"'
+
+
+class DuplicatePhoto(FlowComponent):
+    """DuplicatePhoto.
+
+    Check if Photo is Duplicated and add a column with the result.
+    This component is used to check if a photo is duplicated in the dataset.
+    It uses the image hash to check if the photo is duplicated.
+    The image hash is a unique identifier for the image.
+    The image hash is calculated using the image hash algorithm.
+    The image hash algorithm is a fast and efficient way to calculate the hash of an image.
+    saves a detailed information about matches based on perceptual hash and vector similarity.
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self.id_column: str = kwargs.get("id_column", "photo_id")
+        self.hash_column: str = kwargs.get("hash_column", "image_hash")
+        self.vector_column: str = kwargs.get("vector_column", "image_vector")
+        # 4-6 (exact duplicates)
+        self.hamming_threshold: int = kwargs.get("hamming_threshold", 4)
+        # exact match: 0.05-0.10 (95-99% similarity for duplicates)
+        self.vector_threshold: float = kwargs.get("vector_threshold", 0.05)
+        # Similarity detection
+        # More lenient threshold  8-12 (similar images)
+        self.similar_hamming_threshold: int = kwargs.get("similar_hamming_threshold", 8)
+        # ~95% similarity 0.15-0.25 (75-85% similarity for similar images)
+        self.similar_vector_threshold: float = kwargs.get("similar_vector_threshold", 0.15)
+        self.tablename: str = kwargs.get("tablename", "image_bank")
+        self.schema: str = kwargs.get("schema", "public")
+        self.duplicate_column: str = kwargs.get("duplicate_column", "is_duplicated")
+        self.similar_column: str = kwargs.get("similar_column", "is_similar")
+        self.pool: asyncpg.Pool | None = None
+        super(DuplicatePhoto, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self._semaphore = asyncio.Semaphore(10)  # Adjust the limit as needed
+
+    def _qualified_tablename(self) -> str:
+        """
+        Get the qualified table name.
+        """
+        if not self.schema:
+            raise ConfigError("Schema is not set.")
+        if not self.tablename:
+            raise ConfigError("Table name is not set.")
+        return f"{qid(self.schema)}.{qid(self.tablename)}"
+
+    def _build_phash_sql(self) -> str:
+        return (
+            f"SELECT {qid(self.id_column)}, "
+            f"bit_count(('x' || $1)::bit(256) # ('x' || {qid(self.hash_column)})::bit(256)) as distance "
+            f"FROM {self._qualified_tablename()} "
+            f"WHERE {qid(self.id_column)} IS DISTINCT FROM $3::bigint "
+            f"AND bit_count(('x' || $1)::bit(256) # ('x' || {qid(self.hash_column)})::bit(256)) <= $2::integer "
+            f"ORDER BY distance ASC "
+            f"LIMIT 1;"
+        )
+
+    def _build_vector_sql(self) -> str:
+        return (
+            f"SELECT {qid(self.id_column)}, "
+            f"{qid(self.vector_column)} <-> $1::vector as distance, "
+            f"1 - ({qid(self.vector_column)} <=> $1::vector) as similarity "
+            f"FROM {self._qualified_tablename()} "
+            f"WHERE {qid(self.id_column)} IS DISTINCT FROM $3::bigint "
+            f"AND {qid(self.vector_column)} <-> $1::vector < $2::float8 "
+            f"ORDER BY distance ASC "
+            f"LIMIT 1;"
+        )
+
+    def _build_similar_phash_sql(self) -> str:
+        return (
+            f"SELECT {qid(self.id_column)}, "
+            f"bit_count(('x' || $1)::bit(256) # ('x' || {qid(self.hash_column)})::bit(256)) as distance "
+            f"FROM {self._qualified_tablename()} "
+            f"WHERE {qid(self.id_column)} IS DISTINCT FROM $3::bigint "
+            f"AND bit_count(('x' || $1)::bit(256) # ('x' || {qid(self.hash_column)})::bit(256)) > $2::integer "
+            f"AND bit_count(('x' || $1)::bit(256) # ('x' || {qid(self.hash_column)})::bit(256)) <= $4::integer "
+            f"ORDER BY distance ASC "
+            f"LIMIT 1;"
+        )
+
+    def _build_similar_vector_sql(self) -> str:
+        return (
+            f"SELECT {qid(self.id_column)}, "
+            f"{qid(self.vector_column)} <-> $1::vector as distance, "
+            f"1 - ({qid(self.vector_column)} <=> $1::vector) as similarity "
+            f"FROM {self._qualified_tablename()} "
+            f"WHERE {qid(self.id_column)} IS DISTINCT FROM $3::bigint "
+            f"AND {qid(self.vector_column)} <-> $1::vector >= $2::float8 "
+            f"AND {qid(self.vector_column)} <-> $1::vector < $4::float8 "
+            f"ORDER BY distance ASC "
+            f"LIMIT 1;"
+        )
+
+    async def pgvector_init(self, conn):
+        """
+        Initialize pgvector extension in PostgreSQL.
+        """
+        # Setup jsonb encoder/decoder
+        def _encoder(value):
+            # return json.dumps(value, cls=BaseEncoder)
+            return self._encoder.dumps(value)  # pylint: disable=E1120
+
+        def _decoder(value):
+            return self._encoder.loads(value)  # pylint: disable=E1120
+
+        await conn.set_type_codec(
+            "json",
+            encoder=_encoder,
+            decoder=_decoder,
+            schema="pg_catalog"
+        )
+        await conn.set_type_codec(
+            "jsonb",
+            encoder=_encoder,
+            decoder=_decoder,
+            schema="pg_catalog"
+        )
+
+        await register_vector(conn)
+
+    # ──────────────────────────────────────────────────────────────
+    # Setup / teardown
+    # ──────────────────────────────────────────────────────────────
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+
+        # column checks
+        for col in (self.id_column, self.hash_column, self.vector_column,):
+            if col not in self.data.columns:
+                raise ConfigError(
+                    f"Column '{col}' missing from DataFrame"
+                )
+        self.pool = await asyncpg.create_pool(
+            dsn=default_dsn,
+            min_size=1,
+            max_size=4,
+            max_queries=100,
+            init=self.pgvector_init,
+            timeout=10,
+        )
+        # Check if the table exists
+        if not self.pool:
+            raise ConfigError(
+                "Database connection pool is not initialized."
+            )
+        async with self.pool.acquire() as conn:
+            try:
+                await conn.execute(
+                    f"SELECT 1 FROM {self.schema}.{self.tablename} LIMIT 1"
+                )
+            except asyncpg.exceptions.UndefinedTableError:
+                raise ConfigError(
+                    f"Table {self.schema}.{self.tablename} does not exist."
+                )
+            except asyncpg.exceptions.UndefinedSchemaError:
+                raise ConfigError(
+                    f"Schema {self.schema} does not exist."
+                )
+        if self.duplicate_column not in self.data.columns:
+            self.data[self.duplicate_column] = {
+                "phash": None,
+                "vector": None,
+                "duplicate": False
+            }
+        if self.similar_column not in self.data.columns:
+            self.data[self.similar_column] = {
+                "phash": None,
+                "vector": None,
+                "similar": False,
+                "similarity_percentage": None
+            }
+        # prepare SQL strings
+        self._sql_phash = self._build_phash_sql()
+        self._sql_vector = self._build_vector_sql()
+        self._sql_similar_phash = self._build_similar_phash_sql()
+        self._sql_similar_vector = self._build_similar_vector_sql()
+
+    async def close(self):
+        if self.pool:
+            await self.pool.close()
+
+    # --------------- duplicate test --------------------
+    async def _check_duplicates(
+        self,
+        conn,
+        phash: str,
+        vec: list[float],
+        current_id: int
+    ) -> tuple[Dict[str, Any], Dict[str, Any]]:
+        """
+        Check if the given hash and vector are duplicated in the database.
+        Return detailed information about matches.
+
+        :param conn: Database connection.
+        :param phash: Perceptual hash of the image.
+        :param vec: Vector representation of the image.
+        :param current_id: Current photo ID.
+        :return: Tuple of dictionaries with duplicate and similarity information
+        """
+        duplicate_result = {
+            "phash": None,
+            "vector": None,
+            "duplicate": False
+        }
+        similar_result = {
+            "phash": None,
+            "vector": None,
+            "similar": False,
+            "similarity_percentage": None
+        }
+
+        # Check perceptual hash match for both duplicates and similar images
+        if phash:
+            phash_match = await conn.fetchrow(
+                self._sql_phash,
+                phash,
+                self.hamming_threshold,  # Strict threshold for duplicates
+                current_id
+            )
+
+            if phash_match:
+                distance = int(phash_match["distance"])
+                duplicate_result["phash"] = {
+                    "duplicate": True,
+                    self.id_column: phash_match[self.id_column],
+                    "threshold": distance
+                }
+
+        # Check vector match for both duplicates and similar images
+        vector_match = await conn.fetch(
+            self._sql_vector,
+            vec,
+            self.vector_threshold,
+            current_id
+        )
+
+        if vector_match:
+            distance = float(vector_match["distance"])
+            similarity = float(vector_match.get("similarity", 1 - distance))
+            similarity_pct = similarity * 100
+
+            duplicate_result["vector"] = {
+                "duplicate": True,
+                "photo_id": vector_match[self.id_column],
+                "threshold": distance,
+                "similarity": similarity,
+                "similarity_percentage": similarity_pct
+            }
+
+        # Determine overall duplicate status
+        phash_duplicate = duplicate_result["phash"] is not None and duplicate_result["phash"].get("duplicate", False)
+        vector_duplicate = duplicate_result["vector"] is not None and duplicate_result["vector"].get("duplicate", False)
+
+        if phash_duplicate or vector_duplicate:
+            duplicate_result["duplicate"] = True
+            # If it's a duplicate, don't check for similarity
+            return duplicate_result, similar_result
+
+        similar_phash_match = await conn.fetchrow(
+            self._sql_similar_phash,
+            phash,
+            self.hamming_threshold,      # Duplicate threshold (lower bound)
+            current_id,
+            self.similar_hamming_threshold  # Similar threshold (upper bound)
+        )
+        if similar_phash_match:
+            distance = int(similar_phash_match["distance"])
+            # Calculate perceptual hash similarity percentage
+            hash_similarity_pct = 100 - (distance / 256 * 100)
+            similar_result["phash"] = {
+                "similar": True,
+                self.id_column: similar_phash_match[self.id_column],
+                "threshold": distance,
+                "similarity_percentage": hash_similarity_pct
+            }
+
+        similar_vector_match = await conn.fetchrow(
+            self._sql_similar_vector,
+            vec,
+            self.vector_threshold,      # Duplicate threshold (lower bound)
+            current_id,
+            self.similar_vector_threshold  # Similar threshold (upper bound)
+        )
+
+        if similar_vector_match:
+            distance = float(similar_vector_match["distance"])
+            similarity = float(similar_vector_match.get("similarity", 1 - distance))
+            similarity_pct = similarity * 100
+
+            similar_result["vector"] = {
+                "similar": True,
+                "photo_id": similar_vector_match[self.id_column],
+                "threshold": distance,
+                "similarity": similarity,
+                "similarity_percentage": similarity_pct
+            }
+
+        # Determine overall similarity status
+        phash_similar = similar_result["phash"] is not None and similar_result["phash"].get("similar", False)
+        vector_similar = similar_result["vector"] is not None and similar_result["vector"].get("similar", False)
+
+        if phash_similar or vector_similar:
+            similar_result["similar"] = True
+
+            # Get the best similarity percentage
+            if vector_similar and similar_result["vector"].get("similarity_percentage") is not None:
+                similar_result["similarity_percentage"] = similar_result["vector"]["similarity_percentage"]
+            elif phash_similar and similar_result["phash"].get("similarity_percentage") is not None:
+                similar_result["similarity_percentage"] = similar_result["phash"]["similarity_percentage"]
+
+        return duplicate_result, similar_result
+
+    async def _process_row(self, conn, row) -> Dict[str, Any]:
+        """
+        Process a row and check for duplicates with detailed information.
+
+        :param conn: Database connection.
+        :param row: Row data to process.
+        :return: Dictionary with detailed match information.
+        """
+        phash = row[self.hash_column]
+        vec = row[self.vector_column]
+        current_id = row[self.id_column]
+
+        # Log current processing information for debugging
+        self._logger.debug(
+            f"Processing photo_id: {current_id} with threshold: {self.vector_threshold}"
+        )
+
+        duplicate_info, similar_info = await self._check_duplicates(conn, phash, vec, current_id)
+
+        # Debug information about match results
+        if duplicate_info["vector"]:
+            self._logger.debug(f"Vector match found: {duplicate_info['vector']}")
+        if duplicate_info["phash"]:
+            self._logger.debug(f"Perceptual hash match found: {duplicate_info['phash']}")
+
+        # Update the row with duplicate and similarity information
+        row[self.duplicate_column] = duplicate_info
+        row[self.similar_column] = similar_info
+        if duplicate_info.get('duplicate', False) is True:
+            row['is_duplicated'] = True
+        if similar_info.get('similar', False) is True and not duplicate_info.get('duplicate', False):
+            row['is_similar'] = True
+            # If we have similarity percentage, add it directly to the row
+            if similar_info.get('similarity_percentage') is not None:
+                row['similarity_percentage'] = similar_info['similarity_percentage']
+        return row
+
+    async def run(self):
+        """
+        Run the duplicate detection with enhanced information.
+        """
+        if self.pool is None:
+            raise ConfigError("Database connection pool is not initialized.")
+
+        # Process rows and check for duplicates
+        async def handle(idx):
+            async with self._semaphore, self.pool.acquire() as conn:
+                row = self.data.loc[idx].to_dict()
+                updated_row = await self._process_row(conn, row)
+                # Write duplicate info back into DataFrame
+                return idx, updated_row[self.duplicate_column], updated_row[self.similar_column]
+
+        results = await asyncio.gather(*(handle(i) for i in self.data.index))
+        # Apply results to DataFrame all at once
+        for idx, dup_result, sim_result in results:
+            self.data.at[idx, self.duplicate_column] = dup_result
+            self.data.at[idx, self.similar_column] = sim_result
+
+            # Set flat boolean fields for easier filtering
+            self.data.at[idx, 'is_duplicated'] = dup_result.get('duplicate', False)
+
+            is_similar = sim_result.get('similar', False) and not dup_result.get('duplicate', False)
+            self.data.at[idx, 'is_similar'] = is_similar
+
+            # Set similarity percentage if available
+            if is_similar and sim_result.get('similarity_percentage') is not None:
+                self.data.at[idx, 'similarity_percentage'] = sim_result['similarity_percentage']
+
+        self._result = self.data
+        return self._result

flowtask/components/EmployeeEvaluation.py

@@ -0,0 +1,237 @@
+from collections.abc import Callable
+import asyncio
+from typing import Any
+import pandas as pd
+# Bot Infraestructure:
+from parrot.bots.basic import BasicBot
+from .flow import FlowComponent
+from ..exceptions import ComponentError, ConfigError
+
+class EmployeeEvaluation(FlowComponent):
+    """
+        EmployeeEvaluation
+
+        Overview
+
+            The EmployeeEvaluation class is a component for interacting with an IA Agent evaluating Users chats.
+        .. table:: Properties
+        :widths: auto
+
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | Name             | Required | Description                                                                                      |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+            | output_column    |   Yes    | Column for saving the Customer Satisfaction information.                                         |
+            +------------------+----------+--------------------------------------------------------------------------------------------------+
+        Return
+
+            A Pandas Dataframe with the EmployeeEvaluation statistics.
+
+    """ # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        super().__init__(
+            loop=loop, job=job, stat=stat, **kwargs
+        )
+
+        self._bot_name = kwargs.get('bot_name', 'EmployeeBot')
+        # TaskStorage
+        # Find in the taskstorage, the "prompts" directory.
+        prompt_path = self._taskstore.path.joinpath(self._program, 'prompts')
+        if not prompt_path.exists():
+            raise ConfigError(
+                f"{self.system_prompt} Prompts Path Not Found: {prompt_path}"
+            )
+        self.prompt_path = prompt_path
+        # System Prompt:
+        # is hardcoded to this particular Bot.
+        self.system_prompt_file = 'employee.txt'
+        # Bot Object:
+        self._bot: Any = None
+
+    async def start(self, **kwargs):
+        """
+            start
+
+            Overview
+
+                The start method is a method for starting the EmployeeEvaluation component.
+
+            Return
+
+                True if the EmployeeEvaluation component started successfully.
+
+        """
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError(
+                "EmployeeBot: Data Was Not Found"
+            )
+        if not self.output_column:
+            raise ConfigError(
+                "Employee Evaluation: output_column is required"
+            )
+        # check if Prompt File exists
+        prompt_file = self.prompt_path.joinpath(self.system_prompt_file)
+        if not prompt_file.exists():
+            raise ConfigError(
+                f"{self.system_prompt} Prompt File Not Found: {prompt_file}"
+            )
+        self.system_prompt_file = prompt_file.name
+        # read the prompt file as text:
+        with open(prompt_file, 'r') as f:
+            self.system_prompt = f.read()
+        # Set the Bot:
+        try:
+            self._bot = BasicBot(
+                name=self._bot_name,
+                system_prompt=self.system_prompt,
+                goal="Employee Evaluation using chat messages",
+                use_llm=self.llm.get('name', 'name'),
+                model_name=self.llm.get('model_name', 'gemini-1.5-pro'),
+            )
+            # configure the bot:
+            await self._bot.configure()
+        except Exception as err:
+            raise ComponentError(
+                f"{self.system_prompt} Error Configuring Bot: {err}"
+            ) from err
+        return True
+
+    def generate_prompt(self, df, sender_name, messages):
+        # Filter the dataframe for the given `sender_name`
+        employee_df = df[df['sender_name'] == sender_name].copy()
+
+        # Extract the relevant statistics
+        positive_count = employee_df['Positive_Count'].values[0]
+        negative_count = employee_df['Negative_Count'].values[0]
+        avg_message_length = employee_df['Avg_Message_Length'].values[0]
+        percentage_of_messages = employee_df['Percentage_of_Messages'].values[0]
+        rank_by_message_count = employee_df['Rank_by_Message_Count'].values[0]
+        message_count = employee_df['Message_Count'].values[0]
+
+        # # Extract the chat messages
+        # chat_messages = ""
+        # for chat in messages:
+        #     chat_messages += f"* {chat}\n"
+
+        # Extract the chat messages
+        chat_messages = employee_df['text'].values[0]
+
+        # Define the prompt for Gemini Pro
+        prompt = f"""
+        Please analyze the chat messages and provide insights and useful information about the employee's behavior,
+        including their mood, feelings, and most relevant chat messages.
+
+        Use the following statistics to support your analysis:
+        - Positive_Count: {positive_count}
+        - Negative_Count: {negative_count}
+        - Avg_Message_Length: {avg_message_length}
+        - Percentage_of_Messages: {percentage_of_messages}
+        - Rank_by_Message_Count: {rank_by_message_count}
+        - Message_Count: {message_count}
+
+        Chat messages: {chat_messages}
+        """
+
+        return prompt
+
+    async def run(self):
+        """
+            run
+
+            Overview
+
+                The run method is a method for running the CustomerSatisfaction component.
+
+            Return
+
+                A Pandas Dataframe with the Customer Satisfaction statistics.
+
+        """
+        # Create the summary statistics about employees conversations:
+        # Aggregate the data by `sender_name` and count the number of messages
+        message_counts = self.data.groupby('sender_name')['text'].count().reset_index(name='Message_Count')
+        # Calculate the total number of messages sent by each employee
+        total_messages = message_counts['Message_Count'].sum()
+        # Calculate the percentage of messages sent by each employee
+        message_counts['Percentage_of_Messages'] = (message_counts['Message_Count'] / total_messages) * 100
+        # Rank the employees based on the number of messages sent
+        message_counts['Rank_by_Message_Count'] = message_counts['Message_Count'].rank(ascending=False)
+        # combine message_counts into self.data dataframe:
+        self.data = self.data.merge(message_counts, on='sender_name')
+        # Group the data by `sender_name` and calculate the average message length
+        avg_message_length = self.data.groupby('sender_name')['text'].apply(
+            lambda x: x.str.len().mean()
+        ).reset_index(name='Avg_Message_Length')
+        # combine avg_message_length into self.data dataframe:
+        self.data = self.data.merge(avg_message_length, on='sender_name')
+
+        # Filter for negative emotions and sentiments
+        negative_interactions = self.data[
+            self.data['predicted_emotion'].isin(['anger', 'disgust', 'fear', 'sadness']) | (self.data['predicted_sentiment'] == 'Negative')  # noqa
+        ]
+
+        # Group by `sender_name` and count the number of negative messages
+        negative_counts = negative_interactions.groupby('sender_name').size().reset_index(name='Negative_Count')
+
+        # Sort the results in descending order
+        negative_counts = negative_counts.sort_values(by='Negative_Count', ascending=False)
+
+        # Filter for positive emotions and sentiments
+        positive_interactions = self.data[
+            self.data['predicted_emotion'].isin(['joy'])
+            | (self.data['predicted_sentiment'] == 'Positive')
+        ]
+
+        # Group by `sender_name` and count the number of positive messages
+        positive_counts = positive_interactions.groupby('sender_name').size().reset_index(name='Positive_Count')
+
+        # Sort the results in descending order
+        positive_counts = positive_counts.sort_values(by='Positive_Count', ascending=False)
+
+        # Merge the negative and positive counts to self.data:
+        self.data = self.data.merge(negative_counts, on='sender_name', how='left')
+        self.data = self.data.merge(positive_counts, on='sender_name', how='left')
+        # Fill NaN values with 0
+        self.data = self.data.fillna(0)
+
+        columns = [
+            "sender_name",
+            'Positive_Count',
+            'Negative_Count',
+            'Avg_Message_Length',
+            'Percentage_of_Messages',
+            'Rank_by_Message_Count',
+            'Message_Count'
+        ]
+        # Group by all created columns + sender name, and convert to a list the "text" column.
+        grouped = self.data.groupby(columns)['text'].apply(list).reset_index()
+        employee_evaluation = {}
+        for _, row in grouped.iterrows():
+            employee = row['sender_name']
+            texts = row['text']
+            formatted_question = self.generate_prompt(grouped, employee, texts)
+            result = await self._bot.question(
+                question=formatted_question,
+                return_docs=False
+            )
+            employee_evaluation[employee] = {
+                "answer": result.answer
+            }
+        # Join "grouped" dataset with employee evaluation based on sender_name
+        grouped[self.output_column] = grouped['sender_name'].map(
+            lambda x: employee_evaluation[x]['answer']
+        )
+        # return the grouped dataframe
+        self._result = grouped
+        return self._result
+
+    async def close(self):
+        pass