plexflow 0.0.64__py3-none-any.whl

plexflow/utils/hooks/http.py

@@ -0,0 +1,84 @@
+import os
+import yaml
+import requests
+from typing import Optional, Dict, Any
+from airflow.providers.http.hooks.http import HttpHook
+
+class UniversalHttpHook:
+    """
+    A universal HTTP hook that can work in Airflow as well as standalone.
+    
+    When used with Airflow, connection details are fetched from Airflow Connections.
+    When used standalone, these details should be loaded from a YAML file named after the connection ID.
+    
+    Args:
+        method (str): The HTTP method. Defaults to 'GET'.
+        http_conn_id (str, optional): The connection ID, used as Airflow connection ID or as the name for the YAML file. Defaults to None.
+        config_folder (str, optional): The folder where the YAML configuration file is located. Defaults to None.
+        
+    Attributes:
+        hook (HttpHook, optional): The Airflow HttpHook instance.
+        session (requests.Session, optional): The requests Session instance.
+        config (dict, optional): The configuration loaded from the YAML file.
+        
+    Examples:
+        Using UniversalHttpHook with Airflow:
+            hook = UniversalHttpHook(method='GET', http_conn_id='my_http_connection')
+            response = hook.run('/api/v1/resource')
+            
+        Using UniversalHttpHook in standalone mode with a YAML configuration file:
+            hook = UniversalHttpHook(method='POST', http_conn_id='my_http_connection', config_folder='/path/to/configs')
+            response = hook.run('/api/v1/resource', data={'key': 'value'})
+    """
+    
+    def __init__(self, method: str = 'GET', http_conn_id: Optional[str] = None, config_folder: Optional[str] = None):
+        self.method = method
+        self.http_conn_id = http_conn_id
+        self.config_folder = '' if config_folder is None else config_folder
+        if not self.config_folder:
+            self.hook = HttpHook(http_conn_id=self.http_conn_id, method=self.method)
+        else:
+            self.hook = None
+            if self.http_conn_id is None:
+                raise ValueError("http_conn_id must be provided when running in standalone mode")
+            self.session = requests.Session()
+            config_path = os.path.join(self.config_folder, f"{self.http_conn_id}.yaml")
+            with open(config_path, 'r') as file:
+                self.config = yaml.safe_load(file)
+
+    def get_conn(self, headers: Dict[str, str]) -> Any:
+        """
+        Establishes a connection for making HTTP requests.
+        
+        Args:
+            headers (dict): The headers for the HTTP request.
+            
+        Returns:
+            An object that can be used to make HTTP requests.
+        """
+        if self.hook:
+            return self.hook.get_conn(headers)
+        else:
+            self.session.headers.update(self.config['headers'])
+            return self.session
+
+    def run(self, endpoint: str, data: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, extra_options: Optional[Dict[str, Any]] = None, query_params: Optional[Dict[str, str]] = None, json: Any = None) -> requests.Response:
+        """
+        Makes an HTTP request.
+        
+        Args:
+            endpoint (str): The endpoint for the HTTP request.
+            data (dict, optional): The data for the HTTP request. Defaults to None.
+            headers (dict, optional): The headers for the HTTP request. Defaults to None.
+            extra_options (dict, optional): Extra options for the HTTP request. Defaults to None.
+            query_params (dict, optional): The query parameters for the HTTP request. Defaults to None.
+        
+        Returns:
+            The response from the HTTP request.
+        """
+        if self.hook:
+            return self.hook.run(endpoint, data, headers, extra_options, query_params, json=json)
+        else:
+            url = self.config['base_url'] + endpoint
+            response = self.session.request(self.method, url, data=data, headers=headers, params=query_params, json=json)
+            return response

plexflow/utils/hooks/postgresql.py

@@ -0,0 +1,93 @@
+import os
+import yaml
+import psycopg2
+from typing import Optional, Dict, Any, List
+from airflow.providers.postgres.hooks.postgres import PostgresHook
+from psycopg2.extras import DictCursor
+
+class UniversalPostgresqlHook:
+    """
+    A universal PostgreSQL hook that can work in Airflow as well as standalone.
+    
+    When used with Airflow, connection details are fetched from Airflow Connections.
+    When used standalone, these details should be loaded from a YAML file named after the connection ID.
+    
+    Args:
+        postgres_conn_id (str, optional): The connection ID, used as Airflow connection ID or as the name for the YAML file. Defaults to None.
+        config_folder (str, optional): The folder where the YAML configuration file is located. Defaults to None.
+        
+    Attributes:
+        hook (PostgresHook, optional): The Airflow PostgresHook instance.
+        conn (psycopg2.extensions.connection, optional): The psycopg2 connection instance.
+        config (dict, optional): The configuration loaded from the YAML file.
+    """
+    
+    def __init__(self, postgres_conn_id: Optional[str] = None, config_folder: Optional[str] = None):
+        self.postgres_conn_id = postgres_conn_id
+        self.config_folder = '' if config_folder is None else config_folder
+        if self.config_folder:
+            if self.postgres_conn_id is None:
+                raise ValueError("postgres_conn_id must be provided when running in standalone mode")
+            config_path = os.path.join(self.config_folder, f"{self.postgres_conn_id}.yaml")
+            with open(config_path, 'r') as file:
+                self.config = yaml.safe_load(file)
+            self.conn = psycopg2.connect(
+                dbname=self.config['dbname'],
+                user=self.config['user'],
+                password=self.config['password'],
+                host=self.config['host'],
+                port=self.config['port']
+            )
+        else:
+            self.hook = PostgresHook(postgres_conn_id=self.postgres_conn_id)
+            self.conn = None
+
+    def get_conn(self) -> Any:
+        """
+        Establishes a connection to the PostgreSQL database.
+        
+        Returns:
+            A connection object that can be used to interact with the database.
+        """
+        if self.conn:
+            return self.conn
+        else:
+            return self.hook.get_conn()
+
+    def get_first(self, sql: str, params: Optional[Dict[str, Any]] = None) -> Optional[Dict[str, Any]]:
+        """
+        Executes the SQL query and returns the first result.
+        
+        Args:
+            sql (str): The SQL query to execute.
+            params (dict, optional): The parameters to substitute into the SQL query.
+        
+        Returns:
+            The first result from the executed SQL query as a dictionary where keys are column names and values are column values.
+        """
+        if self.conn:
+            with self.conn.cursor(cursor_factory=DictCursor) as cur:
+                cur.execute(sql, params)
+                result = cur.fetchone()
+            return dict(result) if result else None
+        else:
+            return self.hook.get_first(sql, parameters=params)
+
+    def get_all(self, sql: str, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]:
+        """
+        Executes the SQL query and returns all results.
+        
+        Args:
+            sql (str): The SQL query to execute.
+            params (dict, optional): The parameters to substitute into the SQL query.
+        
+        Returns:
+            All results from the executed SQL query as a list of dictionaries where keys are column names and values are column values.
+        """
+        if self.conn:
+            with self.conn.cursor(cursor_factory=DictCursor) as cur:
+                cur.execute(sql, params)
+                result = cur.fetchall()
+            return [dict(row) for row in result] if result else []
+        else:
+            return self.hook.get_records(sql, parameters=params)

plexflow/utils/hooks/redis.py

@@ -0,0 +1,112 @@
+import os
+import yaml
+import redis
+from typing import Optional, Dict, Any
+from airflow.providers.redis.hooks.redis import RedisHook
+
+class UniversalRedisHook:
+    """
+    A universal Redis hook that can work in Airflow as well as standalone.
+    
+    When used with Airflow, connection details are fetched from Airflow Connections.
+    When used standalone, these details should be loaded from a YAML file named after the connection ID.
+    
+    Args:
+        redis_conn_id (str, optional): The connection ID, used as Airflow connection ID or as the name for the YAML file. Defaults to None.
+        config_folder (str, optional): The folder where the YAML configuration file is located. Defaults to None.
+        
+    Attributes:
+        hook (RedisHook, optional): The Airflow RedisHook instance.
+        redis_client (redis.Redis, optional): The Redis client instance.
+        config (dict, optional): The configuration loaded from the YAML file.
+        
+    Examples:
+        Using UniversalRedisHook with Airflow:
+            hook = UniversalRedisHook(redis_conn_id='my_redis_connection')
+            response = hook.run('GET', 'my_key')
+            
+        Using UniversalRedisHook in standalone mode with a YAML configuration file:
+            hook = UniversalRedisHook(redis_conn_id='my_redis_connection', config_folder='/path/to/configs')
+            response = hook.run('SET', 'my_key', 'my_value')
+    """
+    
+    def __init__(self, redis_conn_id: Optional[str] = None, config_folder: Optional[str] = None):
+        self.redis_conn_id = redis_conn_id
+        self.config_folder = '' if config_folder is None else config_folder
+        if not self.config_folder:
+            self.hook = RedisHook(redis_conn_id=self.redis_conn_id)
+        else:
+            self.hook = None
+            if self.redis_conn_id is None:
+                raise ValueError("redis_conn_id must be provided when running in standalone mode")
+            config_path = os.path.join(self.config_folder, f"{self.redis_conn_id}.yaml")
+            with open(config_path, 'r') as file:
+                self.config = yaml.safe_load(file)
+            self.redis_client = redis.Redis(**self.config)
+
+    def get_conn(self) -> Any:
+        """
+        Establishes a connection to Redis.
+        
+        Returns:
+            An object that can be used to interact with Redis.
+        """
+        if self.hook:
+            return self.hook.get_conn()
+        else:
+            return self.redis_client
+
+    def run(self, command: str, *args, **kwargs) -> Any:
+        """
+        Executes a Redis command.
+        
+        Args:
+            command (str): The Redis command to execute.
+            *args: Positional arguments for the Redis command.
+            **kwargs: Keyword arguments for the Redis command.
+        
+        Returns:
+            The result of the Redis command.
+        """
+        if self.hook:
+            return self.hook.run(command, *args, **kwargs)
+        else:
+            return getattr(self.redis_client, command)(*args, **kwargs)
+
+    def get(self, key: str) -> Optional[str]:
+        """
+        Get the value of a key.
+
+        Args:
+            key (str): The key to retrieve.
+
+        Returns:
+            The value associated with the key, or None if the key does not exist.
+        """
+        if self.hook:
+            return self.hook.get_conn().get(key)
+        else:
+            return self.redis_client.get(key)
+
+    def set(self, key: str, value: str, ex: Optional[int] = None) -> bool:
+        """
+        Set the value of a key.
+
+        Args:
+            key (str): The key to set.
+            value (str): The value to associate with the key.
+            expiration (int, optional): The expiration time in seconds. Defaults to None.
+
+        Returns:
+            True if the key was set successfully, False otherwise.
+        """
+        if self.hook:
+            if ex is None:
+                return self.hook.get_conn().set(key, value)
+            else:
+                return self.hook.get_conn().set(key, value, ex=ex)
+        else:
+            if ex is None:
+                return self.redis_client.set(key, value)
+            else:
+                return self.redis_client.set(key, value, ex=ex)

plexflow/utils/image/storage.py

@@ -0,0 +1,36 @@
+import os
+import cloudinary
+import cloudinary.uploader
+import cloudinary.api
+
+# Retrieve Cloudinary credentials from environment variables
+cloud_name = os.getenv('CLOUDINARY_CLOUD_NAME')
+api_key = os.getenv('CLOUDINARY_API_KEY')
+api_secret = os.getenv('CLOUDINARY_API_SECRET')
+
+# Configure Cloudinary
+cloudinary.config(
+  cloud_name = cloud_name,
+  api_key = api_key,
+  api_secret = api_secret
+)
+
+def upload_image(image, **kwargs):
+    """
+    Uploads an image to Cloudinary and returns the URL of the uploaded image.
+    The image can be provided as a file path or bytes.
+    Additional arguments can be passed to the Cloudinary uploader.
+
+    :param image: Path to the image file or bytes of the image to be uploaded
+    :param kwargs: Additional arguments for the Cloudinary uploader
+    :return: URL of the uploaded image
+    """
+    try:
+        # Ensure the image is private and signed by default
+        kwargs.setdefault('type', 'private')
+        kwargs.setdefault('sign_url', True)
+        
+        response = cloudinary.uploader.upload(image, **kwargs)
+        return response
+    except Exception as e:
+        raise RuntimeError(f"An error occurred: {e}")

plexflow/utils/imdb/imdb_codes.py

@@ -0,0 +1,107 @@
+import re
+
+class IMDbCode:
+    """A class to represent an IMDb code.
+
+    Attributes:
+        code (str): A string representing the IMDb code.
+
+    Methods:
+        __str__(): Returns the IMDb code as a string.
+        __eq__(other): Compares the IMDb code with another IMDb code or string.
+        normalize_code(code: str) -> str: Normalizes an IMDb code by removing the leading 'tt' and any leading zeros.
+    """
+
+    def __init__(self, code):
+        """
+        Constructs an IMDbCode instance with the provided IMDb code.
+
+        Parameters
+        ----------
+        code : str
+            a string representing the IMDb code
+
+        Raises
+        ------
+        ValueError
+            If `code` is not a string.
+        """
+        if not isinstance(code, str):
+            raise ValueError("IMDb code must be a string.") from None
+
+        self.code = code.lower()
+
+    def __str__(self):
+        """Returns the IMDb code as a string."""
+        return self.code
+
+    def __eq__(self, other):
+        """
+        Compares the IMDb code with another IMDb code or string.
+
+        Parameters
+        ----------
+        other : IMDbCode or str
+            the other IMDb code or string to compare with
+
+        Returns
+        -------
+        bool
+            True if the IMDb codes are equal, False otherwise
+
+        Raises
+        ------
+        TypeError
+            If `other` is not an IMDbCode or a string.
+        """
+        if isinstance(other, IMDbCode):
+            return self.normalize_code(self.code) == self.normalize_code(other.code)
+        elif isinstance(other, str):
+            return self.normalize_code(self.code) == self.normalize_code(other)
+        else:
+            raise TypeError("Can only compare IMDbCode with another IMDbCode or a string.") from None
+
+    @staticmethod
+    def normalize_code(code):
+        """
+        Normalizes an IMDb code by removing the leading 'tt' and any leading zeros.
+
+        Parameters
+        ----------
+        code : str
+            the IMDb code to normalize
+
+        Returns
+        -------
+        str
+            the normalized IMDb code
+        """
+        # Remove leading 'tt' if present
+        if code.startswith('tt'):
+            code = code[2:]
+        # Remove leading zeros
+        code = code.lstrip('0')
+        return code
+
+
+def extract_imdb_code(s: str):
+    """
+    Generator function to extract all IMDB codes from a given string.
+
+    IMDB codes are assumed to follow the format 'tt' followed by 7 or more digits.
+
+    Args:
+        s (str): The string to search for IMDB codes.
+
+    Yields:
+        str: The next IMDB code found in the string. If no code is found, the function simply returns.
+
+    Examples:
+        >>> list(extract_imdb_code('https://www.imdb.com/title/tt0111161/ and https://www.imdb.com/title/tt006864600/'))
+        ['tt0111161', 'tt006864600']
+
+        >>> list(extract_imdb_code('this string has no imdb code'))
+        []
+    """
+    for match in re.finditer(r'tt\d{7,}', s):
+        yield match.group(0)

plexflow/utils/pubsub/consume.py

@@ -0,0 +1,82 @@
+import json
+import logging
+from confluent_kafka import Consumer
+import os
+
+def consume_message(topics_with_priority, group_id: str, as_json: bool = False, wait_time: float = 10.0, max_poll_attempts: int = 1):
+    return consume_messages(
+        topics_with_priority=topics_with_priority,
+        group_id=group_id,
+        as_json=as_json,
+        wait_time=wait_time,
+        max_messages=1,
+        max_poll_attempts=max_poll_attempts
+    )
+
+
+def consume_messages(topics_with_priority, group_id: str, as_json: bool = False, wait_time: float = 10.0, max_messages: int = 1, max_poll_attempts: int = 5):
+    if max_messages is None:
+        raise ValueError("max_messages cannot be None to avoid infinite message consumption")
+
+    logging.info("Consuming messages from topics...")
+    consumer = Consumer({
+        'bootstrap.servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'),
+        'group.id': group_id,
+        'auto.offset.reset': 'earliest',
+        'enable.auto.commit': True,  # Disable auto commit
+        'auto.commit.interval.ms': 1000
+    })
+    
+    # Sort topics by priority (assuming lower number means higher priority)
+    sorted_topic_names = (
+        [topics_with_priority] if isinstance(topics_with_priority, str)
+        else [topic['name'] for topic in sorted(topics_with_priority, key=lambda x: x['priority'])]
+    )
+
+    consumed_messages = []
+    consumed_message_count = 0
+
+    for topic_name in sorted_topic_names:
+        logging.info(f"Subscribing to topic: {topic_name}")
+        consumer.subscribe([topic_name])
+
+        current_poll_attempts = 0
+        while consumed_message_count <= max_messages and current_poll_attempts < max_poll_attempts:
+            logging.info(f"Polling messages from topic: {topic_name}")
+            message = consumer.poll(timeout=wait_time)
+
+            if message is None:
+                logging.info(f"No messages found in topic: {topic_name}")
+                current_poll_attempts += 1
+                continue
+
+            if message.error():
+                logging.error(f"Consumer error: {message.error()}")
+                current_poll_attempts += 1
+                continue
+
+            decoded_message = message.value().decode('utf-8')
+            logging.info(f"Consumed message from topic: {message.topic()}")
+
+            if as_json:
+                try:
+                    decoded_message = json.loads(decoded_message)
+                except json.JSONDecodeError as e:
+                    logging.error(f"JSON decode error: {e}")
+                    decoded_message = None
+
+            consumed_messages.append(decoded_message)
+            consumed_message_count += 1
+
+            if consumed_message_count >= max_messages:
+                break
+
+        if consumed_message_count >= max_messages:
+            break
+
+    # Commit offsets once at the end
+    logging.info("Committing offsets for all consumed messages")
+    consumer.commit()
+    consumer.close()
+    logging.info(f"Consumed {consumed_message_count} messages.")
+    return consumed_messages if max_messages > 1 else (consumed_messages[0] if consumed_messages else None)

plexflow/utils/pubsub/produce.py

@@ -0,0 +1,25 @@
+import json
+import logging
+from confluent_kafka import Producer
+import os
+
+def delivery_report(err, msg):
+    if err is not None:
+        logging.error(f"Message delivery failed: {err}")
+    else:
+        logging.info(f"Message delivered to {msg.topic()} [{msg.partition()}] at offset {msg.offset()}")
+
+def produce_message(topic: str, message):
+    logging.info(f"Publishing message to {topic}...")
+    producer = Producer({'bootstrap.servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')})
+    producer.produce(topic, json.dumps(message).encode('utf-8'), callback=delivery_report)
+    producer.flush()
+    logging.info(f"Message published to {topic}.")
+
+def produce_messages(topic: str, messages):
+    logging.info(f"Publishing messages to {topic}...")
+    producer = Producer({'bootstrap.servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')})
+    for message in messages:
+        producer.produce(topic, json.dumps(message).encode('utf-8'), callback=delivery_report)
+    producer.flush()
+    logging.info(f"All messages published to {topic}.")

plexflow/utils/retry/utils.py

@@ -0,0 +1,38 @@
+import time
+from retrying import retry
+
+def execute_until_success(func, delay_type, delay, max_retries=None, retry_exceptions=None, *args, **kwargs):
+    """
+    This function executes a given function until it succeeds and no exceptions are thrown.
+    
+    Parameters:
+    func (function): The function to be executed.
+    delay_type (str): The type of delay between attempts. Can be 'constant' or 'exponential'.
+    delay (int): The delay in seconds.
+    max_retries (int, optional): The maximum number of retries. If not set, it will keep trying until the function succeeds.
+    retry_exceptions (tuple, optional): The exceptions on which to retry. If not provided, retry will be attempted for any exception.
+    *args: Variable length argument list for the function.
+    **kwargs: Arbitrary keyword arguments for the function.
+
+    Returns:
+    The return value of the function, if it succeeds.
+    """
+
+    def retry_on_exception(exc):
+        """This function will be used to determine whether to retry if the function raises an exception."""
+        if retry_exceptions is None or isinstance(exc, tuple(retry_exceptions)):
+            return True  # Retry for specified exceptions or any exception
+        return False
+
+    def wait_strategy(attempt_number, delay_since_first_attempt_ms):
+        """This function determines the delay between retries."""
+        if delay_type == 'constant':
+            return delay  # Constant delay
+        elif delay_type == 'exponential':
+            return 2 ** attempt_number  # Exponential backoff
+
+    @retry(retry_on_exception=retry_on_exception, wait_func=wait_strategy, stop_max_attempt_number=max_retries)
+    def func_with_retry(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return func_with_retry(*args, **kwargs)

plexflow/utils/strings/filesize.py

@@ -0,0 +1,55 @@
+import re
+import bitmath
+from humanfriendly import parse_size as parse_size_to_bytes
+
+def parse_size(sentence):
+    """
+    Parses the sizes from a sentence and returns them in bytes.
+
+    Args:
+        sentence (str): The sentence containing sizes to be parsed.
+
+    Returns:
+        list: A list of sizes in bytes.
+
+    """
+    try:
+        size = bitmath.parse_string(sentence)
+        return [size.to_Byte().value]
+    except ValueError:
+        pass
+    
+    # Regular expression pattern for a size with optional space between number and unit
+    pattern = r'\b\d+(?:\.\d+)?\s*[KkMmGgTtPpEeZzYy]?[i]?[Bb]?\b'
+    
+    # Find all sizes in the sentence
+    matches = re.findall(pattern, sentence, re.IGNORECASE)
+    
+    sizes = []
+    for match in matches:
+        try:
+            # Remove any spaces within the match to ensure bitmath can parse it
+            size_str = match.replace(" ", "")
+            
+            try:
+                # Parse the size to a bitmath object
+                size = bitmath.parse_string(size_str)
+                
+                # Convert the size to bytes and return
+                sizes.append(size.to_Byte().value)
+            except Exception as e:
+                print(e)
+                size = parse_size_to_bytes(size_str)
+                sizes.append(size)
+    
+        except ValueError as e:
+            print(f"Error parsing size: {e}")
+    
+    return sizes
+
+
+if __name__ == '__main__':
+    # Test the parse_size function    
+    sizes = parse_size('My size is 6.40 GB')
+    
+    print(sizes)

plexflow/utils/strings/language.py

@@ -0,0 +1,14 @@
+
+_language_mapping = {
+    "dutch": "nl",
+    "dut": "nl",
+    "eng": "en",
+    "english": "en",
+    "nld": "nl",
+    "ned": "nl",
+    "nl": "nl",
+    "en": "en",
+}
+
+def get_language_code(language: str) -> str:
+    return _language_mapping.get(language.lower(), "unk")