cwyodmodules 0.3.31__py3-none-any.whl → 0.3.33__py3-none-any.whl

cwyodmodules/batch/utilities/common/source_document.py

@@ -4,14 +4,11 @@ import json
 from urllib.parse import urlparse, quote
 from ..helpers.azure_blob_storage_client import AzureBlobStorageClient
 
-from logging import getLogger
-from opentelemetry import trace, baggage
-from opentelemetry.propagate import extract
-
-# logger = getLogger("__main__" + ".base_package")
-logger = getLogger("__main__")
-##tracer = trace.get_tracer("__main__" + ".base_package")
-tracer = trace.get_tracer("__main__")
+from ...utilities.helpers.env_helper import EnvHelper
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 class SourceDocument:
     def __init__(
@@ -54,19 +51,20 @@ class SourceDocument:
             )
         return False
 
+    @logger.trace_function(log_args=False, log_result=False)
     def to_json(self):
-        with tracer.start_as_current_span("SourceDocument.to_json"):
-            json_string = json.dumps(self, cls=SourceDocumentEncoder)
-            logger.debug(f"Serialized SourceDocument to JSON: {json_string}")
-            return json_string
+        json_string = json.dumps(self, cls=SourceDocumentEncoder)
+        logger.debug(f"Serialized SourceDocument to JSON: {json_string}")
+        return json_string
 
     @classmethod
+    @logger.trace_function(log_args=False, log_result=False)
     def from_json(cls, json_string):
-        logger.debug(f"Deserializing SourceDocument from JSON: {json_string}")
         source_document = json.loads(json_string, cls=SourceDocumentDecoder)
         return source_document
 
     @classmethod
+    @logger.trace_function(log_args=False, log_result=False)
     def from_dict(cls, dict_obj):
         logger.debug(f"Creating SourceDocument from dict: {dict_obj}")
         return cls(
@@ -81,6 +79,7 @@ class SourceDocument:
         )
 
     @classmethod
+    @logger.trace_function(log_args=False, log_result=False)
     def from_metadata(
         cls: Type["SourceDocument"],
         content: str,
@@ -114,62 +113,62 @@ class SourceDocument:
         )
         return source_document
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def get_filename(self, include_path=False):
-        with tracer.start_as_current_span("SourceDocument.get_filename"):
-            filename = self.source.replace("_SAS_TOKEN_PLACEHOLDER_", "").replace(
-                "http://", ""
-            )
-            if include_path:
-                filename = filename.split("/")[-1]
-            else:
-                filename = filename.split("/")[-1].split(".")[0]
-            logger.debug(
-                f"Extracted filename: {filename}, include_path: {include_path}"
-            )
-            return filename
+        filename = self.source.replace("_SAS_TOKEN_PLACEHOLDER_", "").replace(
+            "http://", ""
+        )
+        if include_path:
+            filename = filename.split("/")[-1]
+        else:
+            filename = filename.split("/")[-1].split(".")[0]
+        logger.debug(
+            f"Extracted filename: {filename}, include_path: {include_path}"
+        )
+        return filename
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def get_markdown_url(self):
-        with tracer.start_as_current_span("SourceDocument.get_markdown_url"):
-            url = quote(self.source, safe=":/")
-            if "_SAS_TOKEN_PLACEHOLDER_" in url:
-                blob_client = AzureBlobStorageClient()
-                container_sas = blob_client.get_container_sas()
-                url = url.replace("_SAS_TOKEN_PLACEHOLDER_", container_sas)
-            logger.debug(f"Generated markdown URL: {url}")
-            return f"[{self.title}]({url})"
+        url = quote(self.source, safe=":/")
+        if "_SAS_TOKEN_PLACEHOLDER_" in url:
+            blob_client = AzureBlobStorageClient()
+            container_sas = blob_client.get_container_sas()
+            url = url.replace("_SAS_TOKEN_PLACEHOLDER_", container_sas)
+        logger.debug(f"Generated markdown URL: {url}")
+        return f"[{self.title}]({url})"
 
 
 class SourceDocumentEncoder(json.JSONEncoder):
+    @logger.trace_function(log_args=False, log_result=False)
     def default(self, obj):
-        with tracer.start_as_current_span("SourceDocumentEncoder.default"):
-            if isinstance(obj, SourceDocument):
-                logger.debug(f"Encoding SourceDocument: {obj}")
-                return {
-                    "id": obj.id,
-                    "content": obj.content,
-                    "source": obj.source,
-                    "title": obj.title,
-                    "chunk": obj.chunk,
-                    "offset": obj.offset,
-                    "page_number": obj.page_number,
-                    "chunk_id": obj.chunk_id,
-                }
-            return super().default(obj)
+        if isinstance(obj, SourceDocument):
+            logger.debug(f"Encoding SourceDocument: {obj}")
+            return {
+                "id": obj.id,
+                "content": obj.content,
+                "source": obj.source,
+                "title": obj.title,
+                "chunk": obj.chunk,
+                "offset": obj.offset,
+                "page_number": obj.page_number,
+                "chunk_id": obj.chunk_id,
+            }
+        return super().default(obj)
 
 
 class SourceDocumentDecoder(json.JSONDecoder):
+    @logger.trace_function(log_args=False, log_result=False)
     def decode(self, s, **kwargs):
-        with tracer.start_as_current_span("SourceDocumentDecoder.decode"):
-            logger.debug(f"Decoding JSON string: {s}")
-            obj = super().decode(s, **kwargs)
-            source_document = SourceDocument(
-                id=obj["id"],
-                content=obj["content"],
-                source=obj["source"],
-                title=obj["title"],
-                chunk=obj["chunk"],
-                offset=obj["offset"],
-                page_number=obj["page_number"],
-                chunk_id=obj["chunk_id"],
-            )
-            return source_document
+        logger.debug(f"Decoding JSON string: {s}")
+        obj = super().decode(s, **kwargs)
+        source_document = SourceDocument(
+            id=obj["id"],
+            content=obj["content"],
+            source=obj["source"],
+            title=obj["title"],
+            chunk=obj["chunk"],
+            offset=obj["offset"],
+            page_number=obj["page_number"],
+            chunk_id=obj["chunk_id"],
+        )
+        return source_document

cwyodmodules/batch/utilities/document_chunking/fixed_size_overlap.py

@@ -3,12 +3,17 @@ from .document_chunking_base import DocumentChunkingBase
 from langchain.text_splitter import TokenTextSplitter
 from .chunking_strategy import ChunkingSettings
 from ..common.source_document import SourceDocument
-import logging
+from ...utilities.helpers.env_helper import EnvHelper
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 class FixedSizeOverlapDocumentChunking(DocumentChunkingBase):
     def __init__(self) -> None:
         pass
 
+    @logger.trace_function(log_args=False, log_result=False)
     def chunk(
         self, documents: List[SourceDocument], chunking: ChunkingSettings
     ) -> List[SourceDocument]:
@@ -19,8 +24,8 @@ class FixedSizeOverlapDocumentChunking(DocumentChunkingBase):
             document_url = documents[0].source
         except IndexError as e:
             # If no documents are provided, set document_url to None
-            logging.error("No documents provided for chunking.")
-            logging.debug(e)
+            logger.error("No documents provided for chunking.")
+            logger.debug(e)
             document_url = None
         splitter = TokenTextSplitter.from_tiktoken_encoder(
             chunk_size=chunking.chunk_size, chunk_overlap=chunking.chunk_overlap

cwyodmodules/batch/utilities/document_chunking/layout.py

@@ -3,12 +3,17 @@ from .document_chunking_base import DocumentChunkingBase
 from langchain.text_splitter import MarkdownTextSplitter
 from .chunking_strategy import ChunkingSettings
 from ..common.source_document import SourceDocument
-import logging
+from ...utilities.helpers.env_helper import EnvHelper
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 class LayoutDocumentChunking(DocumentChunkingBase):
     def __init__(self) -> None:
         pass
 
+    @logger.trace_function(log_args=False, log_result=False)
     def chunk(
         self, documents: List[SourceDocument], chunking: ChunkingSettings
     ) -> List[SourceDocument]:
@@ -19,8 +24,8 @@ class LayoutDocumentChunking(DocumentChunkingBase):
             document_url = documents[0].source
         except IndexError as e:
             # If no documents are provided, set document_url to None
-            logging.error("No documents provided for chunking.")
-            logging.debug(e)
+            logger.error("No documents provided for chunking.")
+            logger.debug(e)
             document_url = None
         splitter = MarkdownTextSplitter.from_tiktoken_encoder(
             chunk_size=chunking.chunk_size, chunk_overlap=chunking.chunk_overlap

cwyodmodules/batch/utilities/document_chunking/page.py

@@ -3,12 +3,17 @@ from .document_chunking_base import DocumentChunkingBase
 from langchain.text_splitter import MarkdownTextSplitter
 from .chunking_strategy import ChunkingSettings
 from ..common.source_document import SourceDocument
-import logging
+from ...utilities.helpers.env_helper import EnvHelper
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 class PageDocumentChunking(DocumentChunkingBase):
     def __init__(self) -> None:
         pass
 
+    @logger.trace_function(log_args=False, log_result=False)
     def chunk(
         self, documents: List[SourceDocument], chunking: ChunkingSettings
     ) -> List[SourceDocument]:
@@ -16,8 +21,8 @@ class PageDocumentChunking(DocumentChunkingBase):
             document_url = documents[0].source
         except IndexError as e:
             # If no documents are provided, set document_url to None
-            logging.error("No documents provided for chunking.")
-            logging.debug(e)
+            logger.error("No documents provided for chunking.")
+            logger.debug(e)
             document_url = None
         splitter = MarkdownTextSplitter.from_tiktoken_encoder(
             chunk_size=chunking.chunk_size, chunk_overlap=chunking.chunk_overlap

cwyodmodules/batch/utilities/document_loading/read.py

@@ -3,46 +3,42 @@ from .document_loading_base import DocumentLoadingBase
 from ..helpers.azure_form_recognizer_helper import AzureFormRecognizerClient
 from ..common.source_document import SourceDocument
 
-from logging import getLogger
-from opentelemetry import trace, baggage
-from opentelemetry.propagate import extract
-
-# logger = getLogger("__main__" + ".base_package")
-logger = getLogger("__main__")
-# tracer = trace.get_tracer("__main__" + ".base_package")
-tracer = trace.get_tracer("__main__")
+from ...utilities.helpers.env_helper import EnvHelper
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 
 class ReadDocumentLoading(DocumentLoadingBase):
     def __init__(self) -> None:
         super().__init__()
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def load(self, document_url: str) -> List[SourceDocument]:
-        with tracer.start_as_current_span("ReadDocumentLoading.load") as span:
-            logger.info(f"Loading document from URL: {document_url}")
-            try:
-                azure_form_recognizer_client = AzureFormRecognizerClient()
-                pages_content = (
-                    azure_form_recognizer_client.begin_analyze_document_from_url(
-                        document_url, use_layout=False
-                    )
-                )
-                documents = [
-                    SourceDocument(
-                        content=page["page_text"],
-                        source=document_url,
-                        page_number=page["page_number"],
-                        offset=page["offset"],
-                    )
-                    for page in pages_content
-                ]
-                logger.info(
-                    f"Successfully loaded {len(documents)} pages from {document_url}"
+        logger.info(f"Loading document from URL: {document_url}")
+        try:
+            azure_form_recognizer_client = AzureFormRecognizerClient()
+            pages_content = (
+                azure_form_recognizer_client.begin_analyze_document_from_url(
+                    document_url, use_layout=False
                 )
-                return documents
-            except Exception as e:
-                logger.error(
-                    f"Error loading document from {document_url}: {e}", exc_info=True
+            )
+            documents = [
+                SourceDocument(
+                    content=page["page_text"],
+                    source=document_url,
+                    page_number=page["page_number"],
+                    offset=page["offset"],
                 )
-                span.record_exception(e)
-                raise
+                for page in pages_content
+            ]
+            logger.info(
+                f"Successfully loaded {len(documents)} pages from {document_url}"
+            )
+            return documents
+        except Exception as e:
+            logger.error(
+                f"Error loading document from {document_url}: {e}", exc_info=True
+            )
+            raise

cwyodmodules/batch/utilities/helpers/azure_computer_vision_client.py

@@ -1,4 +1,3 @@
-import logging
 from urllib.parse import urljoin
 from ..helpers.azure_identity_helper import AzureIdentityHelper
 
@@ -6,8 +5,10 @@ import requests
 from requests import Response
 
 from .env_helper import EnvHelper
-
-logger = logging.getLogger("__main__")
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 
 class AzureComputerVisionClient:
@@ -28,6 +29,7 @@ class AzureComputerVisionClient:
             env_helper.AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION
         )
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def vectorize_image(self, image_url: str) -> list[float]:
         logger.info(f"Making call to computer vision to vectorize image: {image_url}")
         response = self.__make_request(
@@ -39,6 +41,7 @@ class AzureComputerVisionClient:
         response_json = self.__get_json_body(response)
         return self.__get_vectors(response_json)
 
+    @logger.trace_function(log_args=False, log_result=False)
     def vectorize_text(self, text: str) -> list[float]:
         logger.debug(f"Making call to computer vision to vectorize text: {text}")
         response = self.__make_request(
@@ -50,6 +53,7 @@ class AzureComputerVisionClient:
         response_json = self.__get_json_body(response)
         return self.__get_vectors(response_json)
 
+    @logger.trace_function(log_args=False, log_result=False)
     def __make_request(self, path: str, body) -> Response:
         try:
             headers = {}
@@ -74,12 +78,14 @@ class AzureComputerVisionClient:
         except Exception as e:
             raise Exception("Call to Azure Computer Vision failed") from e
 
+    @logger.trace_function(log_args=False, log_result=log_result)
     def __validate_response(self, response: Response):
         if response.status_code != 200:
             raise Exception(
                 f"Call to Azure Computer Vision failed with status: {response.status_code}, body: {response.text}"
             )
 
+    @logger.trace_function(log_args=False, log_result=False)
     def __get_json_body(self, response: Response) -> dict:
         try:
             return response.json()
@@ -88,6 +94,7 @@ class AzureComputerVisionClient:
                 f"Call to Azure Computer Vision returned malformed response body: {response.text}",
             ) from e
 
+    @logger.trace_function(log_args=False, log_result=log_result)
     def __get_vectors(self, response_json: dict) -> list[float]:
         if self.__RESPONSE_VECTOR_KEY in response_json:
             return response_json[self.__RESPONSE_VECTOR_KEY]

cwyodmodules/batch/utilities/helpers/azure_form_recognizer_helper.py

@@ -1,4 +1,3 @@
-import logging
 from azure.core.credentials import AzureKeyCredential
 from azure.ai.formrecognizer import DocumentAnalysisClient
 from ..helpers.azure_identity_helper import AzureIdentityHelper
@@ -6,7 +5,10 @@ import html
 import traceback
 from .env_helper import EnvHelper
 
-logger = logging.getLogger("__main__")
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 
 class AzureFormRecognizerClient:
@@ -45,6 +47,7 @@ class AzureFormRecognizerClient:
         "paragraph": "p",
     }
 
+    @logger.trace_function(log_args=False, log_result=False)
     def _table_to_html(self, table):
         table_html = "<table>"
         rows = [
@@ -72,6 +75,7 @@ class AzureFormRecognizerClient:
         table_html += "</table>"
         return table_html
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def begin_analyze_document_from_url(
         self, source_url: str, use_layout: bool = True, paragraph_separator: str = ""
     ):

cwyodmodules/batch/utilities/helpers/azure_identity_helper.py

@@ -1,8 +1,4 @@
-import os
 from azure.identity import (
-    ChainedTokenCredential,
-    ManagedIdentityCredential,
-    EnvironmentCredential,
     TokenCachePersistenceOptions,
     get_bearer_token_provider,
     DefaultAzureCredential
@@ -14,6 +10,7 @@ from opentelemetry.propagate import extract
 
 logger = getLogger("__main__")
 tracer = trace.get_tracer("__main__")
+
 class AzureIdentityHelper:
     """
     A helper class to provide a chained Azure token credential.
@@ -21,40 +18,12 @@ class AzureIdentityHelper:
     Token caching is configured for in-memory persistence.
     """
     def __init__(self):
-        # Configure in-memory token cache persistence
-        # For in-memory, unencrypted storage is typically allowed for simplicity during development.
-        # In production, especially with shared environments, consider the security implications.
-        client_secret_available =  os.getenv("AZURE_CLIENT_SECRET") is not None
         
-        token_cache_options = TokenCachePersistenceOptions(allow_unencrypted_storage=True)
+        token_cache_options = TokenCachePersistenceOptions(allow_unencrypted_storage=True)        
 
-        # Create individual credential instances
-        managed_identity_credential = ManagedIdentityCredential(
-            token_cache_persistence_options=token_cache_options
-        )
-        environment_credential = EnvironmentCredential(
+        self._credential = DefaultAzureCredential(
             token_cache_persistence_options=token_cache_options
         )
-        
-
-        # Create a chain of credentials
-        # The chain will try credentials in the order they are provided.
-        if client_secret_available:
-            logger.info("Using Environment Credential first with token cache persistence.")
-            self._credential = ChainedTokenCredential(
-                environment_credential,
-                managed_identity_credential
-            )
-        else:
-            logger.info("Using Managed Identity Credential first with token cache persistence.")
-            # self._credential = ChainedTokenCredential(
-
-            #     managed_identity_credential,
-            #     environment_credential
-            # )
-            self._credential = DefaultAzureCredential(
-                token_cache_persistence_options=token_cache_options
-            )
 
     def get_credential(self):
         """

cwyodmodules/batch/utilities/helpers/azure_postgres_helper.py

@@ -1,11 +1,13 @@
-import logging
 import psycopg2
 from psycopg2.extras import execute_values, RealDictCursor
 from ..helpers.azure_identity_helper import AzureIdentityHelper
 from .llm_helper import LLMHelper
 from .env_helper import EnvHelper
 
-logger = logging.getLogger("__main__")
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 
 class AzurePostgresHelper:
@@ -15,6 +17,7 @@ class AzurePostgresHelper:
         self.azure_identity_helper = AzureIdentityHelper()
         self.conn = None
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def _create_search_client(self):
         """
         Establishes a connection to Azure PostgreSQL using AAD authentication.
@@ -44,6 +47,7 @@ class AzurePostgresHelper:
             logger.error(f"Error establishing a connection to PostgreSQL: {e}")
             raise
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def get_search_client(self):
         """
         Provides a reusable database connection.
@@ -52,6 +56,7 @@ class AzurePostgresHelper:
             self.conn = self._create_search_client()
         return self.conn
 
+    @logger.trace_function(log_args=False, log_result=False)
     def get_vector_store(self, embedding_array):
         """
         Fetches search indexes from PostgreSQL based on an embedding vector.
@@ -80,6 +85,7 @@ class AzurePostgresHelper:
         finally:
             conn.close()
 
+    @logger.trace_function(log_args=False, log_result=log_result)
     def create_vector_store(self, documents_to_upload):
         """
         Inserts documents into the `vector_store` table in batch mode.
@@ -123,6 +129,7 @@ class AzurePostgresHelper:
         finally:
             conn.close()
 
+    @logger.trace_function(log_args=False, log_result=log_result)
     def get_files(self):
         """
         Fetches distinct titles from the PostgreSQL database.
@@ -155,6 +162,7 @@ class AzurePostgresHelper:
         finally:
             conn.close()
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def delete_documents(self, ids_to_delete):
         """
         Deletes documents from the PostgreSQL database based on the provided ids.
@@ -202,6 +210,7 @@ class AzurePostgresHelper:
         finally:
             conn.close()
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def perform_search(self, title):
         """
         Fetches search results from PostgreSQL based on the title.
@@ -228,6 +237,7 @@ class AzurePostgresHelper:
         finally:
             conn.close()
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def get_unique_files(self):
         """
         Fetches unique titles from PostgreSQL.
@@ -252,6 +262,7 @@ class AzurePostgresHelper:
         finally:
             conn.close()
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def search_by_blob_url(self, blob_url):
         """
         Fetches unique titles from PostgreSQL based on a given blob URL.
@@ -278,6 +289,7 @@ class AzurePostgresHelper:
         finally:
             conn.close()
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def store_with_lightrag(self, documents_to_upload):
         """
         Stores documents using LightRAG for enhanced vector and text storage capabilities.