cwyodmodules 0.3.35__py3-none-any.whl → 0.3.36__py3-none-any.whl

cwyodmodules/batch/utilities/helpers/embedders/integrated_vectorization_embedder.py

@@ -1,3 +1,4 @@
+import azure.functions as func
 from .embedder_base import EmbedderBase
 from ..env_helper import EnvHelper
 from ..llm_helper import LLMHelper
@@ -6,9 +7,13 @@ from ...integrated_vectorization.azure_search_indexer import AzureSearchIndexer
 from ...integrated_vectorization.azure_search_datasource import AzureSearchDatasource
 from ...integrated_vectorization.azure_search_skillset import AzureSearchSkillset
 from ..config.config_helper import ConfigHelper
-import logging
 
-logger = logging.getLogger("__main__")
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
+
+bp_add_url_embeddings = func.Blueprint()
 
 
 class IntegratedVectorizationEmbedder(EmbedderBase):
@@ -16,13 +21,15 @@ class IntegratedVectorizationEmbedder(EmbedderBase):
         self.env_helper = env_helper
         self.llm_helper: LLMHelper = LLMHelper()
         logger.info("Initialized IntegratedVectorizationEmbedder.")
-
+     
+    @logger.trace_function(log_args=log_args, log_result=False)
     def embed_file(self, source_url: str, file_name: str = None):
         logger.info(
             f"Starting embed_file for source_url: {source_url}, file_name: {file_name}."
         )
         self.process_using_integrated_vectorization(source_url=source_url)
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def process_using_integrated_vectorization(self, source_url: str):
         logger.info(f"Starting integrated vectorization for source_url: {source_url}.")
         config = ConfigHelper.get_active_config_or_default()
@@ -46,6 +53,7 @@ class IntegratedVectorizationEmbedder(EmbedderBase):
             logger.error(f"Error processing {source_url}: {e}")
             raise e
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def reprocess_all(self):
         logger.info("Starting reprocess_all operation.")
         search_indexer = AzureSearchIndexer(self.env_helper)

cwyodmodules/batch/utilities/helpers/embedders/postgres_embedder.py

@@ -1,5 +1,4 @@
 import json
-import logging
 from typing import List
 
 from ...helpers.llm_helper import LLMHelper
@@ -15,7 +14,10 @@ from ..document_loading_helper import DocumentLoading
 from ..document_chunking_helper import DocumentChunking
 from ...common.source_document import SourceDocument
 
-logger = logging.getLogger("__main__")
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 
 class PostgresEmbedder(EmbedderBase):
@@ -33,6 +35,7 @@ class PostgresEmbedder(EmbedderBase):
             ext = processor.document_type.lower()
             self.embedding_configs[ext] = processor
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def embed_file(self, source_url: str, file_name: str):
         logger.info(f"Embedding file: {file_name} from source: {source_url}")
         file_extension = file_name.split(".")[-1].lower()
@@ -47,6 +50,7 @@ class PostgresEmbedder(EmbedderBase):
                 file_name, {"embeddings_added": "true"}
             )
 
+    @logger.trace_function(log_args=False, log_result=log_result)
     def __embed(
         self, source_url: str, file_extension: str, embedding_config: EmbeddingConfig
     ):
@@ -84,6 +88,7 @@ class PostgresEmbedder(EmbedderBase):
         else:
             logger.warning("No documents to upload.")
 
+    @logger.trace_function(log_args=False, log_result=False)
     def __convert_to_search_document(self, document: SourceDocument):
         logger.info(f"Generating embeddings for document ID: {document.id}")
         embedded_content = self.llm_helper.generate_embeddings(document.content)

cwyodmodules/batch/utilities/helpers/embedders/push_embedder.py

@@ -1,6 +1,5 @@
 import hashlib
 import json
-import logging
 from typing import List
 from urllib.parse import urlparse
 
@@ -19,7 +18,10 @@ from ..document_loading_helper import DocumentLoading
 from ..document_chunking_helper import DocumentChunking
 from ...common.source_document import SourceDocument
 
-logger = logging.getLogger("__main__")
+from logging_config import logger
+env_helper: EnvHelper = EnvHelper()
+log_args = env_helper.LOG_ARGS
+log_result = env_helper.LOG_RESULT
 
 
 class PushEmbedder(EmbedderBase):
@@ -40,6 +42,7 @@ class PushEmbedder(EmbedderBase):
             self.embedding_configs[ext] = processor
         logger.info("Document processors loaded")
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def embed_file(self, source_url: str, file_name: str):
         logger.info(f"Embedding file: {file_name} from URL: {source_url}")
         file_extension = file_name.split(".")[-1].lower()
@@ -55,6 +58,7 @@ class PushEmbedder(EmbedderBase):
                 file_name, {"embeddings_added": "true"}
             )
 
+    @logger.trace_function(log_args=False, log_result=log_result)
     def __embed(
         self, source_url: str, file_extension: str, embedding_config: EmbeddingConfig
     ):
@@ -101,6 +105,7 @@ class PushEmbedder(EmbedderBase):
         else:
             logger.warning("No documents to upload.")
 
+    @logger.trace_function(log_args=log_args, log_result=False)
     def __generate_image_caption(self, source_url):
         logger.info(f"Generating image caption for URL: {source_url}")
         model = self.env_helper.AZURE_OPENAI_VISION_MODEL
@@ -129,6 +134,7 @@ If the image is mostly text, use OCR to extract the text as it is displayed in t
         logger.info("Caption generation completed")
         return caption
 
+    @logger.trace_function(log_args=False, log_result=False)
     def __convert_to_search_document(self, document: SourceDocument):
         logger.info(f"Converting document ID {document.id} to search document format")
         embedded_content = self.llm_helper.generate_embeddings(document.content)
@@ -152,10 +158,12 @@ If the image is mostly text, use OCR to extract the text as it is displayed in t
             self.env_helper.AZURE_SEARCH_OFFSET_COLUMN: document.offset,
         }
 
+    @logger.trace_function(log_args=log_args, log_result=log_result)
     def __generate_document_id(self, source_url: str) -> str:
         hash_key = hashlib.sha1(f"{source_url}_1".encode("utf-8")).hexdigest()
         return f"doc_{hash_key}"
 
+    @logger.trace_function(log_args=False, log_result=False)
     def __create_image_document(
         self,
         source_url: str,

{cwyodmodules-0.3.35.dist-info → cwyodmodules-0.3.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cwyodmodules
-Version: 0.3.35
+Version: 0.3.36
 Summary: Add your description here
 Author-email: Patrik <patrikhartl@gmail.com>
 Classifier: Operating System :: OS Independent

{cwyodmodules-0.3.35.dist-info → cwyodmodules-0.3.36.dist-info}/RECORD

@@ -52,9 +52,9 @@ cwyodmodules/batch/utilities/helpers/config/default_employee_assistant_prompt.tx
 cwyodmodules/batch/utilities/helpers/config/embedding_config.py,sha256=9pCJxpsouln9dngjVHaKGFYP14PrwmSts_UFDytSiVk,950
 cwyodmodules/batch/utilities/helpers/embedders/embedder_base.py,sha256=z34LTNGzjiHr6_YWZ8NejUsX1KKYqXPWcqZ8mW_3CHI,699
 cwyodmodules/batch/utilities/helpers/embedders/embedder_factory.py,sha256=cJ9ZTXZEyOJ5TLB6pDsb9zvUZrJYY-LD40n0l1-qHcw,790
-cwyodmodules/batch/utilities/helpers/embedders/integrated_vectorization_embedder.py,sha256=sJr3ZT1ijn0and2OEgcQONkcHF_OxPchNFBTv3XKcZk,2856
-cwyodmodules/batch/utilities/helpers/embedders/postgres_embedder.py,sha256=aVXn0UxuaIeBfOvIqByqoLkZn1BOsiJpFxDhLymMAkA,4467
-cwyodmodules/batch/utilities/helpers/embedders/push_embedder.py,sha256=uh-JYx1C7MBpMYzsHFfPqZlALzj6GxkkSvSFAsL3NpI,8459
+cwyodmodules/batch/utilities/helpers/embedders/integrated_vectorization_embedder.py,sha256=SWm81GhH149_E9Nd2hi53a-e2uTRz6QIVFQ4dFakCs0,3213
+cwyodmodules/batch/utilities/helpers/embedders/postgres_embedder.py,sha256=fUYRui6whPtoWpaCrJeDvvm3C3F48kuktwwVS18h4kQ,4740
+cwyodmodules/batch/utilities/helpers/embedders/push_embedder.py,sha256=lUc89j5tCYh8EUGsLULnVkI5jM51rgxLwezylPYP__w,8931
 cwyodmodules/batch/utilities/integrated_vectorization/azure_search_datasource.py,sha256=rDwPgr-UCSYscc7hPOUJMwP09a9rX1MXAGf94TubdQo,2231
 cwyodmodules/batch/utilities/integrated_vectorization/azure_search_index.py,sha256=6jNz1nnIAPV57mMSWeKC-A6JQ5bLZW63rjgauF0wtY0,6579
 cwyodmodules/batch/utilities/integrated_vectorization/azure_search_indexer.py,sha256=rSDwIONB3BlpkXk6pm7-B2LpJ8WkZKJJ2WSr1YttVFk,3236
@@ -109,8 +109,8 @@ cwyodmodules/graphrag/query/generate.py,sha256=xBnZs2U9xFWtPk4AfAZgYKbHdcxNcIO6Q
 cwyodmodules/graphrag/query/graph_search.py,sha256=95h3ecSWx4864XgKABtG0fh3Nk8HkqJVzoCrO8daJ-Y,7724
 cwyodmodules/graphrag/query/types.py,sha256=1Iq1dp4I4a56_cuFjOZ0NTgd0A2_MpVFznp_czgt6cI,617
 cwyodmodules/graphrag/query/vector_search.py,sha256=9Gwu9LPjtoAYUU8WKqCvbCHAIg3dpk71reoYd1scLnQ,1807
-cwyodmodules-0.3.35.dist-info/licenses/LICENSE,sha256=UqBDTipijsSW2ZSOXyTZnMsXmLoEHTgNEM0tL4g-Sso,1150
-cwyodmodules-0.3.35.dist-info/METADATA,sha256=Fcv9mPqvwG7iufW4-v9ryWyuG_7L2DTFK9nb2jOQqLM,2002
-cwyodmodules-0.3.35.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-cwyodmodules-0.3.35.dist-info/top_level.txt,sha256=99RENLbkdRX-qpJvsxZ5AfmTL5s6shSaKOWYpz1vwzg,13
-cwyodmodules-0.3.35.dist-info/RECORD,,
+cwyodmodules-0.3.36.dist-info/licenses/LICENSE,sha256=UqBDTipijsSW2ZSOXyTZnMsXmLoEHTgNEM0tL4g-Sso,1150
+cwyodmodules-0.3.36.dist-info/METADATA,sha256=TLCeuQ_6a1nn0kX2gvSfD1UtIuZ53eP90zbcV8svSaE,2002
+cwyodmodules-0.3.36.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+cwyodmodules-0.3.36.dist-info/top_level.txt,sha256=99RENLbkdRX-qpJvsxZ5AfmTL5s6shSaKOWYpz1vwzg,13
+cwyodmodules-0.3.36.dist-info/RECORD,,