wizit-context-ingestor 0.2.5b2__py3-none-any.whl → 0.2.5b3__py3-none-any.whl

wizit_context_ingestor/application/interfaces.py

@@ -92,7 +92,7 @@ class EmbeddingsManager(ABC):
         pass
 
     @abstractmethod
-    def index_documents(self, documents: List[Document]):
+    def index_documents(self, documents: list[Document]):
         """Index documents."""
         pass
 

wizit_context_ingestor/application/transcription_service.py

@@ -19,11 +19,13 @@ class TranscriptionService:
         self,
         ai_application_service: AiApplicationService,
         persistence_service: PersistenceService,
-        target_language: str = 'es'
+        target_language: str = 'es',
+        transcription_additional_instructions: str = ''
     ):
         self.ai_application_service = ai_application_service
         self.persistence_service = persistence_service
         self.target_language = target_language
+        self.transcription_additional_instructions = transcription_additional_instructions
         self.chat_model = self.ai_application_service.load_chat_model()
 
     def parse_doc_page(self, document: ParsedDocPage) -> ParsedDocPage:
@@ -50,6 +52,7 @@ class TranscriptionService:
                         }]
                     ),
                 ]).partial(
+                    transcription_additional_instructions=self.transcription_additional_instructions,
                     format_instructions=transcription_output_parser.get_format_instructions()
                 )
                 model_with_structured_output = self.chat_model.with_structured_output(Transcription)

wizit_context_ingestor/data/prompts.py

@@ -22,6 +22,7 @@ TRANSCRIPTION RULES:
    - Include: footnotes, page numbers, bullet points, lists, captions
    - Preserve: bold, italic, underlined, and other text formatting using markdown
    - Mark unclear text as [unclear] or [illegible] with best guess in brackets
+    - Enclose all underlined content in <UnderlinedContent></UnderlinedContent> tags
 
 2. LANGUAGE REQUIREMENTS:
    - All transcribed content MUST be in the document's primary language
@@ -70,9 +71,15 @@ CRITICAL REMINDERS:
 - Maintain professional transcription standards
 - Complete transcription is mandatory
 
+<additional_instructions>
+    {transcription_additional_instructions}
+</additional_instructions>
+
+
 Generate the optimized transcription following these specifications:
 {format_instructions}
 
+
 """
 
 CONTEXT_CHUNKS_IN_DOCUMENT_SYSTEM_PROMPT = """

wizit_context_ingestor/infra/rag/chroma_embeddings.py

@@ -0,0 +1,132 @@
+from typing_extensions import Sequence
+from test.test_typing import CoolEmployee
+from langchain_core.documents import Document
+from langchain_chroma import Chroma
+from typing import List
+import logging
+from uuid import uuid4
+from ...application.interfaces import EmbeddingsManager
+
+# load_dotenv()
+
+logger = logging.getLogger(__name__)
+
+class ChromaEmbeddingsManager(EmbeddingsManager):
+
+    __slots__ = ("embeddings_model", "chroma_host", "collection_name", "metadata_tags")
+    def __init__(
+        self,
+        embeddings_model,
+        chroma_host,
+        collection_name: str,
+        metadata_tags: dict
+    ):
+        """
+        Initialize the ChromaEmbeddingsManager.
+        Args:
+            embeddings_model: The embeddings model to use for generating vector embeddings
+                              (typically a LangChain embeddings model instance)
+            chroma_host: The Chroma host URL
+            collection_name: The Chroma collection name
+            metadata_tags: Tags to add as metadata to Chroma vector store
+
+        Raises:
+            Exception: If there's an error initializing the RedisEmbeddingsManager
+        """
+        self.collection_name = collection_name
+        self.embeddings_model = embeddings_model
+        self.chroma_host = chroma_host
+        self.metadata_tags_schema = []
+
+        for tag_key in metadata_tags:
+          self.metadata_tags_schema.append({
+              "type": "tag",
+              "name": tag_key
+          })
+
+        try:
+            self.chroma = Chroma(
+                collection_name=self.collection_name,
+                embedding_function=self.embeddings_model,
+                host=self.chroma_host,
+            )
+            logger.info("ChromaEmbeddingsManager initialized")
+        except Exception as e:
+          logger.error(f"Failed to initialize ChromaEmbeddingsManager: {str(e)}")
+          raise
+
+    def configure_vector_store(
+        self,
+        table_name: str = "",
+        vector_size: int = 768,
+        content_column: str = "document",
+        id_column: str = "id",
+        metadata_json_column: str = "cmetadata",
+        pg_record_manager: str = ""
+    ):
+        """Configure the vector store."""
+        pass
+
+    def init_vector_store(
+        self,
+        table_name: str = "",
+        content_column: str = "document",
+        metadata_json_column: str = "cmetadata",
+        id_column: str = "id",
+    ):
+        """Initialize the vector store."""
+        pass
+
+
+    def index_documents(self, documents: list[Document]):
+        """
+        Add documents to the vector store with their embeddings.
+
+        This method takes a list of Document objects, generates embeddings for them
+        using the embeddings model, and stores both the documents and their
+        embeddings in the PostgreSQL database.
+
+        Args:
+          docs: A list of LangChain Document objects to add to the vector store
+                Each Document should have page_content and metadata attributes
+                from langchain_core.documents import Document
+        Returns:
+          None
+
+        Raises:
+          Exception: If there's an error adding documents to the vector store
+        """
+        try:
+            logger.info(f"Indexing {len(documents)} documents in vector store")
+            self.chroma.add_documents(documents)
+        except Exception as e:
+            logger.error(f"Error indexing documents: {str(e)}")
+            raise
+
+    def get_documents_by_id(self, ids: list[str]):
+        """
+        Get document by ID from the vector store.
+        """
+        try:
+            return self.chroma.get_by_ids(ids)
+        except Exception as e:
+            logger.error(f"Error getting documents by ID: {str(e)}")
+            raise
+
+    def delete_documents_by_id(self, ids: list[str]):
+        """
+        Delete documents by ID from the vector store.
+        """
+        try:
+            self.chroma.delete(ids)
+        except Exception as e:
+            logger.error(f"Error deleting documents by ID: {str(e)}")
+            raise
+
+    def get_documents_keys_by_source_id(self, source_id: str):
+        """Get documents keys by source ID."""
+        pass
+
+    def delete_documents_by_source_id(self, source_id: str):
+        """Delete documents by source ID."""
+        pass

wizit_context_ingestor/main.py

@@ -14,8 +14,9 @@ class DeelabTranscribeManager:
         gcp_project_id: str,
         gcp_project_location: str,
         gcp_secret_name: str,
-        llm_model_id: str = "claude-3-5-sonnet-v2@20241022",
+        llm_model_id: str = "claude-sonnet-4@20250514",
         target_language: str = 'es',
+        transcription_additional_instructions: str = ''
     ):
         self.gcp_project_id = gcp_project_id
         self.gcp_project_location = gcp_project_location
@@ -23,6 +24,7 @@ class DeelabTranscribeManager:
         self.gcp_secret_name = gcp_secret_name
         self.llm_model_id = llm_model_id
         self.target_language = target_language
+        self.transcription_additional_instructions = transcription_additional_instructions
         self.gcp_sa_dict = self._get_gcp_sa_dict(gcp_secret_name)
         self.vertex_model = self._get_vertex_model()
 
@@ -55,7 +57,8 @@ class DeelabTranscribeManager:
             transcribe_document_service = TranscriptionService(
                 ai_application_service=self.vertex_model,
                 persistence_service=s3_persistence_service,
-                target_language=self.target_language
+                target_language=self.target_language,
+                transcription_additional_instructions=self.transcription_additional_instructions
             )
             parsed_pages, parsed_document = transcribe_document_service.process_document(file_key)
             origin_bucket_file_tags = s3_persistence_service.retrieve_file_tags(file_key, s3_origin_bucket_name)

{wizit_context_ingestor-0.2.5b2.dist-info → wizit_context_ingestor-0.2.5b3.dist-info}/METADATA

@@ -1,10 +1,11 @@
 Metadata-Version: 2.3
 Name: wizit-context-ingestor
-Version: 0.2.5b2
+Version: 0.2.5b3
 Summary: Contextual Rag with Cloud Solutions
 Requires-Dist: anthropic[vertex]>=0.66.0
 Requires-Dist: boto3>=1.40.23
 Requires-Dist: langchain-aws>=0.2.31
+Requires-Dist: langchain-chroma>=0.2.6
 Requires-Dist: langchain-experimental>=0.3.4
 Requires-Dist: langchain-google-vertexai>=2.0.28
 Requires-Dist: langchain-redis>=0.2.3

{wizit_context_ingestor-0.2.5b2.dist-info → wizit_context_ingestor-0.2.5b3.dist-info}/RECORD

@@ -2,10 +2,10 @@ wizit_context_ingestor/.DS_Store,sha256=c7hZ0C8v2hxprMlCgmvxXDl92phew3iSATJzE1yY
 wizit_context_ingestor/__init__.py,sha256=GQdqSrpsSS7mdbfIn-Osse4EI54PvqlDYeBZwCuuNNA,134
 wizit_context_ingestor/application/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wizit_context_ingestor/application/context_chunk_service.py,sha256=0nnn6vbxnLovoriu0f7EIqiAJA713Pd8L95QNK6fjnM,4916
-wizit_context_ingestor/application/interfaces.py,sha256=qz6XcA9K5THl7dBMyfOtqEuNV2hC6BwGEiULFaamDqc,3193
-wizit_context_ingestor/application/transcription_service.py,sha256=GnUoIP6gEAm3MZ1JgX0yIzcaZAARUsZYKLbfUkmC79A,4073
+wizit_context_ingestor/application/interfaces.py,sha256=W0qonE3t-S-zwAoKtDYc4oyW_GOILKVmrdy8LnC8MVI,3193
+wizit_context_ingestor/application/transcription_service.py,sha256=nYJ3pNdVumTeV0pjFrmLNrsj8ZdIfQczxdL7jpKuQmA,4323
 wizit_context_ingestor/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wizit_context_ingestor/data/prompts.py,sha256=wK4HjgbxMOMo0qKeqYs4ujJ0jZ1oDJ6Jw8kujByDdlY,6721
+wizit_context_ingestor/data/prompts.py,sha256=VG8SCMrp5CvhlKk08D-kvARggNtt-xhND6_PL2Xfk30,6906
 wizit_context_ingestor/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wizit_context_ingestor/domain/models.py,sha256=DV83PArMyh-VoUqnVF_ohcgStsk549ixdYw98B8o2GI,381
 wizit_context_ingestor/domain/services.py,sha256=0i9WwZ0ufBgnzNJ5dt8Iop9VLTeK_AqjcaH8p3Av26I,3347
@@ -14,18 +14,19 @@ wizit_context_ingestor/infra/aws_model.py,sha256=glIaewSdv6PDBXoCe6QgCUIzLCjtM7K
 wizit_context_ingestor/infra/persistence/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wizit_context_ingestor/infra/persistence/local_storage.py,sha256=sDFat-FMN123FUWZp_ztwoVjl0HrzChCDZmicFemy5o,1707
 wizit_context_ingestor/infra/persistence/s3_storage.py,sha256=HYO3gWNE64ECSYYrxrIi9-2jWv1vwwGEE5QX-ZqpOCs,4791
+wizit_context_ingestor/infra/rag/chroma_embeddings.py,sha256=MZls9JoessXm48dqY-an3zRDehO_j3FkWBDF9ls2RAU,4297
 wizit_context_ingestor/infra/rag/pg_embeddings.py,sha256=5m4R4GmwMU3C7AR3Je3nCdgO-2jyIaCG4QN9phGD68Q,8072
 wizit_context_ingestor/infra/rag/redis_embeddings.py,sha256=wlgSBedq_kcrZ3SF4vGVTWM0B350kkd8C894i4mMUA8,4828
 wizit_context_ingestor/infra/rag/semantic_chunks.py,sha256=tM6bSacBvu-VWb3VkxgQNrfskz3zFxOOAU23D2kZWD8,2255
 wizit_context_ingestor/infra/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wizit_context_ingestor/infra/secrets/aws_secrets_manager.py,sha256=1k_R_uzLabptiZ1GXAoqAgYpk8EykXIb-pUDdidUDJQ,1202
 wizit_context_ingestor/infra/vertex_model.py,sha256=Izpz2ZQ4Koh4PSrHAj_0iUv4Rx354SlUqqw-LrLXCOE,7256
-wizit_context_ingestor/main.py,sha256=x1bieggNs3uzLkNHXZk4fBz0fYlOyff1TGvjV3Af2yM,7966
+wizit_context_ingestor/main.py,sha256=dX0sQcbnpyFI0uUiU3g-qn5069xk2KILviK7NqrFOIk,8206
 wizit_context_ingestor/services/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
 wizit_context_ingestor/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wizit_context_ingestor/services/chunks.py,sha256=tQQsdsOscZWzqVY5WxVxr3ii62FOJ3nMARaJJz6CvjQ,2011
 wizit_context_ingestor/services/parse_doc.py,sha256=3CyZoGbiUfxbs0SXUWXjQevtusSzTBgvUVeNNSdxJLE,4491
 wizit_context_ingestor/services/pg_embeddings_manager.py,sha256=n1HOmu_Z_Z71H-rVAyJS3FdPKbBckm5W8_XethY8nuM,4998
-wizit_context_ingestor-0.2.5b2.dist-info/WHEEL,sha256=F3mArEuDT3LDFEqo9fCiUx6ISLN64aIhcGSiIwtu4r8,79
-wizit_context_ingestor-0.2.5b2.dist-info/METADATA,sha256=AaF5SD1HqVjrW0Px_xjhioJryJXPUh9Q9SO4QJtRbwo,3577
-wizit_context_ingestor-0.2.5b2.dist-info/RECORD,,
+wizit_context_ingestor-0.2.5b3.dist-info/WHEEL,sha256=-neZj6nU9KAMg2CnCY6T3w8J53nx1kFGw_9HfoSzM60,79
+wizit_context_ingestor-0.2.5b3.dist-info/METADATA,sha256=sexOso1mw8Gw3AEd5yD-F020VGjep0S-XLbNjJCB6LU,3616
+wizit_context_ingestor-0.2.5b3.dist-info/RECORD,,

{wizit_context_ingestor-0.2.5b2.dist-info → wizit_context_ingestor-0.2.5b3.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: uv 0.8.16
+Generator: uv 0.8.22
 Root-Is-Purelib: true
 Tag: py3-none-any