data-science-document-ai 1.52.1__tar.gz → 1.53.0__tar.gz

{data_science_document_ai-1.52.1 → data_science_document_ai-1.53.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.52.1
+Version: 1.53.0
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.52.1 → data_science_document_ai-1.53.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-science-document-ai"
-version = "1.52.1"
+version = "1.53.0"
 description = "\"Document AI repo for data science\""
 authors = ["Naomi Nguyen <naomi.nguyen@forto.com>", "Kumar Rajendrababu <kumar.rajendrababu@forto.com>", "Igor Tonko <igor.tonko@forto.com>", "Osman Demirel <osman.demirel@forto.com>"]
 packages = [

{data_science_document_ai-1.52.1 → data_science_document_ai-1.53.0}/src/pdf_processing.py

@@ -250,6 +250,7 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
         prompt += "\nFor each field, provide the page number where the information was found. The page numbering starts from 0."
 
     tasks = []
+    semaphore = asyncio.Semaphore(50)
     # Process in chunks if number of pages exceeds threshold and Process all chunks concurrently
     for chunk in (
         split_pdf_into_chunks(file_content, chunk_size=params["chunk_size"])
@@ -257,8 +258,8 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
         else [file_content]
     ):
         tasks.append(
-            process_chunk_with_retry(
-                chunk, prompt, response_schema, llm_client, input_doc_type
+            process_chunk_with_semaphore(
+                semaphore, chunk, prompt, response_schema, llm_client, input_doc_type
             )
         )
 
@@ -270,6 +271,25 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
         return llm_prediction_to_tuples(results[0], number_of_pages=number_of_pages)
 
 
+async def process_chunk_with_semaphore(
+    semaphore,
+    chunk_content,
+    prompt,
+    response_schema,
+    llm_client,
+    input_doc_type,
+):
+    """Process a chunk with a semaphore to limit concurrency."""
+    async with semaphore:
+        return await process_chunk_with_retry(
+            chunk_content,
+            prompt,
+            response_schema,
+            llm_client,
+            input_doc_type,
+        )
+
+
 async def process_chunk_with_retry(
     chunk_content, prompt, response_schema, llm_client, input_doc_type, retries=2
 ):

{data_science_document_ai-1.52.1 → data_science_document_ai-1.53.0}/src/postprocessing/postprocess_partner_invoice.py

@@ -1,4 +1,6 @@
 """This module contains the postprocessing functions for the partner invoice."""
+from collections import defaultdict
+
 from rapidfuzz import fuzz, process
 
 from src.io import logger
@@ -143,6 +145,20 @@ def update_recipient_and_vendor(aggregated_data, is_recipient_forto):
     ] = "Dasbachstraße 15, 54292 Trier, Germany"
 
 
+def select_unique_bank_account(bank_account):
+    # Select the unique bank account if multiple are present
+    if isinstance(bank_account, list) and bank_account:
+        best = defaultdict(lambda: None)
+
+        for item in bank_account:
+            dv = item["documentValue"]
+            if best[dv] is None or item["page"] < best[dv]["page"]:
+                best[dv] = item
+
+        unique = list(best.values())
+        return unique
+
+
 async def process_partner_invoice(params, aggregated_data, document_type_code):
     """Process the partner invoice data."""
     # Post process bundeskasse invoices
@@ -150,6 +166,11 @@ async def process_partner_invoice(params, aggregated_data, document_type_code):
         post_process_bundeskasse(aggregated_data)
         return
 
+    if "bankAccount" in aggregated_data:
+        aggregated_data["bankAccount"] = select_unique_bank_account(
+            aggregated_data["bankAccount"]
+        )
+
     line_items = aggregated_data.get("lineItem", [])
     # Add debug logging
     logger.info(f"Processing partnerInvoice with {len(line_items)} line items")