data-science-document-ai 1.53.0__tar.gz → 1.54.0__tar.gz

{data_science_document_ai-1.53.0 → data_science_document_ai-1.54.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.53.0
+Version: 1.54.0
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.53.0 → data_science_document_ai-1.54.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-science-document-ai"
-version = "1.53.0"
+version = "1.54.0"
 description = "\"Document AI repo for data science\""
 authors = ["Naomi Nguyen <naomi.nguyen@forto.com>", "Kumar Rajendrababu <kumar.rajendrababu@forto.com>", "Igor Tonko <igor.tonko@forto.com>", "Osman Demirel <osman.demirel@forto.com>"]
 packages = [

{data_science_document_ai-1.53.0 → data_science_document_ai-1.54.0}/src/constants.py

@@ -20,10 +20,11 @@ project_parameters = {
     # Fuzzy lookup
     "g_model_fuzzy_lookup_folder": "fuzzy_lookup",
     "item_code_lookup": "line_item_kvp_table.json",
+    "intermodal_partners": "intermodal_partners.json",
     "invoice_classification_lookup": "invoice_classification.json",
     "reverse_charge_sentence_lookup": "reverse_charge_sentences.json",
     # Fuzzy logic params
-    "fuzzy_threshold_item_code": 90,
+    "fuzzy_threshold_item_code": 92,
     "fuzzy_threshold_reverse_charge": 80,
     "fuzzy_threshold_invoice_classification": 70,
     # Chunking params

{data_science_document_ai-1.53.0 → data_science_document_ai-1.54.0}/src/pdf_processing.py

@@ -32,7 +32,6 @@ from src.postprocessing.postprocess_partner_invoice import (
 from src.prompts.prompt_library import prompt_library
 from src.utils import (
     extract_top_pages,
-    generate_schema_structure,
     get_pdf_page_count,
     get_processor_name,
     run_background_tasks,
@@ -250,7 +249,6 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
         prompt += "\nFor each field, provide the page number where the information was found. The page numbering starts from 0."
 
     tasks = []
-    semaphore = asyncio.Semaphore(50)
     # Process in chunks if number of pages exceeds threshold and Process all chunks concurrently
     for chunk in (
         split_pdf_into_chunks(file_content, chunk_size=params["chunk_size"])
@@ -258,8 +256,12 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
         else [file_content]
     ):
         tasks.append(
-            process_chunk_with_semaphore(
-                semaphore, chunk, prompt, response_schema, llm_client, input_doc_type
+            process_chunk_with_retry(
+                chunk,
+                prompt,
+                response_schema,
+                llm_client,
+                input_doc_type,
             )
         )
 
@@ -271,25 +273,6 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
         return llm_prediction_to_tuples(results[0], number_of_pages=number_of_pages)
 
 
-async def process_chunk_with_semaphore(
-    semaphore,
-    chunk_content,
-    prompt,
-    response_schema,
-    llm_client,
-    input_doc_type,
-):
-    """Process a chunk with a semaphore to limit concurrency."""
-    async with semaphore:
-        return await process_chunk_with_retry(
-            chunk_content,
-            prompt,
-            response_schema,
-            llm_client,
-            input_doc_type,
-        )
-
-
 async def process_chunk_with_retry(
     chunk_content, prompt, response_schema, llm_client, input_doc_type, retries=2
 ):

{data_science_document_ai-1.53.0 → data_science_document_ai-1.54.0}/src/postprocessing/postprocess_partner_invoice.py

@@ -188,15 +188,20 @@ async def process_partner_invoice(params, aggregated_data, document_type_code):
         reverse_charge_info["formattedValue"] = reverse_charge_value
         reverse_charge = aggregated_data.pop("reverseChargeSentence", None)
 
+    # Partner Name
+    partner_name = aggregated_data.get("vendorName", {}).get("documentValue", None)
+
     # Process everything in one go
-    processed_items = await process_line_items_batch(params, line_items, reverse_charge)
+    processed_items = await process_line_items_batch(
+        params, line_items, reverse_charge, partner_name
+    )
 
     # Update your main data structure
     aggregated_data["lineItem"] = processed_items
 
 
 async def process_line_items_batch(
-    params: dict, line_items: list[dict], reverse_charge=None
+    params: dict, line_items: list[dict], reverse_charge=None, partner_name=None
 ):
     """
     Processes all line items efficiently using a "Split-Apply-Combine" strategy.
@@ -234,23 +239,12 @@ async def process_line_items_batch(
 
     # Batch API Call for Embedding lookups
     if pending_line_items:
-        values_to_fetch = list(set(pending_line_items.values()))
-        logger.info(f"Mapping {len(values_to_fetch)} line items from Embedding API...")
-
-        # Await the batch response {"desc1": "code1", "desc2": "code2"}
-        api_results = await get_tms_mappings(
-            input_list=values_to_fetch, embedding_type="line_items"
-        )
+        code_map = await fetch_line_item_codes(pending_line_items, partner_name, params)
 
-        # Merge API results back into original list
         for index, desc in pending_line_items.items():
-            # Get result from API response, or None if API failed for that item
-            forto_code = api_results.get(desc)
-
-            # Update the original item
             line_items[index]["itemCode"] = {
                 "documentValue": desc,
-                "formattedValue": forto_code,  # Might be None if API failed
+                "formattedValue": code_map.get(desc),
                 "page": line_items[index]["lineItemDescription"].get("page"),
             }
 
@@ -344,12 +338,13 @@ def find_matching_lineitem(new_lineitem: str, kvp_dict: dict, threshold=90):
     return None
 
 
-async def associate_forto_item_code(line_item_data, params):
+async def associate_forto_item_code(line_item_data, params, partner_name=None):
     """
     Associates Forto item codes to a list of line item descriptions.
     Args:
         line_item_data (dict): A dictionary where keys are original descriptions and values are cleaned descriptions.
         params (dict): Parameters containing lookup data and thresholds.
+        partner_name (str, optional): The name of the partner for context in matching. Defaults to None.
 
     Returns:
         list: A list of dictionaries with 'description' and 'itemCode' keys.
@@ -371,14 +366,51 @@ async def associate_forto_item_code(line_item_data, params):
 
     # Batch API Call for Embedding lookups
     if pending_line_items:
-        api_results = await get_tms_mappings(
-            input_list=list(pending_line_items.values()),
-            embedding_type="line_items",
-        )
+        code_map = await fetch_line_item_codes(pending_line_items, partner_name, params)
 
-        # Merge API results back into original list
         for desc, f_desc in pending_line_items.items():
-            code = api_results.get(f_desc)
-            result.append({"description": desc, "itemCode": code})
+            result.append(
+                {
+                    "description": desc,
+                    "itemCode": code_map.get(f_desc),
+                }
+            )
+
+    return result
+
 
+async def fetch_line_item_codes(
+    pending_line_items: dict,
+    partner_name: str | None,
+    params: dict,
+):
+    """Returns: {original_description: mapped_code_or_None}"""
+    t_mode = (
+        find_matching_lineitem(
+            partner_name.upper(),
+            params["lookup_data"]["intermodal_partners"],
+            threshold=87,
+        )
+        if partner_name
+        else None
+    )
+
+    unique_descs = list(set(pending_line_items.values()))
+    logger.info(f"Mapping {len(unique_descs)} line items from Embedding API...")
+
+    # Build API input map
+    api_input_map = {
+        desc: f"{t_mode} - {desc}" if t_mode else desc for desc in unique_descs
+    }
+
+    api_results = await get_tms_mappings(
+        input_list=list(api_input_map.values()),
+        embedding_type="line_items",
+    )
+
+    # Normalize response back to original descriptions
+    result = {
+        original_desc: api_results.get(api_desc)
+        for original_desc, api_desc in api_input_map.items()
+    }
     return result

{data_science_document_ai-1.53.0 → data_science_document_ai-1.54.0}/src/setup.py

@@ -184,6 +184,9 @@ def setup_lookup_data(params):
     input_path_item_code = (
         f'{params["g_model_fuzzy_lookup_folder"]}/{params["item_code_lookup"]}'
     )
+    input_path_intermodal_partners = (
+        f'{params["g_model_fuzzy_lookup_folder"]}/{params["intermodal_partners"]}'
+    )
     input_path_invoice_classification = f'{params["g_model_fuzzy_lookup_folder"]}/{params["invoice_classification_lookup"]}'  # noqa: E501
     input_path_reverse_charge = f'{params["g_model_fuzzy_lookup_folder"]}/{params["reverse_charge_sentence_lookup"]}'
 
@@ -194,6 +197,9 @@ def setup_lookup_data(params):
         return json.loads(downloaded_data)
 
     data["item_code"] = download_json_from_bucket(input_path_item_code)
+    data["intermodal_partners"] = download_json_from_bucket(
+        input_path_intermodal_partners
+    )
     data["invoice_classification"] = download_json_from_bucket(
         input_path_invoice_classification
     )