data-science-document-ai 1.51.0__tar.gz → 1.56.1__tar.gz

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.51.0
+Version: 1.56.1
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-science-document-ai"
-version = "1.51.0"
+version = "1.56.1"
 description = "\"Document AI repo for data science\""
 authors = ["Naomi Nguyen <naomi.nguyen@forto.com>", "Kumar Rajendrababu <kumar.rajendrababu@forto.com>", "Igor Tonko <igor.tonko@forto.com>", "Osman Demirel <osman.demirel@forto.com>"]
 packages = [

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/src/constants.py

@@ -20,10 +20,11 @@ project_parameters = {
     # Fuzzy lookup
     "g_model_fuzzy_lookup_folder": "fuzzy_lookup",
     "item_code_lookup": "line_item_kvp_table.json",
+    "intermodal_partners": "intermodal_partners.json",
     "invoice_classification_lookup": "invoice_classification.json",
     "reverse_charge_sentence_lookup": "reverse_charge_sentences.json",
     # Fuzzy logic params
-    "fuzzy_threshold_item_code": 90,
+    "fuzzy_threshold_item_code": 92,
     "fuzzy_threshold_reverse_charge": 80,
     "fuzzy_threshold_invoice_classification": 70,
     # Chunking params

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/src/pdf_processing.py

@@ -32,7 +32,6 @@ from src.postprocessing.postprocess_partner_invoice import (
 from src.prompts.prompt_library import prompt_library
 from src.utils import (
     extract_top_pages,
-    generate_schema_structure,
     get_pdf_page_count,
     get_processor_name,
     run_background_tasks,
@@ -258,7 +257,11 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
     ):
         tasks.append(
             process_chunk_with_retry(
-                chunk, prompt, response_schema, llm_client, input_doc_type
+                chunk,
+                prompt,
+                response_schema,
+                llm_client,
+                input_doc_type,
             )
         )
 

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/src/postprocessing/postprocess_partner_invoice.py

@@ -1,4 +1,6 @@
 """This module contains the postprocessing functions for the partner invoice."""
+from collections import defaultdict
+
 from rapidfuzz import fuzz, process
 
 from src.io import logger
@@ -143,6 +145,20 @@ def update_recipient_and_vendor(aggregated_data, is_recipient_forto):
     ] = "Dasbachstraße 15, 54292 Trier, Germany"
 
 
+def select_unique_bank_account(bank_account):
+    # Select the unique bank account if multiple are present
+    if isinstance(bank_account, list) and bank_account:
+        best = defaultdict(lambda: None)
+
+        for item in bank_account:
+            dv = item["documentValue"]
+            if best[dv] is None or item["page"] < best[dv]["page"]:
+                best[dv] = item
+
+        unique = list(best.values())
+        return unique
+
+
 async def process_partner_invoice(params, aggregated_data, document_type_code):
     """Process the partner invoice data."""
     # Post process bundeskasse invoices
@@ -150,6 +166,11 @@ async def process_partner_invoice(params, aggregated_data, document_type_code):
         post_process_bundeskasse(aggregated_data)
         return
 
+    if "bankAccount" in aggregated_data:
+        aggregated_data["bankAccount"] = select_unique_bank_account(
+            aggregated_data["bankAccount"]
+        )
+
     line_items = aggregated_data.get("lineItem", [])
     # Add debug logging
     logger.info(f"Processing partnerInvoice with {len(line_items)} line items")
@@ -167,15 +188,20 @@ async def process_partner_invoice(params, aggregated_data, document_type_code):
         reverse_charge_info["formattedValue"] = reverse_charge_value
         reverse_charge = aggregated_data.pop("reverseChargeSentence", None)
 
+    # Partner Name
+    partner_name = aggregated_data.get("vendorName", {}).get("documentValue", None)
+
     # Process everything in one go
-    processed_items = await process_line_items_batch(params, line_items, reverse_charge)
+    processed_items = await process_line_items_batch(
+        params, line_items, reverse_charge, partner_name
+    )
 
     # Update your main data structure
     aggregated_data["lineItem"] = processed_items
 
 
 async def process_line_items_batch(
-    params: dict, line_items: list[dict], reverse_charge=None
+    params: dict, line_items: list[dict], reverse_charge=None, partner_name=None
 ):
     """
     Processes all line items efficiently using a "Split-Apply-Combine" strategy.
@@ -213,23 +239,12 @@ async def process_line_items_batch(
 
     # Batch API Call for Embedding lookups
     if pending_line_items:
-        values_to_fetch = list(set(pending_line_items.values()))
-        logger.info(f"Mapping {len(values_to_fetch)} line items from Embedding API...")
-
-        # Await the batch response {"desc1": "code1", "desc2": "code2"}
-        api_results = await get_tms_mappings(
-            input_list=values_to_fetch, embedding_type="line_items"
-        )
+        code_map = await fetch_line_item_codes(pending_line_items, partner_name, params)
 
-        # Merge API results back into original list
         for index, desc in pending_line_items.items():
-            # Get result from API response, or None if API failed for that item
-            forto_code = api_results.get(desc)
-
-            # Update the original item
             line_items[index]["itemCode"] = {
                 "documentValue": desc,
-                "formattedValue": forto_code,  # Might be None if API failed
+                "formattedValue": code_map.get(desc),
                 "page": line_items[index]["lineItemDescription"].get("page"),
             }
 
@@ -285,11 +300,14 @@ def if_reverse_charge_sentence(sentence: str, params):
         return False
 
     # Check if the sentence is similar to any of the reverse charge sentences
-    _, is_reverse_charge = get_fuzzy_match_score(
-        sentence, reverse_charge_sentences, threshold
+    match, _ = get_fuzzy_match_score(
+        sentence, list(reverse_charge_sentences.keys()), threshold
     )
 
-    return is_reverse_charge
+    if match:
+        return reverse_charge_sentences[match]
+
+    return False
 
 
 def find_matching_lineitem(new_lineitem: str, kvp_dict: dict, threshold=90):
@@ -320,12 +338,13 @@ def find_matching_lineitem(new_lineitem: str, kvp_dict: dict, threshold=90):
     return None
 
 
-async def associate_forto_item_code(line_item_data, params):
+async def associate_forto_item_code(line_item_data, params, partner_name=None):
     """
     Associates Forto item codes to a list of line item descriptions.
     Args:
         line_item_data (dict): A dictionary where keys are original descriptions and values are cleaned descriptions.
         params (dict): Parameters containing lookup data and thresholds.
+        partner_name (str, optional): The name of the partner for context in matching. Defaults to None.
 
     Returns:
         list: A list of dictionaries with 'description' and 'itemCode' keys.
@@ -347,14 +366,51 @@ async def associate_forto_item_code(line_item_data, params):
 
     # Batch API Call for Embedding lookups
     if pending_line_items:
-        api_results = await get_tms_mappings(
-            input_list=list(pending_line_items.values()),
-            embedding_type="line_items",
-        )
+        code_map = await fetch_line_item_codes(pending_line_items, partner_name, params)
 
-        # Merge API results back into original list
         for desc, f_desc in pending_line_items.items():
-            code = api_results.get(f_desc)
-            result.append({"description": desc, "itemCode": code})
+            result.append(
+                {
+                    "description": desc,
+                    "itemCode": code_map.get(f_desc),
+                }
+            )
+
+    return result
 
+
+async def fetch_line_item_codes(
+    pending_line_items: dict,
+    partner_name: str | None,
+    params: dict,
+):
+    """Returns: {original_description: mapped_code_or_None}"""
+    t_mode = (
+        find_matching_lineitem(
+            partner_name.upper(),
+            params["lookup_data"]["intermodal_partners"],
+            threshold=87,
+        )
+        if partner_name
+        else None
+    )
+
+    unique_descs = list(set(pending_line_items.values()))
+    logger.info(f"Mapping {len(unique_descs)} line items from Embedding API...")
+
+    # Build API input map
+    api_input_map = {
+        desc: f"{t_mode} - {desc}" if t_mode else desc for desc in unique_descs
+    }
+
+    api_results = await get_tms_mappings(
+        input_list=list(api_input_map.values()),
+        embedding_type="line_items",
+    )
+
+    # Normalize response back to original descriptions
+    result = {
+        original_desc: api_results.get(api_desc)
+        for original_desc, api_desc in api_input_map.items()
+    }
     return result

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/src/prompts/library/customsInvoice/other/prompt.txt

@@ -48,7 +48,7 @@ Your role is to accurately extract specific entities from these invoices to supp
 - issueDate: The date the document was issued.
 - dueDate: The date by which the payment should be made. Do Not calculate dueDate based on issueDate or any other date. Extract it directly from the invoice.
 
-- lineItem: Details of each COGS and Customs line item on the invoice. Make sure to extract each amount and currency separately.
+- lineItem: Details of each COGS and Customs line item on the invoice from each page. Make sure to extract each amount and currency separately.
     - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 8 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
     - lineItemDescription: The name or description of the item. Usually, it will be a one line sentence.
     - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
@@ -92,6 +92,7 @@ Your role is to accurately extract specific entities from these invoices to supp
 
 IMPORTANT NOTE:
 - Ensure all extracted values are directly from the document. Do not make assumptions or modifications.
+- CustomSized invoices contain line items in a table format in a attached page. Table with headings Shipment ID, Partner Line Item Description, Quantity, Amount, and VAT. Extract all the line items from each tables from each page.
 - Do not normalize or modify any entity values.
 - Pay attention to the line item details and paymentInformation, as they may vary significantly across different invoices.
 

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/src/prompts/library/partnerInvoice/other/prompt.txt

@@ -46,7 +46,7 @@ Your role is to accurately extract specific entities from these invoices to supp
 
 - eta and etd: Few invoices contains same date for ARRIVED/DEPARTED or ETA/ETD. Extract it for both eta and etd.
 
-- lineItem: Details of each COGS and Customs line item on the invoice. Make sure to extract each amount and currency separately.
+- lineItem: Details of each COGS and Customs line item on the invoice from each page. Make sure to extract each amount and currency separately.
     - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 8 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
     - lineItemDescription: The name or description of the item. Usually, it will be a one line sentence.
     - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
@@ -82,11 +82,10 @@ Your role is to accurately extract specific entities from these invoices to supp
 
 IMPORTANT NOTE:
 - Ensure all extracted values are directly from the document. Do not make assumptions, modifications or calculations.
+- CustomSized invoices contain line items in a table format in the attached page. Table with headings Shipment ID, Partner Line Item Description, Quantity, Amount, and VAT. Extract all the line items from such tables from each page.
 - Do not split the quantity into different line items. e.g., if quantity is 2 or 2 CTR or 2 BIL, do not create 2 separate line items with quantity 1 each.
 - Do not normalize or modify any entity values.
 - Pay attention to the line item details and paymentInformation, as they may vary significantly across different invoices.
 
-PAY ATTENTION TO THE SGS MACO CUSTOMS SERVICE INVOICES:
-- invoices from SGS maco customs service,
-  - "Total Kosten excl. MwSt." is not the vatApplicableAmount
+
 <INSTRUCTIONS>

{data_science_document_ai-1.51.0 → data_science_document_ai-1.56.1}/src/setup.py

@@ -184,6 +184,9 @@ def setup_lookup_data(params):
     input_path_item_code = (
         f'{params["g_model_fuzzy_lookup_folder"]}/{params["item_code_lookup"]}'
     )
+    input_path_intermodal_partners = (
+        f'{params["g_model_fuzzy_lookup_folder"]}/{params["intermodal_partners"]}'
+    )
     input_path_invoice_classification = f'{params["g_model_fuzzy_lookup_folder"]}/{params["invoice_classification_lookup"]}'  # noqa: E501
     input_path_reverse_charge = f'{params["g_model_fuzzy_lookup_folder"]}/{params["reverse_charge_sentence_lookup"]}'
 
@@ -194,6 +197,9 @@ def setup_lookup_data(params):
         return json.loads(downloaded_data)
 
     data["item_code"] = download_json_from_bucket(input_path_item_code)
+    data["intermodal_partners"] = download_json_from_bucket(
+        input_path_intermodal_partners
+    )
     data["invoice_classification"] = download_json_from_bucket(
         input_path_invoice_classification
     )