data-science-document-ai 1.42.5__py3-none-any.whl → 1.57.0__py3-none-any.whl

{data_science_document_ai-1.42.5.dist-info → data_science_document_ai-1.57.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.42.5
+Version: 1.57.0
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com
@@ -38,7 +38,7 @@ Requires-Dist: pdf2image (>=1.17.0,<2.0.0)
 Requires-Dist: pgzip (>=0.3.5,<0.4.0)
 Requires-Dist: pyarrow (==16.1.0)
 Requires-Dist: pymupdf (>=1.23.26,<2.0.0)
-Requires-Dist: pypdf2 (>=3.0.1,<4.0.0)
+Requires-Dist: pypdf (>=6.1.2,<7.0.0)
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: python-multipart (>=0.0.7,<0.0.8)
 Requires-Dist: rapidfuzz (>=3.12.2,<4.0.0)

data_science_document_ai-1.57.0.dist-info/RECORD

@@ -0,0 +1,60 @@
+src/constants.py,sha256=Mr5HcoMSFQlXY24shJ1EpNKtCBJP_Ai0LZj-0i3BVGk,2845
+src/constants_sandbox.py,sha256=Iu6HdjCoNSmOX0AwoL9qUQkhq_ZnIN5U9e-Q2UfNuGc,547
+src/docai.py,sha256=dHuR0ehVjUi1CnoNvdp_yxJtpU_HFXqAZ61ywdz7BEo,5655
+src/docai_processor_config.yaml,sha256=ZewXqbyiftzmVAaS08BoNp5trY6WXx3HMWDfPwmKfaI,256
+src/excel_processing.py,sha256=TRgAzSHvL1WKbUgjHtpXL701bPhiWGH7kk3S6e1UPaA,3074
+src/io.py,sha256=rYjXVLlriEacw1uNuPIYhg12bXNu48Qs9GYMY2YcVTE,5563
+src/llm.py,sha256=a7UYA4ITUNjzct_2fHgM-bma_XWc28VC0FV71g9tnUI,7137
+src/log_setup.py,sha256=RhHnpXqcl-ii4EJzRt47CF2R-Q3YPF68tepg_Kg7tkw,2895
+src/pdf_processing.py,sha256=Fx-Glb9niEUU3WUCrBZ02ZYV-E2vWoUM0ifN7-0A1Q4,19961
+src/postprocessing/common.py,sha256=dagAg0hZGuZc03bXdfOolxekewMEVUfz917IGCiAtWI,26118
+src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
+src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
+src/postprocessing/postprocess_partner_invoice.py,sha256=WuaTQK5D09dV_QNrh29ZoKX9IvQn2Ub-WnAMyRjCsvI,14240
+src/prompts/library/arrivalNotice/other/placeholders.json,sha256=1vzly1amgyKt3jr2JJQbb24kNZsnI289iduvoUo5dJU,3061
+src/prompts/library/arrivalNotice/other/prompt.txt,sha256=QNuU-BvMA8VbdupVNapad4O3WmCotH5cKNxImRMbKDk,2906
+src/prompts/library/bookingConfirmation/evergreen/placeholders.json,sha256=5efq6b--KGWeqGbvASZFTqXJgUEAvsC-0ljo-q0Lhew,5855
+src/prompts/library/bookingConfirmation/evergreen/prompt.txt,sha256=OxNfXZaWppwsFMprthzJpOOr8ApQL4KYEmlu9fSUvxk,3485
+src/prompts/library/bookingConfirmation/hapag-lloyd/placeholders.json,sha256=5efq6b--KGWeqGbvASZFTqXJgUEAvsC-0ljo-q0Lhew,5855
+src/prompts/library/bookingConfirmation/hapag-lloyd/prompt.txt,sha256=I9x52U9Kt3vwiQK96KnaAhD48rEjL47jS0JDm3QT27k,5025
+src/prompts/library/bookingConfirmation/maersk/placeholders.json,sha256=5efq6b--KGWeqGbvASZFTqXJgUEAvsC-0ljo-q0Lhew,5855
+src/prompts/library/bookingConfirmation/maersk/prompt.txt,sha256=S-C5cq8AkEoGKilCO0XiXLZXgZPwz9udQOTm557GG64,3984
+src/prompts/library/bookingConfirmation/msc/placeholders.json,sha256=5efq6b--KGWeqGbvASZFTqXJgUEAvsC-0ljo-q0Lhew,5855
+src/prompts/library/bookingConfirmation/msc/prompt.txt,sha256=bojE6BytnEoQfdXrQebaXYTToDF1Fbyn4YdIGMke2Jo,5463
+src/prompts/library/bookingConfirmation/oocl/placeholders.json,sha256=LqjzD-8LkX9hAq3eOBMwit6tLrSLmVMUXTIyhBEaYxk,6037
+src/prompts/library/bookingConfirmation/oocl/prompt.txt,sha256=pCsj2BNnP-_kwgUEDt8IehO-tyMv6qeD5nyIzXJL3c0,2925
+src/prompts/library/bookingConfirmation/other/placeholders.json,sha256=LqjzD-8LkX9hAq3eOBMwit6tLrSLmVMUXTIyhBEaYxk,6037
+src/prompts/library/bookingConfirmation/other/prompt.txt,sha256=-629upv9-ciO6eG3A0_2TTjy7iLlInMsmQfSwAukjLg,4919
+src/prompts/library/bookingConfirmation/yangming/placeholders.json,sha256=LqjzD-8LkX9hAq3eOBMwit6tLrSLmVMUXTIyhBEaYxk,6037
+src/prompts/library/bookingConfirmation/yangming/prompt.txt,sha256=gySDhfRdQHy3IIomOR3qwY49wlO63Xw73GUSPrEkkr4,3990
+src/prompts/library/bundeskasse/other/placeholders.json,sha256=7xKzi_ypkIICO9nrEl45W9G7-h33uWVRVWnpg2b5lUg,4288
+src/prompts/library/bundeskasse/other/prompt.txt,sha256=miNYoqRZEd6Z1LNisTahX1-tenzr5kEpRA6gvPH7NCw,3316
+src/prompts/library/commercialInvoice/other/placeholders.json,sha256=zUK2mg9MnHiEQRYF6VgTiUiq68WGy5f7_4qL63CWyR0,4700
+src/prompts/library/commercialInvoice/other/prompt.txt,sha256=CJapcVrmcvynJUanETDklkzU-0N9hHdhq5wL4MK7OIY,2683
+src/prompts/library/customsAssessment/other/placeholders.json,sha256=scIV--C9HNWAQbU9zEz3GT_FoAvJqbfuY85YUtt7t-Q,3850
+src/prompts/library/customsAssessment/other/prompt.txt,sha256=z3FuoHZ588Pz1WBJDW7ISAC3J6n7hPJCcS92CdHDTFw,2494
+src/prompts/library/customsInvoice/other/placeholders.json,sha256=BnWYtl4sPooTHb_EHRIlrPawBrfHI8_QVas8zytbqyY,12172
+src/prompts/library/customsInvoice/other/prompt.txt,sha256=hUBDhocFdHTiWdEPgEE8yKHqpIYOfOj-j9CvZd-3YZc,9941
+src/prompts/library/deliveryOrder/other/placeholders.json,sha256=j-9F4V3yDg4610PPsOwU3oOj_S9vAvAB9Ix155WGIwc,3827
+src/prompts/library/deliveryOrder/other/prompt.txt,sha256=RD076vq0x0IjoEVQfh-G0u4nxITCpgKZGrwMlR9YAvk,2695
+src/prompts/library/draftMbl/other/placeholders.json,sha256=Gn8kQ8cMmrzRGLSFH7_8wO1_j2jxhqHd4zeivZP2SjU,4304
+src/prompts/library/draftMbl/other/prompt.txt,sha256=4RjlGT2OFmcBCUJhuCnO9GtmCn3vVesXHi_ml2g3dK8,2386
+src/prompts/library/finalMbL/other/placeholders.json,sha256=Gn8kQ8cMmrzRGLSFH7_8wO1_j2jxhqHd4zeivZP2SjU,4304
+src/prompts/library/finalMbL/other/prompt.txt,sha256=cyeKjK94sepqXiLEeZKB4VpmT0-nqXALP4dih-B67M8,2386
+src/prompts/library/packingList/other/placeholders.json,sha256=cGUUvEFoi4Lm0BAiyD29KbNFbUgzO1s7eit_qK3F0ig,4478
+src/prompts/library/packingList/other/prompt.txt,sha256=6Q9d0KBG6YWmNtzFivvmtQmitaUE2jytfwwc5YwsUgQ,2872
+src/prompts/library/partnerInvoice/other/placeholders.json,sha256=NX6ADT4gxLpP90uoNCYDbmfBvROxxVWRKK0lRFy1n9s,10897
+src/prompts/library/partnerInvoice/other/prompt.txt,sha256=A3nw6QfraU1N6Aui4TC7eFofG3rUyo9cz8Ha1iQbMpU,8141
+src/prompts/library/postprocessing/port_code/placeholders.json,sha256=2TiXf3zSzrglOMPtDOlCntIa5RSvyZQAKG2-IgrCY5A,22
+src/prompts/library/postprocessing/port_code/prompt_port_code.txt,sha256=--1wunSqEr2ox958lEhjO-0JFBfOLzA3qfKYIzG_Iok,884
+src/prompts/library/preprocessing/carrier/placeholders.json,sha256=tQeVDtvembhVqvel9vGoy4qcKp1hOvg-bLCgZRdQj0g,192
+src/prompts/library/preprocessing/carrier/prompt.txt,sha256=NLvRZQCZ6aWC1yTr7Q93jK5z7Vi_b4HBaiFYYnIsO-w,134
+src/prompts/library/shippingInstruction/other/placeholders.json,sha256=eK4AeMfORkGMWVYcqH7NjB56Zb4swHTvcQD5UQbTryg,6374
+src/prompts/library/shippingInstruction/other/prompt.txt,sha256=CbrqlKMtB-sVY-8E460KP1KNmz169YVPMrH3-uEldPg,2135
+src/prompts/prompt_library.py,sha256=VJWHeXN-s501C2GiidIIvQQuZdU6T1R27hE2dKBiI40,2555
+src/setup.py,sha256=8-vZWjC8Iwa3xxdk3iR4412VCjtNtgzVqkXcFon7UBE,7309
+src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
+src/utils.py,sha256=Ow5_Jals88o8mbZ1BoHfZpHZoCfig_UQb5aalH-mpWE,17278
+data_science_document_ai-1.57.0.dist-info/METADATA,sha256=CnYlyCERc0j4WniG4qrEYOKLhcTtEXWZw6nQ5XJEQeE,2152
+data_science_document_ai-1.57.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+data_science_document_ai-1.57.0.dist-info/RECORD,,

src/constants.py

@@ -20,12 +20,16 @@ project_parameters = {
     # Fuzzy lookup
     "g_model_fuzzy_lookup_folder": "fuzzy_lookup",
     "item_code_lookup": "line_item_kvp_table.json",
+    "intermodal_partners": "intermodal_partners.json",
     "invoice_classification_lookup": "invoice_classification.json",
     "reverse_charge_sentence_lookup": "reverse_charge_sentences.json",
     # Fuzzy logic params
-    "fuzzy_threshold_item_code": 70,
+    "fuzzy_threshold_item_code": 92,
     "fuzzy_threshold_reverse_charge": 80,
     "fuzzy_threshold_invoice_classification": 70,
+    # Chunking params
+    "chunk_size": 1,  # page (do not change this without changing the page number logic)
+    "chunk_after": 10,  # pages
     # Big Query
     "g_ai_gbq_db_schema": "document_ai",
     "g_ai_gbq_db_table_out": "document_ai_api_calls_v1",
@@ -33,6 +37,8 @@ project_parameters = {
     # models metadata (confidence),
     "g_model_data_folder": "models",
     "local_model_data_folder": "data",
+    "if_use_docai": False,
+    "if_use_llm": True,  # Keep it always True
     "released_doc_types": {
         "bookingConfirmation",
         "packingList",
@@ -47,25 +53,6 @@ project_parameters = {
         "customsInvoice",
         "bundeskasse",
     },
-    "model_selector": {
-        "stable": {
-            "bookingConfirmation": 1,
-            "packingList": 0,
-            "commercialInvoice": 0,
-            "finalMbL": 0,
-            "draftMbl": 0,
-            "arrivalNotice": 0,
-            "shippingInstruction": 0,
-            "customsAssessment": 0,
-            "deliveryOrder": 0,
-            "partnerInvoice": 0,
-        },
-        "beta": {
-            "bookingConfirmation": 0,
-        },
-    },
-    # this is the model selector for the model to be used from the model_config.yaml
-    # file based on the environment, 0 mean the first model in the list
     # LLM model parameters
     "gemini_params": {
         "temperature": 0,
@@ -83,23 +70,15 @@ project_parameters = {
         "seed": 42,
         "model_id": "gemini-2.5-flash",
     },
-    # Key to combine the LLM results with the Doc Ai results
-    "key_to_combine": {
-        "bookingConfirmation": ["transportLegs"],
-        "finalMbL": ["containers"],
-        "draftMbl": ["containers"],
-        "customsAssessment": ["containers"],
-        "packingList": ["skuData"],
-        "commercialInvoice": ["skus"],
-        "shippingInstruction": ["containers"],
-        "partnerInvoice": ["lineItem"],
-        "customsInvoice": ["lineItem"],
-        "bundeskasse": ["lineItem"],
-    },
 }
 
 # Hardcoded rules for data points formatting that can't be based on label name alone
 formatting_rules = {
-    "bookingConfirmation": {"pickUpTerminal": "depot", "gateInTerminal": "terminal"},
+    "bookingConfirmation": {
+        "pickUpDepotCode": "depot",
+        "dropOffDepotCode": "depot",
+        "gateInTerminalCode": "terminal",
+        "pickUpTerminalCode": "terminal",
+    },
     "deliveryOrder": {"pickUpTerminal": "terminal", "EmptyContainerDepot": "depot"},
 }

src/docai_processor_config.yaml

@@ -7,72 +7,3 @@ model_config:
           display_name: "doc_cap_bookingConfirmation"
           author: "reet.kanjilal@forto.com"
           created_date: ""
-      - id: "3c280b11bdb3ed89"
-        details:
-          display_name: "doc_cap_BC_mlg"
-          author: "igor.tonko@forto.com"
-          created_date: ""
-
-    packingList:
-      - id: "d967005bd9d45aeb"
-        details:
-          display_name: "doc_cap_packingList"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
-    commercialInvoice:
-      - id: "7d37236207f75758"
-        details:
-          display_name: "doc_cap_commercialInvoice"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
-    finalMbL:
-      - id: "1eda2f22d64b1b89"
-        details:
-          display_name: "doc_cap_finalMbL"
-          author: "igor.tonko@forto.com"
-          created_date: ""
-
-    draftMbl:
-      - id: "1eda2f22d64b1b89"
-        details:
-          display_name: "doc_cap_finalMbL"
-          author: "igor.tonko@forto.com"
-          created_date: ""
-
-    shippingInstruction:
-      - id: "c77a0a515d99a8ba"
-        details:
-          display_name: "doc_cap_shippingInstruction"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
-    arrivalNotice:
-      - id: "748b2e2b9161dcf3"
-        details:
-          display_name: "doc_cap_arrivalNotice"
-          author: "osman.demirel@forto.com"
-          created_date: ""
-
-    customsAssessment:
-      - id: "c464a18d82fad9be"
-        details:
-          display_name: "doc_cap_customsAssessment"
-          author: "igor.tonko@forto.com"
-          created_date: ""
-
-    deliveryOrder:
-      - id: "2245a72c7a5dbf5f"
-        details:
-          display_name: "doc_cap_releaseNote"
-          author: "igor.tonko@forto.com"
-          created_date: ""
-
-  beta:
-    bookingConfirmation:
-      - id: "3c280b11bdb3ed89"
-        details:
-          display_name: "doc_cap_BC_mlg"
-          author: "igor.tonko@forto.com"
-          created_date: ""

src/excel_processing.py

@@ -2,22 +2,21 @@
 # flake8: noqa: E402
 import logging
 
-from src.postprocessing.common import llm_prediction_to_tuples
+from ddtrace import tracer
 
 logger = logging.getLogger(__name__)
 
 import asyncio
-import json
 
 import numpy as np
 import pandas as pd
 
-from src.llm import prompt_excel_extraction
-from src.utils import generate_schema_structure, get_excel_sheets
+from src.prompts.prompt_library import prompt_library
+from src.utils import estimate_page_count, get_excel_sheets
 
 
 async def extract_data_from_sheet(
-    params, sheet_name, sheet, response_schema, doc_type=None
+    llm_client, sheet_name, sheet, response_schema, doc_type=None
 ):
     logger.info(f"Processing sheet: {sheet_name}")
     excel_content = pd.DataFrame(sheet.values).dropna(how="all", axis=1)
@@ -29,11 +28,14 @@ async def extract_data_from_sheet(
     )
 
     # Prompt for the LLM JSON
-    prompt_docai = prompt_excel_extraction(worksheet)
+    prompt = prompt_library.library[doc_type]["other"]["prompt"]
+
+    # Join the worksheet content with the prompt
+    prompt = worksheet + "\n" + prompt
 
     try:
-        result = await params["LlmClient"].get_unified_json_genai(
-            prompt_docai,
+        result = await llm_client.get_unified_json_genai(
+            prompt,
             response_schema=response_schema,
             doc_type=doc_type,
         )
@@ -49,6 +51,7 @@ async def extract_data_from_excel(
     input_doc_type,
     file_content,
     mime_type,
+    llm_client,
 ):
     """Extract data from the Excel file.
 
@@ -57,6 +60,7 @@ async def extract_data_from_excel(
         input_doc_type (str): The type of the document.
         file_content (bytes): The content of the Excel file to process.
         mime_type (str): The MIME type of the file.
+        llm_client: The LLM client to use for data extraction.
 
     Returns:
         formatted_data (list): A list of dictionaries containing the extracted data.
@@ -65,15 +69,24 @@ async def extract_data_from_excel(
 
     """
     # Generate the response structure
-    response_schema = generate_schema_structure(params, input_doc_type)
+    response_schema = prompt_library.library[input_doc_type]["other"]["placeholders"]
 
     # Load the Excel file and get ONLY the "visible" sheet names
     sheets, workbook = get_excel_sheets(file_content, mime_type)
 
+    # Track the number of sheets in dd-trace
+    span = tracer.current_span()
+    if span:
+        estimated_page_counts = [
+            estimate_page_count(workbook[sheet]) for sheet in sheets
+        ]
+        est_page_count = sum(estimated_page_counts)
+        span.set_metric("est_page_count", est_page_count)
+
     # Excel files may contain multiple sheets. Extract data from each sheet
     sheet_extract_tasks = [
         extract_data_from_sheet(
-            params,
+            llm_client,
             sheet_name,
             workbook[sheet_name],
             response_schema,
@@ -83,7 +96,4 @@ async def extract_data_from_excel(
     ]
     extracted_data = {k: v for k, v in await asyncio.gather(*sheet_extract_tasks)}
 
-    # Convert LLM prediction dictionary to tuples of (value, page_number).
-    extracted_data = llm_prediction_to_tuples(extracted_data)
-
-    return extracted_data, extracted_data, params["gemini_params"]["model_id"]
+    return extracted_data, extracted_data, llm_client.model_id

src/io.py

@@ -156,4 +156,27 @@ def download_dir_from_bucket(bucket, directory_cloud, directory_local) -> bool:
     return result
 
 
+def bq_logs(data_to_insert, params):
+    """Insert logs into Google BigQuery.
+
+    Args:
+        data_to_insert (list): The data to insert into BigQuery.
+        params (dict): The parameters dictionary.
+    """
+    # Use the pre-initialized BigQuery client
+    bq_client = params["bq_client"]
+    # Get the table string
+    table_string = f"{params['g_ai_project_name']}.{params['g_ai_gbq_db_schema']}.{params['g_ai_gbq_db_table_out']}"
+
+    logger.info(f"Log table: {table_string}")
+    # Insert the rows into the table
+    insert_logs = bq_client.insert_rows_json(table_string, data_to_insert)
+
+    # Check if there were any errors inserting the rows
+    if not insert_logs:
+        logger.info("New rows have been added.")
+    else:
+        logger.info("Errors occurred while inserting rows: ", insert_logs)
+
+
 # type: ignore

src/llm.py

@@ -201,33 +201,4 @@ class LlmClient:
         return response
 
 
-def prompt_excel_extraction(excel_structured_text):
-    """Write a prompt to extract data from Excel files.
-
-    Args:
-        excel_structured_text (str): The structured text of the Excel file.
-
-    Returns:
-        prompt str: The prompt for common json.
-    """
-    prompt = f"""{excel_structured_text}
-
-    Task: Fill in the following dictionary from the information in the given in the above excel data.
-
-    Instructions:
-    - Do not change the keys of the following dictionary.
-    - The values should be filled in as per the schema provided below.
-    - If an entity contains a 'display_name', consider its properties as child data points in the below format.
-    {{'data-field': {{
-        'child-data-field': 'type -occurrence_type- description',
-          }}
-    }}
-    - The entity with 'display_name' can be extracted multiple times. Please pay attention to the occurrence_type.
-    - Ensure the schema reflects the hierarchical relationship.
-    - Use the data field description to understand the context of the data.
-
-    """
-    return prompt
-
-
 # pylint: enable=all