data-science-document-ai 1.40.3__py3-none-any.whl → 1.41.0__py3-none-any.whl

{data_science_document_ai-1.40.3.dist-info → data_science_document_ai-1.41.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.40.3
+Version: 1.41.0
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

data_science_document_ai-1.41.0.dist-info/RECORD

@@ -0,0 +1,57 @@
+src/constants.py,sha256=TF_UblovdXZnKIb1lnyJwUqQncJCbzBVihoelI6foSU,3579
+src/constants_sandbox.py,sha256=Iu6HdjCoNSmOX0AwoL9qUQkhq_ZnIN5U9e-Q2UfNuGc,547
+src/docai.py,sha256=AepGdF3ZuSGkujLpewX393FgOBMy-e4sEudiGKho5EA,5280
+src/docai_processor_config.yaml,sha256=qOMmCIORpLQ_D-ytvejXxFvER0e0uGYuzPVdZBGv4Pc,2105
+src/excel_processing.py,sha256=wArdSxwxdgyj5WVgVTaWsVSmF7z5zK6rq-bUKGENmo4,2660
+src/io.py,sha256=IXz4wWqiHa9mnHNgtrC6X9M2lItYp9eu6rHCThUIh5c,3585
+src/llm.py,sha256=aEK3rL8XvY7CakvkOJQmcHpEKwZRd8PPrLrzHiO-GFk,7827
+src/log_setup.py,sha256=RhHnpXqcl-ii4EJzRt47CF2R-Q3YPF68tepg_Kg7tkw,2895
+src/pdf_processing.py,sha256=GNHQl_ryyVOHu3FK39XzPJzOCrn01NNW3E2HO43Ot_c,15836
+src/postprocessing/common.py,sha256=Vj_NohcgWZRCzipnPGeM-rg11wdDJ-wwCR12QeE6qOY,21451
+src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
+src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
+src/postprocessing/postprocess_partner_invoice.py,sha256=cM4te4qjOI_bXyrF8Zhb6X7eNf5aMKoRaPCFfqFv-98,11538
+src/prompts/library/bookingConfirmation/evergreen/placeholders.json,sha256=IpM9nmSPdyroliZfXB1-NDCjiHZX_Ff5BH7-scNhGqE,1406
+src/prompts/library/bookingConfirmation/evergreen/prompt.txt,sha256=5ivskCG831M2scW3oqQaoltXIyHV-n6DYUygWycXxjw,2755
+src/prompts/library/bookingConfirmation/hapag-lloyd/placeholders.json,sha256=hMPNt9s3LuxR85AxYy7bPcCDleug6gSwVjefm3ismWY,1405
+src/prompts/library/bookingConfirmation/hapag-lloyd/prompt.txt,sha256=XgfhrFTXLJ467L4Cer77K0KTPtWTg_-QJXCsltvLlpI,3430
+src/prompts/library/bookingConfirmation/maersk/placeholders.json,sha256=6p_IQMA1PUgGZqjf_by4ja9jK27ba4loYhEpIa7Oxx4,1406
+src/prompts/library/bookingConfirmation/maersk/prompt.txt,sha256=t-yh1dOrcRa0fm0VPFC1xCRBf0R0Zjp9j_Hb31aZS1w,3223
+src/prompts/library/bookingConfirmation/msc/placeholders.json,sha256=IpM9nmSPdyroliZfXB1-NDCjiHZX_Ff5BH7-scNhGqE,1406
+src/prompts/library/bookingConfirmation/msc/prompt.txt,sha256=_Jfioislp7SNs2BEXoklvnTPVXe6Z0M6myD1IWnBFYQ,4705
+src/prompts/library/bookingConfirmation/oocl/placeholders.json,sha256=JTtWvLSsoxN7huXY8ZNqqPkODM-DOs5wu3YvNHOna3k,1404
+src/prompts/library/bookingConfirmation/oocl/prompt.txt,sha256=xNTrJdUtDalcP3AKkfRiOnHjAdRCbcTvehcBQKurRj0,2201
+src/prompts/library/bookingConfirmation/other/placeholders.json,sha256=IpM9nmSPdyroliZfXB1-NDCjiHZX_Ff5BH7-scNhGqE,1406
+src/prompts/library/bookingConfirmation/other/prompt.txt,sha256=kUK7NgVNDYFMnqOcIblCwWSw2SC0YQEtHsYrspiVUMo,3379
+src/prompts/library/bookingConfirmation/yangming/placeholders.json,sha256=IpM9nmSPdyroliZfXB1-NDCjiHZX_Ff5BH7-scNhGqE,1406
+src/prompts/library/bookingConfirmation/yangming/prompt.txt,sha256=fYKfusDajDFw0v54-nv2iAqUSp2yCeOzc6G7AFe-h2w,3226
+src/prompts/library/bundeskasse/other/placeholders.json,sha256=J57CNPWcz87PV4k3ctb_gVuOu-zGObolqst-y-mESQY,4054
+src/prompts/library/bundeskasse/other/prompt.txt,sha256=t5nsLK-6rpOqcVnfHtU04RxN8wXTi9WpX1f4ASLoZ3E,2923
+src/prompts/library/commercialInvoice/other/prompt.txt,sha256=6sowYMzrKvgmTDpDnAzkeG4OqA44e6-8aUKWRKNziBY,2699
+src/prompts/library/customsAssessment/other/prompt.txt,sha256=XSqWa3k9LM7dTiJtX8AKTp_0x5Z0pCNRKNUWaywwBlY,2191
+src/prompts/library/customsInvoice/other/placeholders.json,sha256=g82I3LbLZr2mwfpFIFoCTK1Y_MEkGaEK4ew7jovr6nw,12172
+src/prompts/library/customsInvoice/other/prompt.txt,sha256=ZMdIysq7B1CvOG93YSHorDJpqNUDxXEnomS4cVNaJ90,9632
+src/prompts/library/deliveryOrder/other/placeholders.json,sha256=7fjqag3kCVMV4mJ52dTjAcLtaBX0paXrDrW48vQVZSk,1250
+src/prompts/library/deliveryOrder/other/prompt.txt,sha256=y3QjN54e8PplEJngNlxoykbdrToBefS3r8gWixCbjfE,2468
+src/prompts/library/draftMbl/hapag-lloyd/prompt.txt,sha256=4FxiO1eHkimZVQZXU6gGNikuDVAWNniYvY8FUdVhpvk,2327
+src/prompts/library/draftMbl/maersk/prompt.txt,sha256=4neW6buJirgoS84iDsy9ZcfQTaMeOFt92Emba01mzJA,2192
+src/prompts/library/draftMbl/other/placeholders.json,sha256=wIN06_NWsESDyNEDfOLPi3F2Vq-XPa4O3U32A32s-_Q,1736
+src/prompts/library/draftMbl/other/prompt.txt,sha256=pj-kgPV51upLhDppSKfhc2s5ylgr06l4IxrkFYjE9uM,2241
+src/prompts/library/finalMbL/hapag-lloyd/prompt.txt,sha256=RhxEJ4eWikAQiE40cuPsssnzizge6AJYFTSJLGUmz_U,2326
+src/prompts/library/finalMbL/maersk/prompt.txt,sha256=4neW6buJirgoS84iDsy9ZcfQTaMeOFt92Emba01mzJA,2192
+src/prompts/library/finalMbL/other/prompt.txt,sha256=pj-kgPV51upLhDppSKfhc2s5ylgr06l4IxrkFYjE9uM,2241
+src/prompts/library/packingList/other/prompt.txt,sha256=6Q9d0KBG6YWmNtzFivvmtQmitaUE2jytfwwc5YwsUgQ,2872
+src/prompts/library/partnerInvoice/other/placeholders.json,sha256=AJNBVKwDGebyNAuyWEwEuaUTL9hbLK0Rjr2H0lNfOBY,8686
+src/prompts/library/partnerInvoice/other/prompt.txt,sha256=HuTUlCpUgDQKUKF5QYYoUoHZ0pkBIqX0g5NzciF_fps,9393
+src/prompts/library/postprocessing/port_code/placeholders.json,sha256=2TiXf3zSzrglOMPtDOlCntIa5RSvyZQAKG2-IgrCY5A,22
+src/prompts/library/postprocessing/port_code/prompt_port_code.txt,sha256=--1wunSqEr2ox958lEhjO-0JFBfOLzA3qfKYIzG_Iok,884
+src/prompts/library/preprocessing/carrier/placeholders.json,sha256=1UmrQNqBEsjLIpOO-a39Az6bQ_g1lxDGlwqZFU3IEt0,408
+src/prompts/library/preprocessing/carrier/prompt.txt,sha256=NLvRZQCZ6aWC1yTr7Q93jK5z7Vi_b4HBaiFYYnIsO-w,134
+src/prompts/library/shippingInstruction/other/prompt.txt,sha256=dT2e-dPuvuz0rVYpwmok_1dWQ2Oa8Qy9NGZ6CCLOUI4,1468
+src/prompts/prompt_library.py,sha256=jPxybNPPGH7mzonqtAOqmw5WcT-RtbGP0pvMqqP22hg,2760
+src/setup.py,sha256=kPSZosrICfaGZeDaajr40Ha7Ok4XK4fo_uq35Omiwr0,7128
+src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
+src/utils.py,sha256=SIEThJlaXWGoWV7236iNoAlabCPNge5oTBpDywTxJw0,15968
+data_science_document_ai-1.41.0.dist-info/METADATA,sha256=MmKbqDbe9voabVucTrE-GoM192GMqFgD09_KNvp6Wsg,2153
+data_science_document_ai-1.41.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+data_science_document_ai-1.41.0.dist-info/RECORD,,

src/excel_processing.py

@@ -2,6 +2,8 @@
 # flake8: noqa: E402
 import logging
 
+from src.postprocessing.common import llm_prediction_to_tuples
+
 logger = logging.getLogger(__name__)
 
 import asyncio
@@ -73,6 +75,8 @@ async def extract_data_from_excel(
     ]
     extracted_data = {k: v for k, v in await asyncio.gather(*sheet_extract_tasks)}
 
+    # Convert LLM prediction dictionary to tuples of (value, page_number).
+    extracted_data = llm_prediction_to_tuples(extracted_data)
     stored_data = json.dumps(extracted_data)
 
     return extracted_data, stored_data, params["gemini_params"]["model_id"]

src/pdf_processing.py

@@ -14,7 +14,7 @@ from google.cloud.documentai_v1 import Document as docaiv1_document
 
 from src.docai import _batch_process_pdf_w_docai, _process_pdf_w_docai
 from src.excel_processing import extract_data_from_excel
-from src.postprocessing.common import format_all_entities, remove_none_values
+from src.postprocessing.common import format_all_entities, remove_none_values, llm_prediction_to_tuples
 from src.postprocessing.postprocess_booking_confirmation import (
     postprocess_booking_confirmation,
 )
@@ -31,6 +31,7 @@ from src.utils import (
     get_processor_name,
     run_background_tasks,
     validate_based_on_schema,
+    transform_schema_strings
 )
 
 
@@ -104,9 +105,16 @@ async def extract_data_from_pdf_w_docai(
     # Extract entities from the result
     for entity in result.entities:
         value = (
-            {child.type_: child.mention_text for child in entity.properties}
+            {child.type_: (child.mention_text,
+                           child.page_anchor.page_refs[0].page
+                           if hasattr(child.page_anchor.page_refs[0], "page")
+                           else 0)
+             for child in entity.properties}
             if entity.properties
-            else entity.mention_text
+            else (entity.mention_text,
+                  entity.page_anchor.page_refs[0].page
+                  if hasattr(entity.page_anchor.page_refs[0], "page")
+                  else 0)
         )
         aggregated_data[entity.type_].append(value)
 
@@ -220,6 +228,9 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
         result = await llm_client.get_unified_json_genai(
             prompt=prompt, document=document, response_schema=response_schema
         )
+
+        result = llm_prediction_to_tuples(result)
+
         return result
     return {}
 

src/postprocessing/common.py

@@ -380,6 +380,11 @@ async def format_label(entity_k, entity_value, document_type_code, params):
             ]
         )
         return entity_k, [v for _, v in format_tasks]
+    if isinstance(entity_value, tuple):
+        page = entity_value[1]
+        entity_value = entity_value[0]
+    else:
+        page = -1
     entity_key = entity_k.lower()
     formatted_value = None
 
@@ -446,10 +451,18 @@ async def format_label(entity_k, entity_value, document_type_code, params):
     elif "reversechargesentence" in entity_key:
         formatted_value = clean_item_description(entity_value, remove_numbers=False)
 
+    elif "quantity" in entity_key:
+        if document_type_code in ["partnerInvoice", "customsInvoice", "bundeskasse"]:
+            # For partner invoice, quantity can be mentioned as whole number
+            formatted_value = decimal_convertor(
+                extract_number(entity_value), quantity=True
+            )
+        else:
+            formatted_value = extract_number(entity_value)
+
     elif any(
         numeric_indicator in entity_key
         for numeric_indicator in [
-            "quantity",
             "value",
             "amount",
             "price",
@@ -466,6 +479,7 @@ async def format_label(entity_k, entity_value, document_type_code, params):
     result = {
         "documentValue": entity_value,
         "formattedValue": formatted_value,
+        "page": page,
     }
     return entity_k, result
 
@@ -514,7 +528,7 @@ async def get_port_code_llm(port: str, llm_client):
             return None
 
 
-def decimal_convertor(value):
+def decimal_convertor(value, quantity=False):
     """Convert EU values to English values."""
     if value is None:
         return None
@@ -522,25 +536,39 @@ def decimal_convertor(value):
     # Remove spaces
     value = value.strip().replace(" ", "")
 
-    # Convert comma to dot for decimal point (e.g., 4.123,45 -> 4123.45)
-    if re.match(r"^\d{1,3}(\.\d{3})*,\d{1,2}$", value):
-        value = value.replace(".", "").replace(",", ".")
+    if not quantity:
+        # Convert comma to dot for decimal point (e.g., 4.123,45 -> 4123.45)
+        if re.match(r"^\d{1,3}(\.\d{3})*,\d{1,2}$", value):
+            value = value.replace(".", "").replace(",", ".")
+
+        # European style integer with thousand separator: 2.500
+        elif re.match(r"^\d{1,3}(\.\d{3})+$", value):
+            value = value.replace(".", "")
+
+        # Format english values as well for consistency (e.g., 4,123.45 -> 4123.45)
+        elif re.match(r"^\d{1,3}(,\d{3})*\.\d{1,2}$", value):
+            value = value.replace(",", "")
 
-    # European style integer with thousand separator: 2.500
-    elif re.match(r"^\d{1,3}(\.\d{3})+$", value):
-        value = value.replace(".", "")
+        # English style integer with thousand separator: 2,500
+        elif re.match(r"^\d{1,3}(,\d{3})+$", value):
+            value = value.replace(",", "")
 
-    # Format english values as well for consistency (e.g., 4,123.45 -> 4123.45)
-    elif re.match(r"^\d{1,3}(,\d{3})*\.\d{1,2}$", value):
-        value = value.replace(",", "")
+        # Just replace comma decimals with dot (e.g., 65,45 -> 65.45)
+        if re.match(r"^\d+,\d{1,2}$", value):
+            value = value.replace(",", ".")
 
-    # English style integer with thousand separator: 2,500
-    elif re.match(r"^\d{1,3}(,\d{3})+$", value):
-        value = value.replace(",", "")
+        # If there are more than 3 0s after decimal point, consider only 2 decimal points (e.g., 8.500000 -> 8.50)
+        elif re.match(r"^\d+\.\d{3,}$", value):
+            value = value[: value.index(".") + 3]
 
-    # Just replace comma decimals with dot (e.g., 65,45 -> 65.45)
-    elif re.match(r"^\d+,\d{1,2}$", value):
-        value = value.replace(",", ".")
+    else:  # quantity=True → only last two
+        # Just replace comma decimals with dot (e.g., 65,45 -> 65.45)
+        if re.match(r"^\d+,\d{1,2}$", value):
+            value = value.replace(",", ".")
+
+        # If there are more than 3 0s after decimal point, consider only 2 decimal points (e.g., 8.500000 -> 8.50)
+        elif re.match(r"^\d+\.\d{3,}$", value):
+            value = value[: value.index(".") + 3]
 
     return value
 
@@ -594,3 +622,24 @@ def remove_stop_words(lineitem: str):
         .upper()
         .strip()
     )
+
+
+def llm_prediction_to_tuples(llm_prediction):
+    """Convert LLM prediction dictionary to tuples of (value, page_number)."""
+    if isinstance(llm_prediction, dict):
+        if "page_number" in llm_prediction.keys() and "value" in llm_prediction.keys():
+            if llm_prediction["value"]:
+                try:
+                    page_number = int(llm_prediction["page_number"])
+                except:  # noqa: E722
+                    page_number = -1
+                return (llm_prediction["value"], page_number)
+            return None
+        for key, value in llm_prediction.items():
+            llm_prediction[key] = llm_prediction_to_tuples(
+                llm_prediction.get(key, value)
+            )
+    elif isinstance(llm_prediction, list):
+        for i, item in enumerate(llm_prediction):
+            llm_prediction[i] = llm_prediction_to_tuples(item)
+    return llm_prediction

src/prompts/library/bookingConfirmation/evergreen/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }

src/prompts/library/bookingConfirmation/evergreen/prompt.txt

@@ -1,3 +1,4 @@
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 ```json
 {
 "mblNumber": "Extract the value after the label 'BOOKING NO.'.",

src/prompts/library/bookingConfirmation/hapag-lloyd/placeholders.json

@@ -1,32 +1,32 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }
     }
   },
   "required": []
-}
+}

src/prompts/library/bookingConfirmation/hapag-lloyd/prompt.txt

@@ -18,7 +18,7 @@ transportLegs:
     vesselName: The name of the vessel for a specific leg.
     voyage: The journey or route taken by the vessel for a specific leg.
 
-your task is to extract the text value of the following entities:
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 SCHEMA_PLACEHOLDER
 
 Keywords for datapoints:

src/prompts/library/bookingConfirmation/maersk/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }

src/prompts/library/bookingConfirmation/maersk/prompt.txt

@@ -18,7 +18,7 @@ transportLegs:
     vesselName: The name of the vessel for a specific leg.
     voyage: The journey or route taken by the vessel for a specific leg.
 
-your task is to extract the text value of the following entities:
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 SCHEMA_PLACEHOLDER
 
 Keywords for datapoints:

src/prompts/library/bookingConfirmation/msc/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }

src/prompts/library/bookingConfirmation/msc/prompt.txt

@@ -18,7 +18,7 @@ transportLegs:
     vesselName: The name of the vessel for a specific leg.
     voyage: The journey or route taken by the vessel for a specific leg.
 
-your task is to extract the text value of the following entities:
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 SCHEMA_PLACEHOLDER
 
 Further explanation and Keywords for the transportLegs part as follows. The below 2 conditions is crucial. Take attention here:

src/prompts/library/bookingConfirmation/oocl/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true}
         },
         "required": []
       }

src/prompts/library/bookingConfirmation/oocl/prompt.txt

@@ -1,4 +1,6 @@
-bookingNumber: Extract the booking number. This information can be found near the labels "BOOKING ACKNOWLEDGEMENT" or "BOOKING NUMBER". 
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
+
+bookingNumber: Extract the booking number. This information can be found near the labels "BOOKING ACKNOWLEDGEMENT" or "BOOKING NUMBER".
 gateInReference: This field should have the same value as the bookingNumber.
 cyCutOff: Look for the "INTENDED FCL CY CUT-OFF" label and extract the date and time value.
 vgmCutOff: Look for the "INTENDED VGM CUT-OFF" label and extract the date and time value.

src/prompts/library/bookingConfirmation/other/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }

src/prompts/library/bookingConfirmation/other/prompt.txt

@@ -18,7 +18,7 @@ transportLegs:
     vesselName: The name of the vessel for a specific leg.
     voyage: The journey or route taken by the vessel for a specific leg.
 
-your task is to extract the text value of the following entities:
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 SCHEMA_PLACEHOLDER
 
 Further explanation for the transportLegs part as follows: