data-science-document-ai 1.37.0__py3-none-any.whl → 1.51.0__py3-none-any.whl

src/postprocessing/postprocess_partner_invoice.py

@@ -1,9 +1,8 @@
 """This module contains the postprocessing functions for the partner invoice."""
-from concurrent.futures import ThreadPoolExecutor
-
-from fuzzywuzzy import fuzz
+from rapidfuzz import fuzz, process
 
 from src.io import logger
+from src.utils import get_tms_mappings
 
 
 def postprocessing_partner_invoice(partner_invoice):
@@ -104,9 +103,18 @@ def post_process_bundeskasse(aggregated_data):
             )
 
         # Check if the deferredDutyPayer is forto
-        deferredDutyPayer = line_item.get("deferredDutyPayer", {})
-        lower = deferredDutyPayer.get("documentValue", "").lower()
-        if any(key in lower for key in ["de789147263644738", "forto"]):
+        KEYWORDS = {"de789147263644738", "forto", "009812"}
+
+        def is_forto_recipient(line_item: dict) -> bool:
+            values_to_check = [
+                line_item.get("deferredDutyPayer", {}).get("documentValue", ""),
+                line_item.get("vatId", {}).get("documentValue", ""),
+            ]
+
+            combined = " ".join(values_to_check).lower()
+            return any(keyword in combined for keyword in KEYWORDS)
+
+        if is_forto_recipient(line_item):
             is_recipient_forto = True
 
     update_recipient_and_vendor(aggregated_data, is_recipient_forto)
@@ -135,38 +143,9 @@ def update_recipient_and_vendor(aggregated_data, is_recipient_forto):
     ] = "Dasbachstraße 15, 54292 Trier, Germany"
 
 
-def process_partner_invoice(params, aggregated_data, embed_manager, document_type_code):
+async def process_partner_invoice(params, aggregated_data, document_type_code):
     """Process the partner invoice data."""
-    # Post process containerNumber.
-    # TODO: Remove this block of code after migrating to LLM completely and update the placeholder in the prompt library
-    if "containerNumber" in aggregated_data and isinstance(
-        aggregated_data["containerNumber"], dict
-    ):
-        container_number = aggregated_data.get("containerNumber", {}).get(
-            "formattedValue", None
-        )
-        if container_number:
-            aggregated_data["containerNumber"] = (
-                [
-                    {
-                        "documentValue": aggregated_data.get("containerNumber", {}).get(
-                            "documentValue", ""
-                        ),
-                        "formattedValue": ctr_number,
-                    }
-                    for ctr_number in container_number
-                ]
-                if isinstance(container_number, list)
-                else [
-                    {
-                        "documentValue": aggregated_data.get("containerNumber", {}).get(
-                            "documentValue", ""
-                        ),
-                        "formattedValue": container_number,
-                    }
-                ]
-            )
-
+    # Post process bundeskasse invoices
     if document_type_code == "bundeskasse":
         post_process_bundeskasse(aggregated_data)
         return
@@ -188,24 +167,84 @@ def process_partner_invoice(params, aggregated_data, embed_manager, document_typ
         reverse_charge_info["formattedValue"] = reverse_charge_value
         reverse_charge = aggregated_data.pop("reverseChargeSentence", None)
 
-    # Process each line item
-    for line_item in line_items:
-        if line_item.get("lineItemDescription", None) is not None:
-            line_item["itemCode"] = associate_forto_item_code(
-                embed_manager,
-                line_item["lineItemDescription"]["formattedValue"],
-                params,
-            )
+    # Process everything in one go
+    processed_items = await process_line_items_batch(params, line_items, reverse_charge)
+
+    # Update your main data structure
+    aggregated_data["lineItem"] = processed_items
+
+
+async def process_line_items_batch(
+    params: dict, line_items: list[dict], reverse_charge=None
+):
+    """
+    Processes all line items efficiently using a "Split-Apply-Combine" strategy.
+    """
+    # To store items that need external API lookup
+    pending_line_items = {}
+
+    # Check Fuzzy Matching
+    logger.info(f"Mapping line item codes with Fuzzy matching....")
+    for i, item in enumerate(line_items):
+        description_obj = item.get("lineItemDescription")
+
+        if not description_obj or not description_obj.get("formattedValue"):
+            continue
+        # Get the formatted description text
+        desc = description_obj["formattedValue"]
+
+        # Find Fuzzy Match
+        matched_code = find_matching_lineitem(
+            desc,
+            params["lookup_data"]["item_code"],
+            params["fuzzy_threshold_item_code"],
+        )
+
+        if matched_code:
+            # Set the code to the line item
+            item["itemCode"] = {
+                "documentValue": desc,
+                "formattedValue": matched_code,
+                "page": description_obj.get("page"),
+            }
+        else:
+            # Store for batch API call
+            pending_line_items[i] = desc
 
-        if reverse_charge:
-            # Distribute reverseChargeSentence to all line items
-            line_item["reverseChargeSentence"] = reverse_charge
+    # Batch API Call for Embedding lookups
+    if pending_line_items:
+        values_to_fetch = list(set(pending_line_items.values()))
+        logger.info(f"Mapping {len(values_to_fetch)} line items from Embedding API...")
 
+        # Await the batch response {"desc1": "code1", "desc2": "code2"}
+        api_results = await get_tms_mappings(
+            input_list=values_to_fetch, embedding_type="line_items"
+        )
 
-def compute_score(args):
-    """Compute the fuzzy matching score between a new line item and a key."""
-    new_lineitem, key = args
-    return key, fuzz.ratio(new_lineitem, key)
+        # Merge API results back into original list
+        for index, desc in pending_line_items.items():
+            # Get result from API response, or None if API failed for that item
+            forto_code = api_results.get(desc)
+
+            # Update the original item
+            line_items[index]["itemCode"] = {
+                "documentValue": desc,
+                "formattedValue": forto_code,  # Might be None if API failed
+                "page": line_items[index]["lineItemDescription"].get("page"),
+            }
+
+    # Add reverse charge here if exists
+    if reverse_charge:
+        [
+            item.update({"reverseChargeSentence": reverse_charge})
+            for item in line_items
+            if (
+                (item.get("itemCode") and item["itemCode"]["formattedValue"] != "CDU")
+                or not item.get("itemCode")
+            )
+        ]
+
+    return line_items
 
 
 def get_fuzzy_match_score(target: str, sentences: list, threshold: int):
@@ -220,16 +259,18 @@ def get_fuzzy_match_score(target: str, sentences: list, threshold: int):
         tuple: (best_match, score) if above threshold, else (None, 0)
     """
     # Use multiprocessing to find the best match
-    with ThreadPoolExecutor() as executor:
-        results = executor.map(compute_score, [(target, s) for s in sentences])
+    result = process.extractOne(
+        target, sentences, scorer=fuzz.WRatio, score_cutoff=threshold
+    )
+
+    if result is None:
+        return None, False
 
-    # Find the best match and score
-    best_match, best_score = max(results, key=lambda x: x[1], default=(None, 0))
+    match, score, index = result
 
-    # return best_match, best_score
-    # If the best match score is above a threshold (e.g., 80), return it
-    if best_score >= threshold:
-        return best_match, True
+    # return best_match if the best match score is above a threshold (e.g., 80)
+    if match:
+        return match, True
 
     return None, False
 
@@ -261,50 +302,59 @@ def find_matching_lineitem(new_lineitem: str, kvp_dict: dict, threshold=90):
     Returns:
         str: The best matching 'Forto SLI' value from the dictionary.
     """
-    new_lineitem = new_lineitem.upper()
-
     # Check if the new line item is already in the dictionary
     if new_lineitem in kvp_dict:
         return kvp_dict[new_lineitem]
 
     # Get the best fuzzy match score for the extracted line item
-    best_match, _ = get_fuzzy_match_score(
-        new_lineitem, list(kvp_dict.keys()), threshold
+    match, _ = get_fuzzy_match_score(
+        new_lineitem,
+        list(kvp_dict.keys()),
+        threshold,
     )
 
-    return kvp_dict.get(best_match, None)
-
+    if match:
+        # find the code from the kvp_dict
+        return kvp_dict[match]
 
-def associate_forto_item_code(embed_manager, input_string, params):
-    """
-    Finds a match for the input string using fuzzy matching first, then embedding fallback.
+    return None
 
-    1. Tries to find a fuzzy match for input_string against the keys in
-       mapping_data using RapidFuzz, requiring a score >= fuzzy_threshold.
-    2. If found, returns the corresponding value from mapping_data.
-    3. If not found above threshold, calls the embedding_fallback function.
 
+async def associate_forto_item_code(line_item_data, params):
+    """
+    Associates Forto item codes to a list of line item descriptions.
     Args:
-        input_string: The string to find a match for.
-        embed_manager: The embedding manager instance to use for fallback.
-        params: Parameters containing the lookup data and fuzzy threshold.
+        line_item_data (dict): A dictionary where keys are original descriptions and values are cleaned descriptions.
+        params (dict): Parameters containing lookup data and thresholds.
 
     Returns:
-        The matched value (from fuzzy match or embedding), or None if no match found.
+        list: A list of dictionaries with 'description' and 'itemCode' keys.
     """
-    # Get the Forto item code using fuzzy matching
-    forto_item_code = find_matching_lineitem(
-        new_lineitem=input_string,
-        kvp_dict=params["lookup_data"]["item_code"],  # TODO: Parse the KVP dictionary
-        threshold=params["fuzzy_threshold_item_code"],
-    )
 
-    if forto_item_code is None:
-        # 2. Fallback to embedding function if no good fuzzy match
-        embeddings_dict = embed_manager.embeddings_dict
-        forto_item_code = embed_manager._find_most_similar_option(
-            input_string, *embeddings_dict["item_codes_label"]
+    result = []
+    pending_line_items = {}
+    for desc, f_desc in line_item_data.items():
+        # Get the Forto item code using fuzzy matching
+        code = find_matching_lineitem(
+            new_lineitem=f_desc,
+            kvp_dict=params["lookup_data"]["item_code"],
+            threshold=params["fuzzy_threshold_item_code"],
         )
+        if code:
+            result.append({"description": desc, "itemCode": code})
+        else:
+            pending_line_items[desc] = f_desc
+
+    # Batch API Call for Embedding lookups
+    if pending_line_items:
+        api_results = await get_tms_mappings(
+            input_list=list(pending_line_items.values()),
+            embedding_type="line_items",
+        )
+
+        # Merge API results back into original list
+        for desc, f_desc in pending_line_items.items():
+            code = api_results.get(f_desc)
+            result.append({"description": desc, "itemCode": code})
 
-    result = {"documentValue": input_string, "formattedValue": forto_item_code}
     return result

src/prompts/library/arrivalNotice/other/placeholders.json

@@ -0,0 +1,70 @@
+{
+  "type": "OBJECT",
+  "properties": {
+      "bookingNumber": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The booking number associated with the Arrival Notice document. They are often referred to as 'Booking Number', 'Booking No.', 'Booking Ref.', 'Booking Reference', 'Booking ID', 'carrier's reference' or 'Order Ref'."
+      },
+      "destinationTerminal": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The terminal at the destination port where the container will be delivered."
+      },
+      "eta": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "Estimated Time of Arrival (ETA) is the expected date when the shipment will arrive at its destination."
+      },
+      "mblNumber": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "Bill of Lading number (B/L NO.), a document issued by the carrier."
+      },
+      "portOfDischarge": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The port where the goods are discharged from the vessel. This is the destination port for the shipment."
+      },
+      "vesselName": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The name of the vessel carrying the shipment."
+      },
+      "containers": {
+          "type": "ARRAY",
+          "items": {
+              "type": "OBJECT",
+              "properties": {
+                  "containerNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The unique identifier for each container. It always starts with 4 capital letters and followed by 7 digits. Example: TEMU7972458."
+                  },
+                  "containerType": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc."
+                  },
+                  "grossWeight": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The gross weight of the container. Usually mentioned as G.W or GW or Gross Weight, etc.."
+                  },
+                  "measurements": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The volume of the container. Usually, it is measured in 'Cubic Meter (cbm)' or dimensions. But volume in 'cbm' is preferred."
+                  },
+                  "sealNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The seal number associated with the container Number. But it is not same as the container number."
+                  }
+              },
+              "required": ["containerNumber", "containerType", "grossWeight"]
+          }
+      }
+},
+  "required": ["bookingNumber", "destinationTerminal", "eta", "portOfDischarge", "vesselName", "containers"]
+}

src/prompts/library/arrivalNotice/other/prompt.txt

@@ -0,0 +1,40 @@
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
+
+<TASK> Your task is to extract data from Arrival Notice documents as per the given response schema structure. <TASK>
+
+<CONTEXT>
+The Freight Forwarding company receives Arrival Notice from shipping lines.
+These documents contain various details related to arrival of a shipment to the port of destination such as container numbers, estimated time of arrival, vessel details and containers information.
+They may be written in different languages such as English, German, Italian and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these Arrival Notices to support efficient processing and accurate record-keeping.
+<CONTEXT>
+
+<INSTRUCTIONS>
+- Populate fields as defined in the response schema.
+- Multiple Containers entries may exist, capture all instances under "containers".
+- Use the data field description to understand the context of the data.
+
+- bookingNumbers:
+    - Booking numbers are unique identifiers for shipments. They are often referred to as "Booking Number", "Booking No.", "Booking Ref.", "Booking Reference", "Booking ID", "SACO-Pos.", "Order Ref", "Unsere Referenz", or "Unsere Position"
+    - If there is a unique_id that starts with "S" followed by 6 or 8 digits, it is a shipmentID, not a bookingNumber.
+
+- destinationTerminal:
+    - Destination Terminal can also be referred to as "Destination Termina;", "Pickup Location", "Delivery Location", "Delivery Terminal", "Empfangsort", "Entladeort", or "Abladestelle".
+
+- mblNumbers:
+    - Commonly known as "Bill of Lading Number", "BILL OF LADING NO.", "BL Number", "BL No.", "B/L No.", "BL-Nr.", "B/L", "HBL No.", or "M-AWB Nummer".
+    - Bill of Lading Number is known as mblNumber. Not a shipmentID even if it starts with "S".
+    - mblNumber from Hapag-Lloyd always starts with HLC.... (e.g., "HLCUTS12303AWNT3) and named as SEA WAYBILL or "SWB-NR.
+
+- eta:
+    - Estimated Time of Arrival (ETA) is the expected date and time when the shipment will arrive at the destination port.
+    - It can be referred to as "ETA", "Estimated Arrival", "Voraussichtliche Ankunft", "Ankunftszeit", "Arrivo", "Due to arrive at Terminal"
+
+- vesselName:
+    - Vessel Name is the name of the ship carrying the cargo. It can be referred to as "Vessel", "Ship Name", "Schiff", "Schiffsname", "Nave", or "Vessel/Flight No.".
+
+- containers: Details of each container on the arrival notice. Make sure to extract each container information separately.
+    - containerNumber: Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU 7222892).
+    - sealNumber: Seal numbers are unique identifiers for shipping seals. They are usually mentioned as seal numbers in the document but they are definitely not container numbers.
+
+<INSTRUCTIONS>

src/prompts/library/bookingConfirmation/evergreen/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }

src/prompts/library/bookingConfirmation/evergreen/prompt.txt

@@ -1,3 +1,4 @@
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 ```json
 {
 "mblNumber": "Extract the value after the label 'BOOKING NO.'.",

src/prompts/library/bookingConfirmation/hapag-lloyd/placeholders.json

@@ -1,32 +1,32 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }
     }
   },
   "required": []
-}
+}

src/prompts/library/bookingConfirmation/hapag-lloyd/prompt.txt

@@ -18,7 +18,7 @@ transportLegs:
     vesselName: The name of the vessel for a specific leg.
     voyage: The journey or route taken by the vessel for a specific leg.
 
-your task is to extract the text value of the following entities:
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 SCHEMA_PLACEHOLDER
 
 Keywords for datapoints:

src/prompts/library/bookingConfirmation/maersk/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }

src/prompts/library/bookingConfirmation/maersk/prompt.txt

@@ -18,7 +18,7 @@ transportLegs:
     vesselName: The name of the vessel for a specific leg.
     voyage: The journey or route taken by the vessel for a specific leg.
 
-your task is to extract the text value of the following entities:
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 SCHEMA_PLACEHOLDER
 
 Keywords for datapoints:

src/prompts/library/bookingConfirmation/msc/placeholders.json

@@ -1,28 +1,28 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "string", "nullable": true, "description": "he date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "string", "nullable": true},
-    "cyCutOff": {"type": "string", "nullable": true},
-    "gateInReference": {"type": "string", "nullable": true},
-    "gateInTerminal": {"type": "string", "nullable": true},
-    "mblNumber": {"type": "string", "nullable": true},
-    "pickUpReference": {"type": "string", "nullable": true},
-    "pickUpTerminal": {"type": "string", "nullable": true},
-    "siCutOff": {"type": "string", "nullable": true},
-    "vgmCutOff": {"type": "string", "nullable": true},
+    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
+    "bookingNumber": {"type": "STRING", "nullable": true},
+    "cyCutOff": {"type": "STRING", "nullable": true},
+    "gateInReference": {"type": "STRING", "nullable": true},
+    "gateInTerminal": {"type": "STRING", "nullable": true},
+    "mblNumber": {"type": "STRING", "nullable": true},
+    "pickUpReference": {"type": "STRING", "nullable": true},
+    "pickUpTerminal": {"type": "STRING", "nullable": true},
+    "siCutOff": {"type": "STRING", "nullable": true},
+    "vgmCutOff": {"type": "STRING", "nullable": true},
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "string", "nullable": true},
-            "etd": {"type": "string", "nullable": true},
-            "imoNumber": {"type": "string", "nullable": true},
-            "portOfDischarge": {"type": "string", "nullable": true},
-            "portOfLoading": {"type": "string", "nullable": true},
-            "vesselName": {"type": "string", "nullable": true},
-            "voyage": {"type": "string", "nullable": true}
+            "eta": {"type": "STRING", "nullable": true},
+            "etd": {"type": "STRING", "nullable": true},
+            "imoNumber": {"type": "STRING", "nullable": true},
+            "portOfDischarge": {"type": "STRING", "nullable": true},
+            "portOfLoading": {"type": "STRING", "nullable": true},
+            "vesselName": {"type": "STRING", "nullable": true},
+            "voyage": {"type": "STRING", "nullable": true}
           },
         "required": []
       }

src/prompts/library/bookingConfirmation/msc/prompt.txt

@@ -18,7 +18,7 @@ transportLegs:
     vesselName: The name of the vessel for a specific leg.
     voyage: The journey or route taken by the vessel for a specific leg.
 
-your task is to extract the text value of the following entities:
+your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
 SCHEMA_PLACEHOLDER
 
 Further explanation and Keywords for the transportLegs part as follows. The below 2 conditions is crucial. Take attention here: