data-science-document-ai 1.47.1__py3-none-any.whl → 1.48.0__py3-none-any.whl

{data_science_document_ai-1.47.1.dist-info → data_science_document_ai-1.48.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.47.1
+Version: 1.48.0
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.47.1.dist-info → data_science_document_ai-1.48.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
-src/constants.py,sha256=IduR732AwH_ihYe3OFRmCzZ6LodYE91sp1Yy9pGUG8I,3611
+src/constants.py,sha256=k5bBnJN-kmXiAtIAlz6Kg6fDyR9n0DuIudCZ9ZHO_Jw,3528
 src/constants_sandbox.py,sha256=Iu6HdjCoNSmOX0AwoL9qUQkhq_ZnIN5U9e-Q2UfNuGc,547
 src/docai.py,sha256=dHuR0ehVjUi1CnoNvdp_yxJtpU_HFXqAZ61ywdz7BEo,5655
-src/docai_processor_config.yaml,sha256=I6yQZgnvbX0MX_RMrnA0vXXiglX-oBUwH5utplb-Nuk,1182
-src/excel_processing.py,sha256=_vP2q1xEIeyjO8TvZlSTeEM-M1PMceyDSuYGfyZeceY,3361
+src/docai_processor_config.yaml,sha256=4yKKZPvFCA-3S56jDYSqMGKXGFND-768OiU2seRiAzE,604
+src/excel_processing.py,sha256=TRgAzSHvL1WKbUgjHtpXL701bPhiWGH7kk3S6e1UPaA,3074
 src/io.py,sha256=rYjXVLlriEacw1uNuPIYhg12bXNu48Qs9GYMY2YcVTE,5563
-src/llm.py,sha256=OE4IEIqcM-hYK9U7e0x1rAfcqdpeo4iXPHBp64L5Qz0,8199
+src/llm.py,sha256=a7UYA4ITUNjzct_2fHgM-bma_XWc28VC0FV71g9tnUI,7137
 src/log_setup.py,sha256=RhHnpXqcl-ii4EJzRt47CF2R-Q3YPF68tepg_Kg7tkw,2895
-src/pdf_processing.py,sha256=0CA6H_FlvBLHaJkkHt8u-JJSIuumQ6fHskqJn7-rccI,20499
+src/pdf_processing.py,sha256=oKVPnIu_keiN17XLOGImeyJ4iMT2H51x4OD1Tp9yw1s,19992
 src/postprocessing/common.py,sha256=dagAg0hZGuZc03bXdfOolxekewMEVUfz917IGCiAtWI,26118
 src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
 src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
@@ -31,11 +31,12 @@ src/prompts/library/bundeskasse/other/placeholders.json,sha256=1ll8AI58F2zRDSwQq
 src/prompts/library/bundeskasse/other/prompt.txt,sha256=MBv4MIMASMstkzDS7H0q_pNJbPQeadP1vcmhCRrpjQ4,2906
 src/prompts/library/commercialInvoice/other/placeholders.json,sha256=zUK2mg9MnHiEQRYF6VgTiUiq68WGy5f7_4qL63CWyR0,4700
 src/prompts/library/commercialInvoice/other/prompt.txt,sha256=CJapcVrmcvynJUanETDklkzU-0N9hHdhq5wL4MK7OIY,2683
-src/prompts/library/customsAssessment/other/prompt.txt,sha256=XSqWa3k9LM7dTiJtX8AKTp_0x5Z0pCNRKNUWaywwBlY,2191
+src/prompts/library/customsAssessment/other/placeholders.json,sha256=scIV--C9HNWAQbU9zEz3GT_FoAvJqbfuY85YUtt7t-Q,3850
+src/prompts/library/customsAssessment/other/prompt.txt,sha256=z3FuoHZ588Pz1WBJDW7ISAC3J6n7hPJCcS92CdHDTFw,2494
 src/prompts/library/customsInvoice/other/placeholders.json,sha256=BnWYtl4sPooTHb_EHRIlrPawBrfHI8_QVas8zytbqyY,12172
 src/prompts/library/customsInvoice/other/prompt.txt,sha256=1dR73TQZJAfO9dKl-h7VhiJkdli498IV4e5JgBlOoYw,9695
-src/prompts/library/deliveryOrder/other/placeholders.json,sha256=7fjqag3kCVMV4mJ52dTjAcLtaBX0paXrDrW48vQVZSk,1250
-src/prompts/library/deliveryOrder/other/prompt.txt,sha256=y3QjN54e8PplEJngNlxoykbdrToBefS3r8gWixCbjfE,2468
+src/prompts/library/deliveryOrder/other/placeholders.json,sha256=j-9F4V3yDg4610PPsOwU3oOj_S9vAvAB9Ix155WGIwc,3827
+src/prompts/library/deliveryOrder/other/prompt.txt,sha256=RD076vq0x0IjoEVQfh-G0u4nxITCpgKZGrwMlR9YAvk,2695
 src/prompts/library/draftMbl/other/placeholders.json,sha256=Gn8kQ8cMmrzRGLSFH7_8wO1_j2jxhqHd4zeivZP2SjU,4304
 src/prompts/library/draftMbl/other/prompt.txt,sha256=4RjlGT2OFmcBCUJhuCnO9GtmCn3vVesXHi_ml2g3dK8,2386
 src/prompts/library/finalMbL/other/placeholders.json,sha256=Gn8kQ8cMmrzRGLSFH7_8wO1_j2jxhqHd4zeivZP2SjU,4304
@@ -48,11 +49,12 @@ src/prompts/library/postprocessing/port_code/placeholders.json,sha256=2TiXf3zSzr
 src/prompts/library/postprocessing/port_code/prompt_port_code.txt,sha256=--1wunSqEr2ox958lEhjO-0JFBfOLzA3qfKYIzG_Iok,884
 src/prompts/library/preprocessing/carrier/placeholders.json,sha256=tQeVDtvembhVqvel9vGoy4qcKp1hOvg-bLCgZRdQj0g,192
 src/prompts/library/preprocessing/carrier/prompt.txt,sha256=NLvRZQCZ6aWC1yTr7Q93jK5z7Vi_b4HBaiFYYnIsO-w,134
-src/prompts/library/shippingInstruction/other/prompt.txt,sha256=dT2e-dPuvuz0rVYpwmok_1dWQ2Oa8Qy9NGZ6CCLOUI4,1468
+src/prompts/library/shippingInstruction/other/placeholders.json,sha256=eK4AeMfORkGMWVYcqH7NjB56Zb4swHTvcQD5UQbTryg,6374
+src/prompts/library/shippingInstruction/other/prompt.txt,sha256=CbrqlKMtB-sVY-8E460KP1KNmz169YVPMrH3-uEldPg,2135
 src/prompts/prompt_library.py,sha256=VJWHeXN-s501C2GiidIIvQQuZdU6T1R27hE2dKBiI40,2555
 src/setup.py,sha256=M-p5c8M9ejKcSZ9N86VtmtPc4TYLxe1_4_dxf6jpfVc,7262
 src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
 src/utils.py,sha256=Ow5_Jals88o8mbZ1BoHfZpHZoCfig_UQb5aalH-mpWE,17278
-data_science_document_ai-1.47.1.dist-info/METADATA,sha256=__UACWYq3Q9AQ3ZwUu7P47AbY9JT23DIh-89fsZoawU,2152
-data_science_document_ai-1.47.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-data_science_document_ai-1.47.1.dist-info/RECORD,,
+data_science_document_ai-1.48.0.dist-info/METADATA,sha256=TEE-nCF6lxV4DdmQIhbij2ZXGrVwM0wBSLJxJPdUJLE,2152
+data_science_document_ai-1.48.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+data_science_document_ai-1.48.0.dist-info/RECORD,,

src/constants.py

@@ -53,10 +53,6 @@ project_parameters = {
     "model_selector": {
         "stable": {
             "bookingConfirmation": 1,
-            "shippingInstruction": 0,
-            "customsAssessment": 0,
-            "deliveryOrder": 0,
-            "partnerInvoice": 0,
         },
         "beta": {
             "bookingConfirmation": 0,
@@ -87,6 +83,7 @@ project_parameters = {
         "arrivalNotice": ["containers"],
         "finalMbL": ["containers"],
         "draftMbl": ["containers"],
+        "deliveryOrder": ["Equipment", "TransportLeg"],
         "customsAssessment": ["containers"],
         "packingList": ["skuData"],
         "commercialInvoice": ["skus"],

src/docai_processor_config.yaml

@@ -13,27 +13,6 @@ model_config:
           author: "igor.tonko@forto.com"
           created_date: ""
 
-    shippingInstruction:
-      - id: "c77a0a515d99a8ba"
-        details:
-          display_name: "doc_cap_shippingInstruction"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
-    customsAssessment:
-      - id: "c464a18d82fad9be"
-        details:
-          display_name: "doc_cap_customsAssessment"
-          author: "igor.tonko@forto.com"
-          created_date: ""
-
-    deliveryOrder:
-      - id: "2245a72c7a5dbf5f"
-        details:
-          display_name: "doc_cap_releaseNote"
-          author: "igor.tonko@forto.com"
-          created_date: ""
-
   beta:
     bookingConfirmation:
       - id: "3c280b11bdb3ed89"

src/excel_processing.py

@@ -11,9 +11,8 @@ import asyncio
 import numpy as np
 import pandas as pd
 
-from src.llm import prompt_excel_extraction
 from src.prompts.prompt_library import prompt_library
-from src.utils import estimate_page_count, generate_schema_structure, get_excel_sheets
+from src.utils import estimate_page_count, get_excel_sheets
 
 
 async def extract_data_from_sheet(
@@ -29,11 +28,14 @@ async def extract_data_from_sheet(
     )
 
     # Prompt for the LLM JSON
-    prompt_docai = prompt_excel_extraction(worksheet)
+    prompt = prompt_library.library[doc_type]["other"]["prompt"]
+
+    # Join the worksheet content with the prompt
+    prompt = worksheet + "\n" + prompt
 
     try:
         result = await llm_client.get_unified_json_genai(
-            prompt_docai,
+            prompt,
             response_schema=response_schema,
             doc_type=doc_type,
         )
@@ -67,19 +69,7 @@ async def extract_data_from_excel(
 
     """
     # Generate the response structure
-    response_schema = (
-        prompt_library.library[input_doc_type]["other"]["placeholders"]
-        if input_doc_type
-        in [
-            "partnerInvoice",
-            "customsInvoice",
-            "bundeskasse",
-            "commercialInvoice",
-            "packingList",
-            "bookingConfirmation",
-        ]
-        else generate_schema_structure(params, input_doc_type)
-    )
+    response_schema = prompt_library.library[input_doc_type]["other"]["placeholders"]
 
     # Load the Excel file and get ONLY the "visible" sheet names
     sheets, workbook = get_excel_sheets(file_content, mime_type)

src/llm.py

@@ -201,33 +201,4 @@ class LlmClient:
         return response
 
 
-def prompt_excel_extraction(excel_structured_text):
-    """Write a prompt to extract data from Excel files.
-
-    Args:
-        excel_structured_text (str): The structured text of the Excel file.
-
-    Returns:
-        prompt str: The prompt for common json.
-    """
-    prompt = f"""{excel_structured_text}
-
-    Task: Fill in the following dictionary from the information in the given in the above excel data.
-
-    Instructions:
-    - Do not change the keys of the following dictionary.
-    - The values should be filled in as per the schema provided below.
-    - If an entity contains a 'display_name', consider its properties as child data points in the below format.
-    {{'data-field': {{
-        'child-data-field': 'type -occurrence_type- description',
-          }}
-    }}
-    - The entity with 'display_name' can be extracted multiple times. Please pay attention to the occurrence_type.
-    - Ensure the schema reflects the hierarchical relationship.
-    - Use the data field description to understand the context of the data.
-
-    """
-    return prompt
-
-
 # pylint: enable=all

src/pdf_processing.py

@@ -202,23 +202,8 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
     number_of_pages = get_pdf_page_count(file_content)
     logger.info(f"processing {input_doc_type} with {number_of_pages} pages...")
 
-    # get the schema placeholder from the Doc AI and generate the response structure
-    response_schema = (
-        prompt_library.library[input_doc_type]["other"]["placeholders"]
-        if input_doc_type
-        in [
-            "partnerInvoice",
-            "customsInvoice",
-            "bundeskasse",
-            "commercialInvoice",
-            "packingList",
-            "bookingConfirmation",
-            "arrivalNotice",
-            "finalMbL",
-            "draftMbl",
-        ]  # Move this to constants or remove after complete migration to LLM
-        else generate_schema_structure(params, input_doc_type)
-    )
+    # get the schema placeholder
+    response_schema = prompt_library.library[input_doc_type]["other"]["placeholders"]
 
     carrier = "other"
     carrier_schema = (

src/prompts/library/customsAssessment/other/placeholders.json

@@ -0,0 +1,70 @@
+{
+  "type": "OBJECT",
+  "properties": {
+      "consignee": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The receiver or buyer of the goods. It can be find with the keywords like Importeur, Anmelder, Empfanger, Consignee, Buyer, Receiver, etc.."
+      },
+      "countryOfOrigin": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The country where the goods were manufactured or produced. It can be identified as Land van oorsprong, Ursprungsland in the document."
+      },
+      "MRN": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "Movement Reference Number (MRN) is a unique identifier assigned to each customs declaration for goods being imported or exported within the European Union (EU). It is used to track and monitor the movement of goods across EU member states. It can be found with MRN, Reg. Nr., Reg. Kennzeigechen, etc.."
+      },
+      "shipper": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The seller or shipper of the goods. It is often indicated by the term Shipper, Speditore, Esportatore, Exporteur, Versender."
+      },
+      "totalValueOfGoods": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The total monetary value of the goods being shipped, usually declared for customs purposes. It can be found with Waarde, Warenwert, Factuurwaarde, Invoice Value, etc.."
+      },
+      "containers": {
+          "type": "ARRAY",
+          "items": {
+              "type": "OBJECT",
+              "properties": {
+                  "containerNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The unique identifier for each container. It always starts with 4 capital letters and followed by 7 digits. Example: TEMU7972458."
+                  },
+                  "goodsDescription": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "A brief description of the goods contained within the container. It can be found with goods description, Bezeichnung, goederenomschrijving."
+                  },
+                  "grossWeight": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The gross weight of the container. Usually mentioned as G.W or GW, Bruto, or Gross Weight, etc.."
+                  },
+                  "nettWeight": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The net weight of the goods inside the container. Usually mentioned as N.W or NW, Net Weight, or Netto, Eigenmasse, etc.."
+                  },
+                "packagingNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The quantity of the goods. Usually, the quantity is in pallets, PLT, cartons, CTNS, pieces, PCS, packages, boxes, etc. Please prioritize the packaging types based on their size, as follows: Pallets (PLT) >> Cartons (CTNS) >> Pieces (PCS). Extract the Larger packaging types that will have a lower count."
+                  },
+                  "packagingType": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The packaging type is the unit of packagingNumber. Example; pallets, PLT, cartons, CTNS, pieces, PCS, packages, etc. Sometimes, the packaging type is available in the column name of the packagingNumber."
+                }
+              },
+              "required": ["containerNumber", "goodsDescription", "grossWeight", "nettWeight", "packagingNumber", "packagingType"]
+          }
+      }
+},
+  "required": ["countryOfOrigin", "MRN", "totalValueOfGoods", "containers"]
+}

src/prompts/library/customsAssessment/other/prompt.txt

@@ -1,42 +1,29 @@
-You are a document entity extraction specialist. Your task is to extract data from a customs assessment document.
-Customs assessment contain necessary information about imported / exported goods and containers.
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
 
-consignee: Legal Entity that is responsible for importing goods, name and address.
-shipper: Legal Entity that is responsible for sending goods, name and address.
-countryOfOrigin: Country of origin of goods.
-MRN: MRN code.
-totalValueOfGoods: Total value of goods.
-containers:
-    containerNumber: Unique ID for tracking the shipping container.
-    grossWeight: Total weight of the cargo, including the tare weight of the container.
-    packagingNumber: Packaging number.
-    nettWeight: Weight of the goods excluding packaging and containers.
-    packagingType: Type of packaging used (e.g., cartons, pallets, barrels).
-    goodsDescription: Goods description.
+<TASK> Your task is to extract data from delivery order documents as per the given response schema structure. <TASK>
 
+<CONTEXT>
+The Freight Forwarding company receives Customs Assessment from customs partners.
+These documents contain various details related to shipper, buyer, MRN, and container data such as container number, goods details at container level.
+They may be written in different languages such as English, German, Vietnamese, Chinese, and other European languages, and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these Customs Assessment to support efficient processing and accurate record-keeping.
+<CONTEXT>
 
-Your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
-SCHEMA_PLACEHOLDER
+<INSTRUCTIONS>
+- Populate fields as defined in the response schema.
+- Multiple containers entries may exist — capture all instances under "containers".
+- Use the data field description to understand the context of the data.
 
-Keywords for datapoints:
-- consignee: Importeur, Anmelder, Empfanger.
-- shipper: Speditore, Esportatore, Exporteur, Versender.
-- countryOfOrigin: Land van oorsprong, Ursprungsland.
-- MRN: MRN, Reg. Nr., Reg. Kennzeigechen.
-- totalValueOfGoods: Waarde, Warenwert, Factuurwaarde.
-- containers:
-    - containerNumber: container number, cntr. nos., containernummern, cont. nr.
-    - grossWeight: gross weight, Bruto.
-    - nettWeight: Weight of the goods excluding packaging and containers, Netto, Eigenmasse.
-    - packagingNumber: package number, Anzahl.
-    - packagingType: Type of packaging used (e.g., cartons, pallets, barrels), number and kind of packages, description of goods.
-    - goodsDescription: goods description, Bezeichnung, goederenomschrijving.
+- MRN: Movement Reference Number (MRN) is a unique identifier assigned to each customs declaration for goods being imported or exported within the European Union (EU). It is used to track and monitor the movement of goods across EU member states. It can be found with MRN, Reg. Nr., Reg. Kennzeigechen, etc..
 
-
-You must apply the following rules:
-- The JSON schema must be followed during the extraction.
-- The values must only include text found in the document
-- Do not normalize any entity value.
-- nettWeight can't be equal to grossWeight.
-- Validate the JSON make sure it is a valid JSON ! No extra text, no missing comma!
-- Add an escape character (backwards slash) in from of all quotes in values
+- containers: Details of each container on the Customs Assessment. Make sure to extract each container information separately.
+    - containerNumber: Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU 7222892). It can be identified as container number, cntr. nos., containernummern, cont. nr.
+    - goodsDescription: Extract only the description of the goods for the "goodsDescription" but not other information like packing, marks, etc.
+    - packagingNumber:
+        - Prioritize the "Pallets/PLTS/Cartons/CTNS/Package" over "PCS" count to extract the data for the "packagingNumber".
+        - example: If the table has "17CTNS", "9PLTS", "850", "850PCS", prioritize "9PLTS"
+        - Do not extract the pack Quantity field such as "50PCS/CTN", "5PC/Box" (these represent quantity per carton, not total shipped quantity).
+    - packagingType:
+            - Extract the unit associated with the "packagingNumber" in the table to extract the "packagingType"
+            - Sometimes it can be found on the column name of the "packagingNumber" in the table to extract the "packagingType"
+<INSTRUCTIONS>

src/prompts/library/deliveryOrder/other/placeholders.json

@@ -1,29 +1,82 @@
 {
-    "type": "OBJECT", 
-    "properties": {
-      "EmptyContainerDepot": {"type": "STRING", "nullable": true},
-      "Equipment": {"type": "ARRAY",
-        "items": {
-          "type": "OBJECT", "properties": {
-            "CargoGrossWeight": {"type": "STRING", "nullable": true},
-            "ContainerNumber": {"type": "STRING", "nullable": true},
-            "ContainerType": {"type": "STRING", "nullable": true},
-            "EmptyReturnReference": {"type": "STRING", "nullable": true},
-            "Pin": {"type": "STRING",  "nullable": true},
-            "TareWeight": {"type": "STRING", "nullable": true}
-          }, "required": []}
-      }, 
-      "pickUpTerminal": {"type": "STRING", "nullable": true},
-      "TransportLeg": {"type": "ARRAY",
-        "items": {
-          "type": "OBJECT", "properties": {
-            "eta": {"type": "STRING", "nullable": true},
-            "etd": {"type": "STRING", "nullable": true},
-            "portOfDischarge": {"type": "STRING", "nullable": true},
-            "portOfLoading": {"type": "STRING", "nullable": true},
-            "vesselName": {"type": "STRING", "nullable": true},
-            "voyage": {"type": "STRING", "nullable": true}
-          }, "required": []}
-      },
-    "required": []}
+  "type": "OBJECT",
+  "properties": {
+    "EmptyContainerDepot": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The depot where the empty container is returned."
+    },
+    "Equipment": {
+      "type": "ARRAY",
+      "items": {
+        "type": "OBJECT",
+        "properties": {
+          "CargoGrossWeight": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The gross weight of the Cargo. Usually mentioned as G.W or GW or Gross Weight, etc.."},
+          "ContainerNumber": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The container number associated with the document. They MUST consist of 4 letters followed by 7 digits (e.g., 'CMAU1234567', 'BMOU 575538/3', 'XLXU 1277652'). It can be found in the document as 'Container No.', 'Container Number', 'Cont. No.', 'Cont Nr.', 'Seefrachtcontainer-Nr.', or 'Containernummer'."},
+          "ContainerType": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The size or Type of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc."},
+          "EmptyReturnReference": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The reference number or code for the return of the empty container."},
+          "Pin": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The PIN code associated with the container, often used for security or access purposes."},
+          "TareWeight": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The weight of the empty container itself, without any cargo inside. Usually mentioned as T.W or TW or Tare Weight, etc.."}
+        },
+        "required": ["CargoGrossWeight", "ContainerNumber", "EmptyReturnReference", "Pin", "TareWeight"]
+      }
+    },
+    "pickUpTerminal": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The terminal where the container or cargo is picked up."
+    },
+    "TransportLeg": {
+      "type": "ARRAY",
+      "items": {
+        "type": "OBJECT",
+        "properties": {
+          "eta": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "Estimated Time of Arrival (ETA) is the expected date when the shipment will arrive at its destination."},
+          "etd": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "Estimated Time of Departure (ETD) is the expected date when the shipment will leave the origin port."},
+          "portOfDischarge": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The port where the goods are discharged from the vessel. This is the destination port for the shipment."},
+          "portOfLoading": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The origin port where the goods are loaded onto the vessel. Find information like 'Ladehafen' or 'Port of Loading' in the invoice."},
+          "vesselName": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The name of the vessel carrying the container or shipment"},
+          "voyage": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The unique voyage number or identifier assigned to a vessel’s specific journey. This typically corresponds to the scheduled sailing associated with the shipment and can often be found near vessel information on shipping documents. such as voyage, voy. no, voyage-no."}
+        },
+        "required": ["eta", "etd", "portOfDischarge", "portOfLoading", "vesselName", "voyage"]
+      }
+    }
+  },
+    "required": ["Equipment", "TransportLeg"]
 }

src/prompts/library/deliveryOrder/other/prompt.txt

@@ -1,50 +1,36 @@
-You are a document entity extraction specialist. Given a document, the explained datapoint need to extract.
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
 
+<TASK> Your task is to extract data from Delivery Order documents as per the given response schema structure. <TASK>
+
+<CONTEXT>
+The Freight Forwarding company receives Delivery Order from Carrier (Shipping Lines) partners.
+These documents contain various details related to shipments, equipment details, transport leg details, delivery / pickup details, vessel details, pick up terminal data.
+They may be written in different languages such as English, German, Vietnamese, Chinese, and other European languages, and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these Delivery Orders to support efficient processing and accurate record-keeping.
+<CONTEXT>
+
+<INSTRUCTIONS>
+- Populate fields as defined in the response schema.
+- Multiple Equipment and TransportLeg entries may exist — capture all instances under "Equipment" and "TransportLeg".
+- Use the data field description to understand the context of the data.
 
 EmptyContainerDepot: Empty container depot address.
-Equipment:
-      CargoGrossWeight: Total weight of the cargo, including the tare weight of the container.
-      ContainerNumber: Unique ID for tracking the shipping container.
+Equipment: Details of each Equipment on the Delivery Order. Make sure to extract each Equipment information separately.
+      CargoGrossWeight: Total weight of the cargo, including the tare weight of the container. Weight(incl. tare), Cargo Weight, Weight (KG)
+      ContainerNumber: Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU 7222892).
       ContainerType: Type of the shipping container, usually related to it's size.
-      EmptyReturnReference:  A reference code for empty container return.
-      Pin: Container release reference.
-      TareWeight: Tare weight.
-pickUpTerminal: The specific terminal for cargo pickup.
-TransportLeg:
+      EmptyReturnReference:  A reference code for empty container return. Find it as Drop off reference, turn-in reference in the document.
+      Pin: Container release reference or PIN code to pick up the container. Can be found as Release reference, pin
+      TareWeight: Weight of the empty container without cargo. It can be found as Tare weight, tare.
+
+pickUpTerminal: The specific terminal for cargo pickup. It can also be found as  pick up at depot, empty container depot, pickup depot, empty pickup location in the doc.
+
+TransportLeg: Details of each TransportLeg on the Delivery Order. Make sure to extract each TransportLeg information separately.
     eta: The estimated time of arrival for a specific leg.
     etd: The estimated time of departure for a specific leg.
     portOfDischarge: The port where cargo is unloaded.
     portOfLoading: The port where cargo is loaded.
     vesselName: The name of the vessel.
-    voyage: The journey or route code taken by the vessel.
-
-Your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
-SCHEMA_PLACEHOLDER
-
-Keywords for datapoints:
-- EmptyContainerDepot: Empty Container Depot
-- Equipment:
-      - CargoGrossWeight: Weight(incl. tare), Cargo Weight, Weight (KG)
-      - ContainerNumber: Container, Container Number, Container No.
-      - ContainerType: Type, Size/type
-      - EmptyReturnReference:  Drop off reference, turn-in reference
-      - Pin: Release reference, pin
-      - TareWeight: Tare weight, tare
-- pickUpTerminal: pick up at depot, empty container depot, pickup depot, empty pickup location
-- TransportLeg:
-    - eta: eta, ETA
-    - etd: etd, ETD
-    - portOfDischarge: to, PORT OF DISCHARGE
-    - portOfLoading: from, PORT OF LOADING
-    - vesselName: vessel
-    - voyage: voyage, voy. no, voyage-no.
-
+    voyage: The journey or route code taken by the vessel. It is often identified as voyage, voy. no, voyage-no in the document.
 
-You must apply the following rules:
-- The JSON schema must be followed during the extraction.
-- The values must only include text found in the document
-- Do not normalize any entity value.
-- portOfLoading and portOfDischarge are name of the Ports. You can rely on the port names from all over the world.
-- portOfLoading and portOfDischarge distinctly denotes the name of the ports. If you find abbreviation of the port use it, if not you can use the full name of the port
-- Validate the JSON make sure it is a valid JSON ! No extra text, no missing comma!
-- Add an escape character (backwards slash) in from of all quotes in values
+<INSTRUCTIONS>

src/prompts/library/shippingInstruction/other/placeholders.json

@@ -0,0 +1,115 @@
+{
+  "type": "OBJECT",
+  "properties": {
+      "consignee": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The receiver or buyer of the goods. It can be find with the keywords like Importeur, Anmelder, Empfanger, Consignee, Buyer, Receiver, etc.."
+      },
+      "finalDestination": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The ultimate location where the goods are to be delivered, marking the end point of the shipment's journey."
+      },
+      "freight": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The cost type associated with transporting goods. Can be classified as 'prepaid' or 'collect'."
+      },
+      "hblType": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The type of House Bill of Lading such as Telex Released, ORIGINAL B/L, EXPRESS, Sur Bill, Sea WayBill, etc., indicating the document issued by a freight forwarder that outlines the terms and details of the shipment."
+      },
+      "notify": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The party to be informed upon the arrival of the shipment at the destination. often responsible for coordinating the delivery. Extract the notify details including the address."
+      },
+      "placeOfReceipt": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The location where the goods are initially handed over to the freight forwarder or carrier for transportation"
+      },
+      "portOfDischarge": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The port where the goods are discharged from the vessel. This is the destination port for the shipment."
+      },
+      "portOfLoading": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The origin port where the goods are loaded onto the vessel. Find information like 'Ladehafen' or 'Port of Loading' in the invoice."
+      },
+      "shipper": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The sender or exporter of the goods. It can be find with the keywords like Absender, Versender, Shipper, Exporter, Supplier, Seller, etc.."
+      },
+      "containers": {
+          "type": "ARRAY",
+          "items": {
+              "type": "OBJECT",
+              "properties": {
+                  "cargoDescription": {
+                      "type": "STRING",
+                      "nullable": true,
+                      "description": "A brief description of the goods contained within the container. It can be found with goods description, Bezeichnung, goederenomschrijving."
+                  },
+                  "marksAndNumbers": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "Identification details printed or attached to packages for easy recognition during handling and customs procedures, ensuring accurate delivery. Extract the details including the numbers."
+                  },
+                  "hsCode": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "A numerical code from the Harmonized System used for classifying traded products. It helps in determining tariffs and regulations for the goods being shipped. Extract the full HS code including all digits."
+                  },
+                  "containerNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The unique identifier for each container. It always starts with 4 capital letters and followed by 7 digits. Example: TEMU7972458."
+                  },
+                  "containerType": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc."
+                  },
+                  "grossWeight": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The gross weight of the container. Usually mentioned as G.W or GW, Bruto, or Gross Weight, etc.."
+                  },
+                  "nettWeight": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The net weight of the container. Usually mentioned as N.W or NW, Net Weight, or Netto, Eigenmasse, etc.."
+                  },
+                  "measurements": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The volume of the goods. Usually, it is measured in 'Cubic Meter (cbm)' or dimensions. But volume in 'Cubic Meter (cbm)' is preferred if it’s available in the skus"
+                  },
+                  "packageQuantity": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The quantity of the goods. Usually, the quantity is in pallets, PLT, cartons, CTNS, pieces, PCS, packages, boxes, etc. Please prioritize the packaging types based on their size, as follows: Pallets (PLT) >> Cartons (CTNS) >> Pieces (PCS). Extract the Larger packaging types that will have a lower count."
+                  },
+                  "packagingType": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The packaging type is the unit of packageQuantity. Example; pallets, PLT, cartons, CTNS, pieces, PCS, packages, etc. Sometimes, the packaging type is available in the column name of the packageQuantity."
+                },
+                  "sealNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "A unique number associated with the container number. But it is not a container number. Usually mentioned as Seal No., Seal Number, Siegelnummer, etc.."
+                }
+              },
+              "required": ["cargoDescription", "containerNumber", "hsCode", "grossWeight", "nettWeight", "packageQuantity", "packagingType"]
+          }
+      }
+},
+  "required": ["shipper", "consignee", "portOfLoading", "portOfDischarge", "placeOfReceipt", "finalDestination", "freight", "hblType", "notify", "containers"]
+}

src/prompts/library/shippingInstruction/other/prompt.txt

@@ -1,16 +1,28 @@
-Task: Extract data from the provided shipping instruction PDF document and populate the following dictionary based on the given schema.
-Your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
 
-### Instructions:
-1. Extract all data points from the shipping instruction document.
-2. Each extracted data point must be part of a master field called "containers". There may be multiple "containers" entries in the document. Ensure you extract details for all instances.
-3. "Containers" Data Fields:
-   - Fill in the data fields as per the response schema provided.
-   - Always search for the Quantity mentioned as pallets, PLT, cartons, CTNS, pieces, PCS, packages, boxes, etc...
-   - If a field such as `containerNumber`, `sealNumber`, 'hsCode' or any other fields are not found within the "containers" section, search for these fields elsewhere in the document. Once located, populate the respective fields in all relevant "containers" entities.
-   - If the document contains only one container, use the total values for attributes like `grossWeight`, `netWeight`, `measurements`, and `packageQuantity` to populate the single container entry.
-   - Avoid creating separate entries for these shared attributes; instead, merge the data into the existing "containers" entries.
+<TASK> Your task is to extract data from Shipping Instruction documents as per the given response schema structure. <TASK>
 
-4. Output:
-   - Return the extracted data in JSON format.
-   - Exclude all other information from the response.
+<CONTEXT>
+The Freight Forwarding company receives Shipping Instruction from customers or shipper.
+These Shipping Instruction contain various details related to shipping information, as well as container data such as goods, HS code, container details and gross and net weight.
+They may be written in different languages such as English, German, Vietnamese, Chinese, and other European languages, and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these Shipping Instruction to support efficient processing and accurate record-keeping.
+<CONTEXT>
+
+<INSTRUCTIONS>
+- Populate fields as defined in the response schema.
+- Multiple Container entries may exist, capture all instances under "containers".
+- Use the data field description to understand the context of the data.
+
+- "containers" Data Fields: Details of each container on the Shipping Instruction. Make sure to extract each container information separately.
+   - containerNumber: Container Number always starts with 4 letters and is followed by 7 digits (e.g., ABCD1234567, XALU 8593678).
+   - cargoDescription: Extract only the description of the goods for the "cargoDescription" but not other information like packing, marks, etc.
+   - packageQuantity:
+     - Prioritize the "Pallets/PLTS/Cartons/CTNS/Package" over "PCS" count to extract the data for the "packageQuantity".
+     - example: If the table has "17CTNS", "9PLTS", "850", "850PCS", prioritize "9PLTS"
+     - Do not extract the pack Quantity field such as "50PCS/CTN", "5PC/Box" (these represent quantity per carton, not total shipped quantity).
+   - packagingType:
+     - Extract the unit associated with the "packageQuantity" in the table to extract the "packagingType"
+     - Sometimes it can be found on the column name of the "packageQuantity" in the table to extract the "packagingType"
+
+<INSTRUCTIONS>