data-science-document-ai 1.13.0__py3-none-any.whl → 1.56.1__py3-none-any.whl

src/prompts/library/partnerInvoice/other/placeholders.json

@@ -1,61 +1,181 @@
 {
     "type": "OBJECT",
     "properties": {
-        "bankAccount": {"type": "STRING"},
-        "contractNumber": {"type": "STRING"},
+        "bankAccount": {
+            "type": "ARRAY",
+            "items": {
+                "type": "STRING",
+                "nullable": true,
+                "description": "The bank account(s) number(s) of the vendor. This is the account to which the payment should be made. Extract all the relevant bank account numbers mentioned in the invoice."
+            }
+        },
+        "contractNumber": {"type": "STRING", "nullable": true,
+                    "description": "It's a contract number between the carrier and Forto Logistics SE & Co KG."},
         "currencyExchange": {
             "type": "OBJECT",
             "properties": {
-                "from": {"type": "STRING"},
-                "fxRate": {"type": "STRING"},
-                "to": {"type": "STRING"}
+                "from": {"type": "STRING",
+                    "nullable": true,
+                    "description": "The currency code from which the exchange rate is applied."
+                },
+                "fxRate": {"type": "STRING",
+                    "nullable": true,
+                    "description": "The exchange rate applied to convert the amount from the 'from' currency to the 'to' currency."
+                },
+                "to": {"type": "STRING",
+                    "nullable": true,
+                    "description": "The currency code to which the exchange rate is applied."}
             }
         },
-        "documentType": {"type": "STRING"},
-        "dueDate": {"type": "STRING"},
-        "eta": {"type": "STRING"},
-        "etd": {"type": "STRING"},
-        "fortoEntity": {"type": "STRING"},
-        "hblNumber": {"type": "STRING"},
-        "currencyCode": {"type": "STRING"},
-        "grandTotal": {"type": "STRING"},
-        "vatAmount": {"type": "STRING"},
-        "vatApplicableAmount": {"type": "STRING"},
-        "vatPercentage": {"type": "STRING"},
-        "invoiceNumber": {"type": "STRING"},
-        "issueDate": {"type": "STRING"},
+        "documentType": {"type": "STRING", "nullable": true},
+        "dueDate": {"type": "STRING", "nullable": true,
+            "description": "The date by which the payment should be made by Forto Logistics SE & Co KG. Do Not calculate dueDate based on issueDate or any other date. Extract it directly from the invoice."},
+        "eta": {"type": "STRING", "nullable": true,
+            "description": "Estimated Time of Arrival (ETA) is the expected date when the shipment will arrive at its destination."},
+        "etd": {"type": "STRING", "nullable": true,
+            "description": "Estimated Time of Departure (ETD) is the expected date when the shipment will leave the origin port."},
+        "fortoEntity": {"type": "STRING", "nullable": true,
+            "description": "The entity of 'Forto Logistics SE & Co KG' that is responsible for the invoice. The Forto organization or branch managing the shipment."
+        },
+        "hblNumber": {"type": "STRING", "nullable": true,
+            "description": "House Bill of Lading number, a document issued by a freight forwarder."
+        },
+        "currencyCode": {"type": "STRING", "nullable": true,
+            "description": "The currency code in which the invoice is issued, such as EUR, USD, etc."
+        },
+        "grandTotal": {"type": "STRING", "nullable": true,
+            "description": "The total amount of the invoice, including all line items and taxes."
+        },
+        "vatAmount": {"type": "STRING", "nullable": true,
+            "description": "The total VAT amount applied to the invoice. This is the tax charged on the vatApplicableAmount of the invoice. Bitte Zahlen is not the vatAmount."
+        },
+        "vatApplicableAmount": {"type": "STRING", "nullable": true,
+            "description": "The amount on which VAT is applicable. This is the net amount before VAT is applied (without VAT)."
+        },
+        "vatPercentage": {"type": "STRING", "nullable": true,
+            "description": "The percentage rate of VAT applied to the vatApplicableAmount. This is used to calculate the vatAmount."
+        },
+        "invoiceNumber": {"type": "STRING", "nullable": true,
+            "description": "The unique identifier for the invoice. This is used to track and reference the invoice in financial records."
+        },
+        "issueDate": {"type": "STRING", "nullable": true,
+            "description": "The date when the invoice was issued."
+        },
         "lineItem": {
             "type": "ARRAY",
             "items": {
                 "type": "OBJECT",
                 "properties": {
-                    "uniqueId": {"type": "STRING"},
-                    "lineItemDescription": {"type": "STRING"},
-                    "totalAmount": {"type": "STRING"},
-                    "totalAmountCurrency": {"type": "STRING"},
-                    "totalAmountEuro": {"type": "STRING"},
-                    "quantity": {"type": "STRING"},
-                    "unitPrice": {"type": "STRING"},
-                    "unitPriceCurrency": {"type": "STRING"},
-                    "vatAmount": {"type": "STRING"},
-                    "vatPercentage": {"type": "STRING"},
-                    "containerNumber": {"type": "STRING"},
-                    "containerSize": {"type": "STRING"}
+                    "uniqueId": {
+                        "type": "STRING",
+                        "nullable": true,
+                        "description": "A line item can belong to a different shipments. Hence, the unique IDs of a line item need to be extracted that you see only on the line item level. UniqueIds are containerNumber, shipmentId, or sealNumber."
+                    },
+                    "lineItemDescription": {"type": "STRING", "nullable": true,
+                        "description": "A description of the line item (COGS or Customs line items), which can include details about the service provided."},
+                    "totalAmount": {"type": "STRING", "nullable": true,
+                        "description": "The total amount for the line item, which may include the cost of services, and applicable taxes."},
+                    "totalAmountCurrency": {"type": "STRING", "nullable": true,
+                        "description": "The currency code for the total amount, such as EUR, USD, etc."},
+                    "totalAmountEuro": {"type": "STRING", "nullable": true,
+                        "description": "The total amount converted to Euro, if applicable. You can find it by looking for the term 'Total EUR' or 'Amount in Euro' in the line item."},
+                    "quantity": {"type": "STRING", "nullable": true,
+                        "description": "The quantity of the item or service provided in the line item."},
+                    "unitPrice": {"type": "STRING", "nullable": true,
+                        "description": "The price per unit of the item or service in the line item. Check the naming in a different languages, such as 'Einzelpreis', 'Unit Price', 'Prezzo unitario', 'Preis pro Einheit', etc.. Refer to 'Prezzo unitario' field in the italian invoice example"},
+                    "unitPriceCurrency": {"type": "STRING", "nullable": true,
+                        "description": "The currency code for the unit price, such as EUR, USD, etc."},
+                    "vatAmount": {"type": "STRING", "nullable": true,
+                        "description": "The VAT amount applied to the line item. This is the tax charged on the totalAmount of the line item."},
+                    "vatPercentage": {"type": "STRING", "nullable": true,
+                        "description": "The percentage rate of VAT applied to the totalAmount of the line item. This is used to calculate the vatAmount."
+                    },
+                    "containerNumber": {"type": "STRING", "nullable": true,
+                        "description": "The container number associated with the line item. containerNumber MUST start with 4 letters followed by 7 digits (e.g., CMAU1234567)"},
+                    "containerSize": {"type": "STRING", "nullable": true,
+                        "description": "The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc."}
                 }
-            }
+            },
+            "required": [
+                "uniqueId",
+                "lineItemDescription",
+                "totalAmount",
+                "totalAmountCurrency",
+                "totalAmountEuro",
+                "unitPrice",
+                "unitPriceCurrency",
+                "vatAmount",
+                "vatPercentage",
+                "containerNumber",
+                "containerSize"
+            ]
+        },
+        "mblNumber": {"type": "STRING", "nullable": true,
+            "description": "Bill of Lading number (B/L NO.), a document issued by the carrier."
+        },
+        "partnerReference": {"type": "STRING", "nullable": true,
+            "description": "A partnerReference can be a shipment ID.  It starts with 'S' followed by 6 or 8 digits (e.g., 'S1234567')."
         },
-        "mblNumber": {"type": "STRING"},
-        "partnerReference": {"type": "STRING"},
-        "paymentTerm": {"type": "STRING"},
-        "portOfDischarge": {"type": "STRING"},
-        "portOfLoading": {"type": "STRING"},
-        "recipientAddress": {"type": "STRING"},
-        "recipientName": {"type": "STRING"},
-        "serviceDate": {"type": "STRING"},
-        "vatId": {"type": "STRING"},
-        "vendorAddress": {"type": "STRING"},
-        "vendorName": {"type": "STRING"},
-        "reverseChargeSentence": {"type": "STRING"}
+        "paymentTerm": {"type": "STRING", "nullable": true,
+            "description": "The payment term indicates the conditions under which the payment should be made. E.g., 'In 10 TAGEN', '14 TAGEN', '14 days', etc."},
+        "portOfDischarge": {"type": "STRING", "nullable": true,
+            "description": "The port where the goods are discharged from the vessel. This is the destination port for the shipment."},
+        "portOfLoading": {"type": "STRING", "nullable": true,
+            "description": "The origin port where the goods are loaded onto the vessel. Find information like 'Ladehafen' or 'Port of Loading' in the invoice."},
+        "recipientAddress": {"type": "STRING", "nullable": true,
+            "description": "Majority of the times, it is 'Forto Logistics SE & Co KG' Address depends on the entity."},
+        "recipientName": {"type": "STRING", "nullable": true,
+            "description": "The name of the recipient who is responsible for making the payment. This is often the 'Forto Logistics SE & Co KG' entity or partner."},
+        "serviceDate": {"type": "STRING", "nullable": true,
+            "description": "The date when the service was provided. If Service date is not available in the invoice, Estimated Time of Arrival (ETA) can be used."},
+        "vatId": {"type": "STRING", "nullable": true,
+            "description": "The VAT ID of the vendor. This is used for tax purposes and to identify the vendor in financial transactions."},
+        "vendorAddress": {"type": "STRING", "nullable": true,
+            "description": "The address of the vendor to whom the payment should be made."},
+        "vendorName": {"type": "STRING", "nullable": true,
+            "description": "The name of the vendor to whom the payment should be made. Extract the main vendor details incase the invoice contains 'As Agent For'."},
+        "agentName": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The name of the agent or intermediary involved in the transaction, if applicable."},
+        "agentKeyWord": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "A keyword or phrase that indicates the presence of an agent or intermediary in the transaction, such as 'As Agent For', 'Acting Agent', 'Issuing agent', 'Contact Agent', or similar words."},
+
+        "reverseChargeSentence": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "A sentence which indicate that the reverse charge applies. Mostly found as VAT/Tax Clause."
+        }
+
     },
-    "required": []
-}
+    "required": [
+        "bankAccount",
+        "contractNumber",
+        "currencyExchange",
+        "documentType",
+        "eta",
+        "etd",
+        "fortoEntity",
+        "currencyCode",
+        "grandTotal",
+        "vatAmount",
+        "vatApplicableAmount",
+        "vatPercentage",
+        "invoiceNumber",
+        "issueDate",
+        "lineItem",
+        "mblNumber",
+        "partnerReference",
+        "paymentTerm",
+        "portOfDischarge",
+        "portOfLoading",
+        "recipientAddress",
+        "recipientName",
+        "serviceDate",
+        "vatId",
+        "vendorAddress",
+        "vendorName"
+    ]
+}

src/prompts/library/partnerInvoice/other/prompt.txt

@@ -1,53 +1,91 @@
-You are a document entity extraction specialist. Your task is to extract data from shipment and invoice documents.
-These documents contain critical information for invoices and cost items.
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
+
+<TASK> Your task is to extract data from invoice documents as per the given response schema structure. <TASK>
+
+<CONTEXT>
+The Freight Forwarding company receives invoices from Carrier (Shipping Lines) partners and Customs Brokers. These include Partner Invoices (COGS Invoices) and COGS Customs Invoices.
+These invoices contain various details related to shipments, cost line items, vendor and recipient information, as well as other financial data.
+They may be written in different languages such as English, German, Vietnamese, Chinese, and other European languages, and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these invoices to support efficient processing and accurate record-keeping.
+<CONTEXT>
+
+
+<INSTRUCTIONS>
+- Populate fields as defined in the response schema.
+- Multiple line item entries may exist, capture all instances under "lineItem".
+- Use the data field description to understand the context of the data.
 
-Entities to Extract:
-- bankAccount: The bank account number associated with the transaction.
 - contractNumber: The reference number for the contract related to the shipment.
 - currencyExchange: Details about currency conversion involved
     - from: The original currency code.
-    - fxRate: The exchange rate applied.
+    - fxRate: The exchange rate applied. In the most cases the value is between 0.0 and 1.5 and rarely exceeds 1.5.
     - to: The target currency code.
-- documentType: The classification of the document.
-- dueDate: The date by which payment or action is due.
-- eta: The estimated time of arrival for the shipment.
-- etd: The estimated time of departure for the shipment.
-- fortoEntity: The entity responsible for the shipment.
-- hblNumber: The House Bill of Lading number.
-- currencyCode: The currency in which the invoice is issued.
-- grandTotal: The overall total amount.
-- vatAmount: The value-added tax amount.
-- vatApplicableAmount: The amount to which VAT is applicable.
-- vatPercentage: The percentage rate of VAT applied.
-- invoiceNumber: The unique identifier for the invoice.
+
+- fortoEntity: The entity responsible for the shipment. It could be Forto branch from a different country. If branch is not available, it could be "Forto Logistics SE & Co KG".
+- hblNumber: The House Bill of Lading number. Commonly known as "Bill of Lading Number", "BILL OF LADING NO.", "BL Number", "BL No.", "B/L No.", "BL-Nr.", "B/L", or "HBL No.".
+
+- grandTotal:
+    - The overall total Gross amount. Including all line items, taxes, and fees.
+
+- vatApplicableAmount:
+    - Do not get confused with the prepaid and due amount. zu zahlen or Bitte Zahlen is not the vatApplicableAmount.
+    - vatApplicableAmount is the Net Amount of the invoice. Basically the total amount before VAT is applied. You can find it in the invoice total section and sometimes as "Netto"
+    - If vatAmount is not applied, then vatApplicableAmount can be same as grandTotal.
+    - Invoices from SGS maco customs service, "Total Kosten excl. MwSt." is not the vatApplicableAmount
+
+- vatAmount:
+    - The total VAT amount applied to the invoice. This is the tax charged on the vatApplicableAmount of the invoice.
+    - Do not get confused with the prepaid and due amount.
+    - Do not extract vatAmount from the line items, it should be extracted from the invoice total section.
+
+- vatId: Extract Vendor's VAT ID. "DE348812473" is Forto's VAT ID in Germany. Do not confuse it with the vendor's VAT ID.
+    - Remove "TVA" characters from VAT ID in MSC invoices. For example, CHE-111.954.803 TVA should be extracted as CHE-111.954.803
+
 - issueDate: The date the document was issued.
-- lineItem: Details of each item on the invoice
-    - uniqueId: A unique which associated with the lineItem. Either a shipmentId starting with an S and followed by 6 or 7 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
-    - lineItemDescription: The name or description of the item.
-    - totalAmount: The total amount for the item.
-    - totalAmountCurrency: The currency of the total amount.
-    - totalAmountEuro: The total amount converted to Euros.
-    - quantity: The unit of measurement for the item.
-    - unitPrice: The price per unit.
-    - unitPriceCurrency: The currency of the unit price.
-    - vatAmount: The VAT amount for the item.
-    - vatPercentage: The VAT percentage rate for the item.
-    - containerNumber: The unique identifier for each container.
-    - containerSize: The size classification of the container.
-- mblNumber: The Master Bill of Lading number.
-- partnerReference: The reference number for the partner involved.
-- paymentTerm: The terms of payment agreed upon.
-- pod: The port of discharge.
-- pol: The port of loading.
-- recipientAddress: The address of the recipient.
-- recipientName: The name of the recipient.
-- serviceDate: The date of service or transaction.
-- vatId: The VAT identification number.
-- vendorAddress: The address of the vendor.
-- vendorName: The name of the vendor.
+- dueDate: The date by which the payment should be made. Do Not calculate dueDate based on issueDate or any other date. Extract it directly from the invoice.
+
+- eta and etd: Few invoices contains same date for ARRIVED/DEPARTED or ETA/ETD. Extract it for both eta and etd.
+
+- lineItem: Details of each COGS and Customs line item on the invoice from each page. Make sure to extract each amount and currency separately.
+    - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 8 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
+    - lineItemDescription: The name or description of the item. Usually, it will be a one line sentence.
+    - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
+    - totalAmount: The total amount for the item. It can be in different currencies, so ensure to capture the currency as well for the totalAmountCurrency.
+    - totalAmountEuro: Few line items contains a total amount in Euro. You can find it by looking for the term "Total EUR" or "Amount in Euro" in the line item but it's always in the EURO / € currency. Sometimes, it can be same as totalAmount if the line item is already in Euro.
+    - quantity: The quantity of the item or service provided in the line item. Pay attention to 2 x 40HC or 2x40HC. It means, quantity is 2 and 40HC is containerSize but not 240.
+    - containerNumber: Container Number always starts with 4 letters and is followed by 7 digits (e.g., ABCD1234567, XALU 8593678).
+
+- hblNumber and mblNumber:
+    - The Master Bill of Lading number. Commonly known as "Bill of Lading Number", "BILL OF LADING NO.", "BL Number", "BL No.", "B/L No.", "BL-Nr.", "B/L", or "HBL No.".
+    - Do not confuse with the containerNumber that always starts with 4 letters and is followed by 7 digits (e.g., SEGU3090389). This is not the mblNumber or hblNumber.
+- partnerReference:
+    - Shipment_ID can be a reference number for the partner. Shipment_ID always starts with "S" followed by 6 or 8 digits (e.g., S2654361).
+    - If Shipment_ID is not available, extract any Booking Number as partnerReference.
+
+- vendorName and vendorAddress:
+    - Extract the name and address of the vendor who is the main parent company in the invoice. Do not extract the agent name and address as the vendorName or vendorAddress.
+    - If the invoice contains phrases such as "As Agent For" or "As Agent Of" in the vendor name or address, extract the main vendor’s details (the entity after / in front of the phrase) and ignore the local agent’s details.
+    - Example:
+        - "COSCO SHIPPING Lines Italy, Poland, or France S.R.L. – Genova Office – As Agent For COSCO SHIPPING Lines Co.,Ltd."
+        - vendorName: COSCO SHIPPING Lines Co.,Ltd.
+    - From Hapag-Lloyd invoices, look for "Ballindamm 25" address to extract the vendorAddress.
+
+- agentName: Name of the agent. Agencies are offices authorized to act on behalf of a company. This details usually available including the branch name of the parent company name in the invoice.
+- agentKeyWord:
+    - A keyword or phrase that indicates the presence of an agent or intermediary in the transaction, such as 'As Agent For', 'Acting Agent', 'Acting on behalf of', 'Issuing agent', 'Contact Agent', 'Service rendered by', 'C/O as agent for', 'Invoice issued in the name of', 'Diese Rechnung wird von uns als Agent im Namen und für Rechnung der XX ausgestellt', or similar words.
+    - Extract this field only if the agentName is present in the invoice. Do not assume from the references given above.
+
+- recipientName and recipientAddress: This is often the 'Forto Logistics SE & Co KG' entity or partner, and it's address. The address depends on the Forto entity responsible for the shipment.
+
+- serviceDate: The date of service provided. If the serviceDate is not specifically mentioned in the invoice, you can use the ETA of the shipment as a serviceDate.
 - reverseChargeSentence: A sentence which indicate that the reverse charge applies. Mostly fund as Tax Clause.
 
-Important Note:
-- Ensure all extracted values are directly from the document.
+IMPORTANT NOTE:
+- Ensure all extracted values are directly from the document. Do not make assumptions, modifications or calculations.
+- CustomSized invoices contain line items in a table format in the attached page. Table with headings Shipment ID, Partner Line Item Description, Quantity, Amount, and VAT. Extract all the line items from such tables from each page.
+- Do not split the quantity into different line items. e.g., if quantity is 2 or 2 CTR or 2 BIL, do not create 2 separate line items with quantity 1 each.
 - Do not normalize or modify any entity values.
-- If an entity is not found in the document, set the entity value to null.
+- Pay attention to the line item details and paymentInformation, as they may vary significantly across different invoices.
+
+
+<INSTRUCTIONS>

src/prompts/library/preprocessing/carrier/placeholders.json

@@ -10,21 +10,5 @@
       "OOCL",
       "Other"
     ]
-  },
-  "finalMbL": {
-    "type": "string",
-    "enum": [
-      "Hapag-Lloyd",
-      "Maersk",
-      "Other"
-    ]
-  },
-  "draftMbl": {
-  "type": "string",
-  "enum": [
-    "Hapag-Lloyd",
-    "Maersk",
-    "Other"
-  ]
   }
 }

src/prompts/library/shippingInstruction/other/placeholders.json

@@ -0,0 +1,115 @@
+{
+  "type": "OBJECT",
+  "properties": {
+      "consignee": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The receiver or buyer of the goods. It can be find with the keywords like Importeur, Anmelder, Empfanger, Consignee, Buyer, Receiver, etc.."
+      },
+      "finalDestination": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The ultimate location where the goods are to be delivered, marking the end point of the shipment's journey."
+      },
+      "freight": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The cost type associated with transporting goods. Can be classified as 'prepaid' or 'collect'."
+      },
+      "hblType": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The type of House Bill of Lading such as Telex Released, ORIGINAL B/L, EXPRESS, Sur Bill, Sea WayBill, etc., indicating the document issued by a freight forwarder that outlines the terms and details of the shipment."
+      },
+      "notify": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The party to be informed upon the arrival of the shipment at the destination. often responsible for coordinating the delivery. Extract the notify details including the address."
+      },
+      "placeOfReceipt": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The location where the goods are initially handed over to the freight forwarder or carrier for transportation"
+      },
+      "portOfDischarge": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The port where the goods are discharged from the vessel. This is the destination port for the shipment."
+      },
+      "portOfLoading": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The origin port where the goods are loaded onto the vessel. Find information like 'Ladehafen' or 'Port of Loading' in the invoice."
+      },
+      "shipper": {
+        "type": "STRING",
+        "nullable": true,
+        "description": "The sender or exporter of the goods. It can be find with the keywords like Absender, Versender, Shipper, Exporter, Supplier, Seller, etc.."
+      },
+      "containers": {
+          "type": "ARRAY",
+          "items": {
+              "type": "OBJECT",
+              "properties": {
+                  "cargoDescription": {
+                      "type": "STRING",
+                      "nullable": true,
+                      "description": "A brief description of the goods contained within the container. It can be found with goods description, Bezeichnung, goederenomschrijving."
+                  },
+                  "marksAndNumbers": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "Identification details printed or attached to packages for easy recognition during handling and customs procedures, ensuring accurate delivery. Extract the details including the numbers."
+                  },
+                  "hsCode": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "A numerical code from the Harmonized System used for classifying traded products. It helps in determining tariffs and regulations for the goods being shipped. Extract the full HS code including all digits."
+                  },
+                  "containerNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The unique identifier for each container. It always starts with 4 capital letters and followed by 7 digits. Example: TEMU7972458."
+                  },
+                  "containerType": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc."
+                  },
+                  "grossWeight": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The gross weight of the container. Usually mentioned as G.W or GW, Bruto, or Gross Weight, etc.."
+                  },
+                  "nettWeight": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The net weight of the container. Usually mentioned as N.W or NW, Net Weight, or Netto, Eigenmasse, etc.."
+                  },
+                  "measurements": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The volume of the goods. Usually, it is measured in 'Cubic Meter (cbm)' or dimensions. But volume in 'Cubic Meter (cbm)' is preferred if it’s available in the skus"
+                  },
+                  "packageQuantity": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The quantity of the goods. Usually, the quantity is in pallets, PLT, cartons, CTNS, pieces, PCS, packages, boxes, etc. Please prioritize the packaging types based on their size, as follows: Pallets (PLT) >> Cartons (CTNS) >> Pieces (PCS). Extract the Larger packaging types that will have a lower count."
+                  },
+                  "packagingType": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "The packaging type is the unit of packageQuantity. Example; pallets, PLT, cartons, CTNS, pieces, PCS, packages, etc. Sometimes, the packaging type is available in the column name of the packageQuantity."
+                },
+                  "sealNumber": {
+                    "type": "STRING",
+                    "nullable": true,
+                    "description": "A unique number associated with the container number. But it is not a container number. Usually mentioned as Seal No., Seal Number, Siegelnummer, etc.."
+                }
+              },
+              "required": ["cargoDescription", "containerNumber", "hsCode", "grossWeight", "nettWeight", "packageQuantity", "packagingType"]
+          }
+      }
+},
+  "required": ["shipper", "consignee", "portOfLoading", "portOfDischarge", "placeOfReceipt", "finalDestination", "freight", "hblType", "notify", "containers"]
+}

src/prompts/library/shippingInstruction/other/prompt.txt

@@ -1,15 +1,28 @@
-Task: Extract data from the provided shipping instruction PDF document and populate the following dictionary based on the given schema.
-
-### Instructions:
-1. Extract all data points from the shipping instruction document.
-2. Each extracted data point must be part of a master field called "containers". There may be multiple "containers" entries in the document. Ensure you extract details for all instances.
-3. "Containers" Data Fields:
-   - Fill in the data fields as per the response schema provided.
-   - Always search for the Quantity mentioned as pallets, PLT, cartons, CTNS, pieces, PCS, packages, boxes, etc...
-   - If a field such as `containerNumber`, `sealNumber`, 'hsCode' or any other fields are not found within the "containers" section, search for these fields elsewhere in the document. Once located, populate the respective fields in all relevant "containers" entities.
-   - If the document contains only one container, use the total values for attributes like `grossWeight`, `netWeight`, `measurements`, and `packageQuantity` to populate the single container entry.
-   - Avoid creating separate entries for these shared attributes; instead, merge the data into the existing "containers" entries.
-
-4. Output:
-   - Return the extracted data in JSON format.
-   - Exclude all other information from the response.
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
+
+<TASK> Your task is to extract data from Shipping Instruction documents as per the given response schema structure. <TASK>
+
+<CONTEXT>
+The Freight Forwarding company receives Shipping Instruction from customers or shipper.
+These Shipping Instruction contain various details related to shipping information, as well as container data such as goods, HS code, container details and gross and net weight.
+They may be written in different languages such as English, German, Vietnamese, Chinese, and other European languages, and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these Shipping Instruction to support efficient processing and accurate record-keeping.
+<CONTEXT>
+
+<INSTRUCTIONS>
+- Populate fields as defined in the response schema.
+- Multiple Container entries may exist, capture all instances under "containers".
+- Use the data field description to understand the context of the data.
+
+- "containers" Data Fields: Details of each container on the Shipping Instruction. Make sure to extract each container information separately.
+   - containerNumber: Container Number always starts with 4 letters and is followed by 7 digits (e.g., ABCD1234567, XALU 8593678).
+   - cargoDescription: Extract only the description of the goods for the "cargoDescription" but not other information like packing, marks, etc.
+   - packageQuantity:
+     - Prioritize the "Pallets/PLTS/Cartons/CTNS/Package" over "PCS" count to extract the data for the "packageQuantity".
+     - example: If the table has "17CTNS", "9PLTS", "850", "850PCS", prioritize "9PLTS"
+     - Do not extract the pack Quantity field such as "50PCS/CTN", "5PC/Box" (these represent quantity per carton, not total shipped quantity).
+   - packagingType:
+     - Extract the unit associated with the "packageQuantity" in the table to extract the "packagingType"
+     - Sometimes it can be found on the column name of the "packageQuantity" in the table to extract the "packagingType"
+
+<INSTRUCTIONS>