data-science-document-ai 1.60.2__tar.gz → 1.61.0__tar.gz

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.60.2
+Version: 1.61.0
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-science-document-ai"
-version = "1.60.2"
+version = "1.61.0"
 description = "\"Document AI repo for data science\""
 authors = ["Naomi Nguyen <naomi.nguyen@forto.com>", "Kumar Rajendrababu <kumar.rajendrababu@forto.com>", "Igor Tonko <igor.tonko@forto.com>", "Osman Demirel <osman.demirel@forto.com>"]
 packages = [

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/postprocessing/common.py

@@ -723,6 +723,17 @@ async def format_all_entities(result, document_type_code, params, mime_type):
     if document_type_code in ["partnerInvoice", "bundeskasse"]:
         await process_partner_invoice(params, aggregated_data, document_type_code)
 
+    # TODO: This is a temporary change until the terminal codes are updated
+    if document_type_code == "bookingConfirmation":
+        if "gateInTerminalCode" in aggregated_data:
+            aggregated_data["gateInTerminal"] = aggregated_data.pop(
+                "gateInTerminalCode"
+            )
+        if "pickUpTerminalCode" in aggregated_data:
+            aggregated_data["pickUpTerminal"] = aggregated_data.pop(
+                "pickUpTerminalCode"
+            )
+
     logger.info("Data Extraction completed successfully")
     return aggregated_data
 

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/bundeskasse/other/placeholders.json

@@ -93,14 +93,14 @@
       "invoiceNumber": {
         "type": "STRING",
         "nullable": true,
-        "description": "Invoice Number is a unique identifier for the invoice, it starts with \"ATC\", \"AT-C\", or \"AT/C\" only (e.g., ATC40, AT-C-40-, AT/C/40/....). Do NOT extract \"NIZZA-Registrierkennzeichen number."
+        "description": "Invoice Number is a unique identifier for the invoice, it starts with ATC, AT-C, or AT/C only (e.g., ATC40, AT-C-40-, AT/C/40/....) It can be found just below the title of the invoice or in the top section of the invoice. Do NOT extract NIZZA-Registrierkennzeichen number (e.g. ATC0040M00...)."
       },
       "containerNumber": {
         "type": "ARRAY",
         "items": {
           "type": "STRING",
           "nullable": true,
-          "description": "The unique identifier for each container. It always starts with 4 capital letters and followed by 7 digits. Example: TEMU7972458."
+          "description": "The unique identifier for each container. It always starts with 4 capital letters and followed by 7 digits. Example: TEMU7972458. Do not get confused between 0 vs O in the 7 digits of container number."
       }
   },
       "creditNoteInvoiceNumber": {

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/bundeskasse/other/prompt.txt

@@ -16,7 +16,7 @@ Your role is to accurately extract specific entities from these Customs invoices
 - The amount and the currency is always in EUR both for grandTotal and line items.
 
 - containerNumber:
-    - Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU7222892).
+    - Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU7222892). Do not get confused between 0 vs O in the 7 digits of container number.
     - Few invoices contains multiple container numbers, in that case, all container numbers should be captured.
 
 - shipmentID:
@@ -26,7 +26,8 @@ Your role is to accurately extract specific entities from these Customs invoices
 
 - invoiceNumber:
     - Invoice Number is a unique identifier for the invoice, it starts with "ATC", "AT-C", or "AT/C" only (e.g., ATC40..., AT-C-40-..., AT/C/40/....).
-    - Do NOT extract if the text is about vehicle registrations, license plates, or location identifiers (e.g., "NIZZA-Registrierkennzeichen: ATC0040....")
+    - It can be found just below the title of the invoice or in the top section of the invoice.
+    - Do NOT extract if the text is about vehicle registrations, license plates, or location identifiers (e.g., "NIZZA-Registrierkennzeichen: ATC0040M00....")
 
 - creditNoteInvoiceNumber:
     - Credit Note Invoice Number is a unique identifier for the credit note, it starts with "ATS" only (e.g., ATS.....).

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/customsInvoice/other/placeholders.json

@@ -75,25 +75,34 @@
                     },
                     "lineItemDescription": {"type": "STRING", "nullable": true,
                         "description": "A description of the line item (COGS or Customs line items), which can include details about the service provided."},
+                    "totalAmountCurrency": {"type": "STRING", "nullable": true,
+                        "description": "The currency code for the line item's total amount, such as EUR, USD, etc."},
                     "totalAmount": {"type": "STRING", "nullable": true,
                         "description": "The total amount for the line item, which may include the cost of services, and applicable taxes."},
-                    "totalAmountCurrency": {"type": "STRING", "nullable": true,
-                        "description": "The currency code for the total amount, such as EUR, USD, etc."},
+                    "totalAmountInOriginalCurrency": {
+                        "type": "STRING",
+                        "description": "The total amount for the line item in original currency"
+                    },
+                    "totalAmountInInvoiceCurrency": {
+                        "type": "STRING",
+                        "description": "The total amount for the line item in the currency code the whole invoice's total uses"
+                    },
                     "totalAmountEuro": {"type": "STRING", "nullable": true,
                         "description": "The total amount converted to Euro, if applicable. You can find it by looking for the term 'Total EUR' or 'Amount in Euro' in the line item."},
                     "quantity": {"type": "STRING", "nullable": true,
                         "description": "The quantity of the item or service provided in the line item."},
-                    "unitPrice": {"type": "STRING", "nullable": true,
-                        "description": "The price per unit of the item or service in the line item. Check the naming in a different languages, such as 'Einzelpreis', 'Unit Price', 'Prezzo unitario', 'Preis pro Einheit', etc.. Refer to 'Prezzo unitario' field in the italian invoice example"},
                     "unitPriceCurrency": {"type": "STRING", "nullable": true,
                         "description": "The currency code for the unit price, such as EUR, USD, etc."},
+                    "unitPrice": {"type": "STRING", "nullable": true,
+                        "description": "The price per unit of the item or service in the line item. Check the naming in a different languages, such as 'Einzelpreis', 'Unit Price', 'Prezzo unitario', 'Preis pro Einheit', etc.. Refer to 'Prezzo unitario' field in the italian invoice example"},
                     "vatAmount": {"type": "STRING", "nullable": true,
                         "description": "The VAT amount applied to the line item. This is the tax charged on the totalAmount of the line item."},
                     "vatPercentage": {"type": "STRING", "nullable": true,
                         "description": "The percentage rate of VAT applied to the totalAmount of the line item. This is used to calculate the vatAmount."
                     },
                     "containerNumber": {"type": "STRING", "nullable": true,
-                        "description": "The container number associated with the line item. containerNumber MUST start with 4 letters followed by 7 digits (e.g., CMAU1234567)"},
+                        "description": "The container number associated with the line item. containerNumber MUST start with 4 letters followed by 7 digits (e.g., CMAU1234567). Do not get confused between 0 vs O in the 7 digits of container number."
+                    },
                     "containerSize": {"type": "STRING", "nullable": true,
                         "description": "The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc."}
                 }

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/customsInvoice/other/prompt.txt

@@ -55,11 +55,18 @@ Your role is to accurately extract specific entities from these invoices to supp
 - lineItem: Details of each COGS and Customs line item on the invoice from each page. Make sure to extract each amount and currency separately.
     - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 8 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
     - lineItemDescription: The name or description of the item. Usually, it will be a one line sentence.
-    - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
-    - totalAmount: The total amount for the item. It can be in different currencies, so ensure to capture the currency as well for the totalAmountCurrency.
+    - totalAmountCurrency: the original/native currency code for the line item's total amount. In 3 letters such as USD, EUR
+    - totalAmount: The total amount for the item. It can be in different currencies, so ensure to capture the amount according to the totalAmountCurrency.
+    - totalAmountInOriginalCurrency: line item amount in its original/native currency
+    - totalAmountInInvoiceCurrency: line item amount converted to the invoice's main currency (the currency used for grandTotal)
     - totalAmountEuro: Few line items contains a total amount in Euro. You can find it by looking for the term "Total EUR" or "Amount in Euro" in the line item but it's always in the EURO / € currency. Sometimes, it can be same as totalAmount if the line item is already in Euro.
-    - quantity: The quantity of the item or service provided in the line item. Pay attention to 2 x 40HC or 2x40HC. It means, quantity is 2 and 40HC is containerSize but not 240.
-    - containerNumber: Container Number always starts with 4 letters and is followed by 7 digits (e.g., ABCD1234567).
+    - quantity: The quantity of the item or service provided in the line item. Pay attention to 2 x 40HC or 2x40HC. It means, quantity is 2 and 40HC is containerSize
+    - unitPriceCurrency: The original/native currency code for the unit price. 3 letters
+    - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
+    - vatAmount: The VAT amount applied to the line item. This is the tax charged on the totalAmount of the line item
+    - vatPercentage: The percentage rate of VAT applied to the totalAmount of the line item. This is used to calculate the vatAmount
+    - containerNumber: Container Number always starts with 4 letters and is followed by 7 digits (e.g., ABCD1234567, XALU 8593678).
+    - containerSize: The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc.
 
 - hblNumber and mblNumber:
     - The Master Bill of Lading number. Commonly known as "Bill of Lading Number", "BILL OF LADING NO.", "BL Number", "BL No.", "B/L No.", "BL-Nr.", "B/L", or "HBL No.".

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/draftMbl/other/prompt.txt

@@ -28,7 +28,7 @@ Your role is to accurately extract specific entities from these draftMBLs to sup
     - Vessel Name is the name of the ship carrying the cargo. It can be referred to as "Vessel", "Ship Name", "Schiff", "Schiffsname", "Nave", or "Vessel/Flight No.".
 
 - containers: Details of each container on the draftMBL. Make sure to extract each container information separately.
-    - containerNumber: Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU 7222892).
+    - containerNumber: Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU 7222892). Do not get confused between 0 vs O in the 7 digits of container number.
     - sealNumber: Seal numbers are unique identifiers for shipping seals. They are usually mentioned as seal numbers in the document but they are definitely not container numbers.
 
 <INSTRUCTIONS>

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/finalMbL/other/prompt.txt

@@ -28,7 +28,7 @@ Your role is to accurately extract specific entities from these finalMBLs to sup
     - Vessel Name is the name of the ship carrying the cargo. It can be referred to as "Vessel", "Ship Name", "Schiff", "Schiffsname", "Nave", or "Vessel/Flight No.".
 
 - containers: Details of each container on the finalMBL. Make sure to extract each container information separately.
-    - containerNumber: Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU 7222892).
+    - containerNumber: Container Number consists of 4 capital letters followed by 7 digits (e.g., TEMU7972458, CAIU 7222892). Do not get confused between 0 vs O in the 7 digits of container number.
     - sealNumber: Seal numbers are unique identifiers for shipping seals. They are usually mentioned as seal numbers in the document but they are definitely not container numbers.
 
 <INSTRUCTIONS>

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/partnerInvoice/other/placeholders.json

@@ -73,25 +73,34 @@
                     },
                     "lineItemDescription": {"type": "STRING", "nullable": true,
                         "description": "A description of the line item (COGS or Customs line items), which can include details about the service provided."},
+                    "totalAmountCurrency": {"type": "STRING", "nullable": true,
+                        "description": "The currency code for the line item's total amount, such as EUR, USD, etc."},
                     "totalAmount": {"type": "STRING", "nullable": true,
                         "description": "The total amount for the line item, which may include the cost of services, and applicable taxes."},
-                    "totalAmountCurrency": {"type": "STRING", "nullable": true,
-                        "description": "The currency code for the total amount, such as EUR, USD, etc."},
+                    "totalAmountInOriginalCurrency": {
+                        "type": "STRING",
+                        "description": "The total amount for the line item in original currency"
+                    },
+                    "totalAmountInInvoiceCurrency": {
+                        "type": "STRING",
+                        "description": "The total amount for the line item in the currency code the whole invoice's total uses"
+                    },
                     "totalAmountEuro": {"type": "STRING", "nullable": true,
                         "description": "The total amount converted to Euro, if applicable. You can find it by looking for the term 'Total EUR' or 'Amount in Euro' in the line item."},
                     "quantity": {"type": "STRING", "nullable": true,
                         "description": "The quantity of the item or service provided in the line item."},
-                    "unitPrice": {"type": "STRING", "nullable": true,
-                        "description": "The price per unit of the item or service in the line item. Check the naming in a different languages, such as 'Einzelpreis', 'Unit Price', 'Prezzo unitario', 'Preis pro Einheit', etc.. Refer to 'Prezzo unitario' field in the italian invoice example"},
                     "unitPriceCurrency": {"type": "STRING", "nullable": true,
                         "description": "The currency code for the unit price, such as EUR, USD, etc."},
+                    "unitPrice": {"type": "STRING", "nullable": true,
+                        "description": "The price per unit of the item or service in the line item. Check the naming in a different languages, such as 'Einzelpreis', 'Unit Price', 'Prezzo unitario', 'Preis pro Einheit', etc.. Refer to 'Prezzo unitario' field in the italian invoice example"},
                     "vatAmount": {"type": "STRING", "nullable": true,
                         "description": "The VAT amount applied to the line item. This is the tax charged on the totalAmount of the line item."},
                     "vatPercentage": {"type": "STRING", "nullable": true,
                         "description": "The percentage rate of VAT applied to the totalAmount of the line item. This is used to calculate the vatAmount."
                     },
                     "containerNumber": {"type": "STRING", "nullable": true,
-                        "description": "The container number associated with the line item. containerNumber MUST start with 4 letters followed by 7 digits (e.g., CMAU1234567)"},
+                        "description": "The container number associated with the line item. containerNumber MUST start with 4 letters followed by 7 digits (e.g., CMAU1234567). Do not get confused between 0 vs O in the 7 digits of container number."
+                    },
                     "containerSize": {"type": "STRING", "nullable": true,
                         "description": "The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc."}
                 }

{data_science_document_ai-1.60.2 → data_science_document_ai-1.61.0}/src/prompts/library/partnerInvoice/other/prompt.txt

@@ -53,11 +53,18 @@ Your role is to accurately extract specific entities from these invoices to supp
 - lineItem: Details of each COGS and Customs line item on the invoice from each page. Make sure to extract each amount and currency separately.
     - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 8 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
     - lineItemDescription: The name or description of the item. Usually, it will be a one line sentence.
-    - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
-    - totalAmount: The total amount for the item. It can be in different currencies, so ensure to capture the currency as well for the totalAmountCurrency.
+    - totalAmountCurrency: original/native currency code for the line item's total amount. In 3 letters such as USD, EUR
+    - totalAmount: The total amount for the item. It can be in different currencies, so ensure to capture the amount according to the totalAmountCurrency.
+    - totalAmountInOriginalCurrency: line item amount in its original/native currency
+    - totalAmountInInvoiceCurrency: line item amount converted to the invoice's main currency (the currency used for grandTotal)
     - totalAmountEuro: Few line items contains a total amount in Euro. You can find it by looking for the term "Total EUR" or "Amount in Euro" in the line item but it's always in the EURO / € currency. Sometimes, it can be same as totalAmount if the line item is already in Euro.
-    - quantity: The quantity of the item or service provided in the line item. Pay attention to 2 x 40HC or 2x40HC. It means, quantity is 2 and 40HC is containerSize but not 240.
+    - quantity: The quantity of the item or service provided in the line item. Pay attention to 2 x 40HC or 2x40HC. It means, quantity is 2 and 40HC is containerSize
+    - unitPriceCurrency: The original/native currency code for the unit price. 3 letters
+    - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
+    - vatAmount: The VAT amount applied to the line item. This is the tax charged on the totalAmount of the line item
+    - vatPercentage: The percentage rate of VAT applied to the totalAmount of the line item. This is used to calculate the vatAmount
     - containerNumber: Container Number always starts with 4 letters and is followed by 7 digits (e.g., ABCD1234567, XALU 8593678).
+    - containerSize: The size of the container associated with the containerNumber, such as 20ft, 40ft, 40HC, 20DC etc.
 
 - hblNumber and mblNumber:
     - The Master Bill of Lading number. Commonly known as "Bill of Lading Number", "BILL OF LADING NO.", "BL Number", "BL No.", "B/L No.", "BL-Nr.", "B/L", or "HBL No.".