data-science-document-ai 1.42.3__tar.gz → 1.42.4__tar.gz

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.42.3
+Version: 1.42.4
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "data-science-document-ai"
-version = "1.42.3"
+version = "1.42.4"
 description = "\"Document AI repo for data science\""
 authors = ["Naomi Nguyen <naomi.nguyen@forto.com>", "Kumar Rajendrababu <kumar.rajendrababu@forto.com>", "Igor Tonko <igor.tonko@forto.com>", "Osman Demirel <osman.demirel@forto.com>"]
 packages = [

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/src/postprocessing/common.py

@@ -84,16 +84,16 @@ def clean_shipment_id(shipment_id):
     """
     if not shipment_id:
         return
-    # '#S123456@-1' -> 'S123456'
-    # Find the pattern of a shipment ID that starts with 'S' followed by 5 to 7 digits
-    match = re.findall(r"S\d{5,7}", shipment_id)
+    # '#S1234565@-1' -> 'S1234565'
+    # Find the pattern of a shipment ID that starts with 'S' followed by 7 to 8 digits
+    match = re.findall(r"S\d{6,8}", shipment_id)
     stripped_value = match[0] if match else None
 
     if not stripped_value:
         return None
 
     # Check if length is valid (should be either 7 or 8)
-    if len(stripped_value) not in (6, 7, 8):
+    if len(stripped_value) not in (7, 8, 9):
         return None
 
     return stripped_value
@@ -425,7 +425,7 @@ async def format_label(entity_k, entity_value, document_type_code, params):
         formatted_value = clean_invoice_number(entity_value)
 
     elif entity_key in ("shipmentid", "partnerreference"):
-        # Clean the shipment ID to match Forto's standard (starts with 'S' followed by 5 to 7 digits)
+        # Clean the shipment ID to match Forto's standard (starts with 'S' followed by 7 or 8 digits)
         formatted_value = clean_shipment_id(entity_value)
 
     elif entity_key == "containernumber":

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/src/prompts/library/bundeskasse/other/placeholders.json

@@ -34,7 +34,7 @@
       "shipmentId": {
         "type": "STRING",
         "nullable": true,
-        "description": "Starting with an \"S\" and followed by 6 or 7 digits. Example: S124321"
+        "description": "Starting with an \"S\" and followed by 6 or 8 digits. Example: S1243213 or S12876549"
       },
       "vendorName": {
         "type": "STRING",

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/src/prompts/library/bundeskasse/other/prompt.txt

@@ -19,7 +19,7 @@ Your role is to accurately extract specific entities from these Customs invoices
     - Few invoices contains multiple container numbers, in that case, all container numbers should be captured.
 
 - shipmentID:
-    - Shipment ID is a unique identifier for the shipment, it starts with "S" followed by 6-7 digits (e.g., S123058).
+    - Shipment ID is a unique identifier for the shipment, it starts with "S" followed by 6-8 digits (e.g., S1230583 or S12305876).
     - It can be found in the top section of the invoice. Few times, it can be found in between a long string of numbers as well. (e.g., "FORTO-S136748-").
     - It can also be referred to as "Bezugsnummer" in the invoice.
 

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/src/prompts/library/customsInvoice/other/placeholders.json

@@ -116,7 +116,7 @@
             "description": "Bill of Lading number (B/L NO.), a document issued by the carrier."
         },
         "partnerReference": {"type": "STRING", "nullable": true,
-            "description": "A partnerReference can be a shipment ID.  It starts with 'S' followed by 6 or 7 digits (e.g., 'S1234567')."
+            "description": "A partnerReference can be a shipment ID.  It starts with 'S' followed by 6 or 8 digits (e.g., 'S1234567')."
         },
         "paymentTerm": {"type": "STRING", "nullable": true,
             "description": "The payment term indicates the conditions under which the payment should be made. E.g., 'In 10 TAGEN', '14 TAGEN', '14 days', etc."},

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/src/prompts/library/customsInvoice/other/prompt.txt

@@ -49,7 +49,7 @@ Your role is to accurately extract specific entities from these invoices to supp
 - dueDate: The date by which the payment should be made. Do Not calculate dueDate based on issueDate or any other date. Extract it directly from the invoice.
 
 - lineItem: Details of each COGS and Customs line item on the invoice. Make sure to extract each amount and currency separately.
-    - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 7 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
+    - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 8 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
     - lineItemDescription: The name or description of the item. Usually, it will be a one line sentence.
     - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
     - totalAmount: The total amount for the item. It can be in different currencies, so ensure to capture the currency as well for the totalAmountCurrency.
@@ -60,7 +60,7 @@ Your role is to accurately extract specific entities from these invoices to supp
 - hblNumber and mblNumber:
     - The Master Bill of Lading number. Commonly known as "Bill of Lading Number", "BILL OF LADING NO.", "BL Number", "BL No.", "B/L No.", "BL-Nr.", "B/L", or "HBL No.".
     - Do not confuse with the containerNumber that always starts with 4 letters and is followed by 7 digits (e.g., SEGU3090389). This is not the mblNumber or hblNumber.
-- partnerReference: Shipment_ID can be a reference number for the partner. Shipment_ID always starts with "S" followed by 6 or 7 digits (e.g., S2654361).
+- partnerReference: Shipment_ID can be a reference number for the partner. Shipment_ID always starts with "S" followed by 6 or 8 digits (e.g., S2654361).
 
 - vendorName and vendorAddress:
     - The name and address of the vendor providing the service and to whom the payment should be made.

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/src/prompts/library/partnerInvoice/other/placeholders.json

@@ -110,18 +110,45 @@
                 "containerSize"
             ]
         },
-        "mblNumber": {"type": "STRING", "nullable": true},
-        "partnerReference": {"type": "STRING", "nullable": true},
-        "paymentTerm": {"type": "STRING", "nullable": true},
-        "portOfDischarge": {"type": "STRING", "nullable": true},
-        "portOfLoading": {"type": "STRING", "nullable": true},
-        "recipientAddress": {"type": "STRING", "nullable": true},
-        "recipientName": {"type": "STRING", "nullable": true},
-        "serviceDate": {"type": "STRING", "nullable": true},
-        "vatId": {"type": "STRING", "nullable": true},
-        "vendorAddress": {"type": "STRING", "nullable": true},
-        "vendorName": {"type": "STRING", "nullable": true},
-        "reverseChargeSentence": {"type": "STRING", "nullable": true}
+        "mblNumber": {"type": "STRING", "nullable": true,
+            "description": "Bill of Lading number (B/L NO.), a document issued by the carrier."
+        },
+        "partnerReference": {"type": "STRING", "nullable": true,
+            "description": "A partnerReference can be a shipment ID.  It starts with 'S' followed by 6 or 8 digits (e.g., 'S1234567')."
+        },
+        "paymentTerm": {"type": "STRING", "nullable": true,
+            "description": "The payment term indicates the conditions under which the payment should be made. E.g., 'In 10 TAGEN', '14 TAGEN', '14 days', etc."},
+        "portOfDischarge": {"type": "STRING", "nullable": true,
+            "description": "The port where the goods are discharged from the vessel. This is the destination port for the shipment."},
+        "portOfLoading": {"type": "STRING", "nullable": true,
+            "description": "The origin port where the goods are loaded onto the vessel. Find information like 'Ladehafen' or 'Port of Loading' in the invoice."},
+        "recipientAddress": {"type": "STRING", "nullable": true,
+            "description": "Majority of the times, it is 'Forto Logistics SE & Co KG' Address depends on the entity."},
+        "recipientName": {"type": "STRING", "nullable": true,
+            "description": "The name of the recipient who is responsible for making the payment. This is often the 'Forto Logistics SE & Co KG' entity or partner."},
+        "serviceDate": {"type": "STRING", "nullable": true,
+            "description": "The date when the service was provided. If Service date is not available in the invoice, Estimated Time of Arrival (ETA) can be used."},
+        "vatId": {"type": "STRING", "nullable": true,
+            "description": "The VAT ID of the vendor. This is used for tax purposes and to identify the vendor in financial transactions."},
+        "vendorAddress": {"type": "STRING", "nullable": true,
+            "description": "The address of the vendor to whom the payment should be made."},
+        "vendorName": {"type": "STRING", "nullable": true,
+            "description": "The name of the vendor to whom the payment should be made. Extract the main vendor details incase the invoice contains 'As Agent For'."},
+        "agentName": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The name of the agent or intermediary involved in the transaction, if applicable."},
+        "agentKeyWord": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "A keyword or phrase that indicates the presence of an agent or intermediary in the transaction, such as 'As Agent For', 'Acting Agent', 'Issuing agent', 'Contact Agent', or similar words."},
+
+        "reverseChargeSentence": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "A sentence which indicate that the reverse charge applies. Mostly found as VAT/Tax Clause."
+        }
+
     },
     "required": [
         "bankAccount",

{data_science_document_ai-1.42.3 → data_science_document_ai-1.42.4}/src/prompts/library/partnerInvoice/other/prompt.txt

@@ -47,7 +47,7 @@ Your role is to accurately extract specific entities from these invoices to supp
 - eta and etd: Few invoices contains same date for ARRIVED/DEPARTED or ETA/ETD. Extract it for both eta and etd.
 
 - lineItem: Details of each COGS and Customs line item on the invoice. Make sure to extract each amount and currency separately.
-    - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 7 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
+    - uniqueId: A unique id which associated with the lineItem as each line item can belong to a different shipment. Extract only if its available in the line item. Either a shipmentId starting with an S and followed by 6 or 8 numeric values or a mblNumber. If shipmentId or mblNumber does not exist, set it to containerNumber.
     - lineItemDescription: The name or description of the item. Usually, it will be a one line sentence.
     - unitPrice: Even if the quantity is not mentioned, you can still extract the unit price. Check the naming of the columns in a different languages, it can be "Unit Price", "Prezzo unitario", "Prix Unitaire", "Unitario", etc. Refer to "Prezzo unitario" field in the italian invoice example.
     - totalAmount: The total amount for the item. It can be in different currencies, so ensure to capture the currency as well for the totalAmountCurrency.
@@ -59,7 +59,7 @@ Your role is to accurately extract specific entities from these invoices to supp
     - The Master Bill of Lading number. Commonly known as "Bill of Lading Number", "BILL OF LADING NO.", "BL Number", "BL No.", "B/L No.", "BL-Nr.", "B/L", or "HBL No.".
     - Do not confuse with the containerNumber that always starts with 4 letters and is followed by 7 digits (e.g., SEGU3090389). This is not the mblNumber or hblNumber.
 - partnerReference:
-    - Shipment_ID can be a reference number for the partner. Shipment_ID always starts with "S" followed by 6 or 7 digits (e.g., S2654361).
+    - Shipment_ID can be a reference number for the partner. Shipment_ID always starts with "S" followed by 6 or 8 digits (e.g., S2654361).
     - If Shipment_ID is not available, extract any Booking Number as partnerReference.
 
 - vendorName and vendorAddress:
@@ -79,16 +79,6 @@ Your role is to accurately extract specific entities from these invoices to supp
 - serviceDate: The date of service provided. If the serviceDate is not specifically mentioned in the invoice, you can use the ETA of the shipment as a serviceDate.
 - reverseChargeSentence: A sentence which indicate that the reverse charge applies. Mostly fund as Tax Clause.
 
-- paymentInformation:
-    - Some partners receive prepayment before providing the service. They later send a final invoice that includes both the amount already paid and the remaining amount due.
-    - This applies when the invoice contains prepayment-related terms such as Vorschuss, BEREITS BEZAHLT, or similar at the invoice total section.
-    - do not get confused with the paidAmount and remainingAmountToPay. Few invoices may not have a paidAmount or remainingAmountToPay in such cases pay attention to the sentence field alignment.
-    - Extract the following fields, if applicable:
-        - paidAmount: The amount that has already been paid. You can identify this in the invoice by looking for terms like "Vorschuss", "BEREITS BEZAHLT".
-        - remainingAmountToPay: The amount still due. This can be negative if the paid amount exceeds the total invoice amount. Ensure the negative sign is captured if applicable. You can identify this by looking for terms like "Bitte Zahlen", "Zu zahlen", "Remaining Amount", "To Pay", "Due", or "Unpaid".
-        - currency: The currency of both the paid and remaining amounts.
-        - sentence: A sentence from the invoice indicating the payment status (e.g., "Vorschuss", "Prepayment", "Paid", "Partially Paid", "Unpaid"). This helps summarize the overall payment status of the invoice.
-
 IMPORTANT NOTE:
 - Ensure all extracted values are directly from the document. Do not make assumptions, modifications or calculations.
 - Do not normalize or modify any entity values.
@@ -96,8 +86,5 @@ IMPORTANT NOTE:
 
 PAY ATTENTION TO THE SGS MACO CUSTOMS SERVICE INVOICES:
 - invoices from SGS maco customs service,
-  - Extract only "Vorschuss" as a paidAmount but not "Vorauszahlung".
-  - Extract "Zu zahlen" or "Bitte Zahlen" as a remainingAmountToPay.
-  - do not get confused with the paidAmount and remainingAmountToPay. Few invoices may not have a paidAmount or remainingAmountToPay. In such cases, pay attention to the sentence field alignment.
   - "Total Kosten excl. MwSt." is not the vatApplicableAmount
 <INSTRUCTIONS>