data-science-document-ai 1.48.0__py3-none-any.whl → 1.50.0__py3-none-any.whl

{data_science_document_ai-1.48.0.dist-info → data_science_document_ai-1.50.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.48.0
+Version: 1.50.0
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.48.0.dist-info → data_science_document_ai-1.50.0.dist-info}/RECORD

@@ -10,7 +10,7 @@ src/pdf_processing.py,sha256=oKVPnIu_keiN17XLOGImeyJ4iMT2H51x4OD1Tp9yw1s,19992
 src/postprocessing/common.py,sha256=dagAg0hZGuZc03bXdfOolxekewMEVUfz917IGCiAtWI,26118
 src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
 src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
-src/postprocessing/postprocess_partner_invoice.py,sha256=p-oC0kUJDI9wsOfIDSrvqeEji3TYiCvP0WJ4qfGVvZ4,12538
+src/postprocessing/postprocess_partner_invoice.py,sha256=Fv4Y6Lc8e6aFFcwX0kLOal2y4TrR-XfAzjtuQnBwo0o,12815
 src/prompts/library/arrivalNotice/other/placeholders.json,sha256=1vzly1amgyKt3jr2JJQbb24kNZsnI289iduvoUo5dJU,3061
 src/prompts/library/arrivalNotice/other/prompt.txt,sha256=QNuU-BvMA8VbdupVNapad4O3WmCotH5cKNxImRMbKDk,2906
 src/prompts/library/bookingConfirmation/evergreen/placeholders.json,sha256=IpM9nmSPdyroliZfXB1-NDCjiHZX_Ff5BH7-scNhGqE,1406
@@ -27,8 +27,8 @@ src/prompts/library/bookingConfirmation/other/placeholders.json,sha256=IpM9nmSPd
 src/prompts/library/bookingConfirmation/other/prompt.txt,sha256=kUK7NgVNDYFMnqOcIblCwWSw2SC0YQEtHsYrspiVUMo,3379
 src/prompts/library/bookingConfirmation/yangming/placeholders.json,sha256=IpM9nmSPdyroliZfXB1-NDCjiHZX_Ff5BH7-scNhGqE,1406
 src/prompts/library/bookingConfirmation/yangming/prompt.txt,sha256=fYKfusDajDFw0v54-nv2iAqUSp2yCeOzc6G7AFe-h2w,3226
-src/prompts/library/bundeskasse/other/placeholders.json,sha256=1ll8AI58F2zRDSwQq_r0gxQdxlQB521l5CuiJ-8G6us,4068
-src/prompts/library/bundeskasse/other/prompt.txt,sha256=MBv4MIMASMstkzDS7H0q_pNJbPQeadP1vcmhCRrpjQ4,2906
+src/prompts/library/bundeskasse/other/placeholders.json,sha256=eSIVplLYCseVN17tygyV5TyheJtOTeybfJzFfKyPuUE,4275
+src/prompts/library/bundeskasse/other/prompt.txt,sha256=55b-umkE1648V-UoT-3UJjDEzs53TGQNZaOwVCjQPyc,3089
 src/prompts/library/commercialInvoice/other/placeholders.json,sha256=zUK2mg9MnHiEQRYF6VgTiUiq68WGy5f7_4qL63CWyR0,4700
 src/prompts/library/commercialInvoice/other/prompt.txt,sha256=CJapcVrmcvynJUanETDklkzU-0N9hHdhq5wL4MK7OIY,2683
 src/prompts/library/customsAssessment/other/placeholders.json,sha256=scIV--C9HNWAQbU9zEz3GT_FoAvJqbfuY85YUtt7t-Q,3850
@@ -52,9 +52,9 @@ src/prompts/library/preprocessing/carrier/prompt.txt,sha256=NLvRZQCZ6aWC1yTr7Q93
 src/prompts/library/shippingInstruction/other/placeholders.json,sha256=eK4AeMfORkGMWVYcqH7NjB56Zb4swHTvcQD5UQbTryg,6374
 src/prompts/library/shippingInstruction/other/prompt.txt,sha256=CbrqlKMtB-sVY-8E460KP1KNmz169YVPMrH3-uEldPg,2135
 src/prompts/prompt_library.py,sha256=VJWHeXN-s501C2GiidIIvQQuZdU6T1R27hE2dKBiI40,2555
-src/setup.py,sha256=M-p5c8M9ejKcSZ9N86VtmtPc4TYLxe1_4_dxf6jpfVc,7262
+src/setup.py,sha256=EHfAl3Pvb082dl_s6Tk93IjtE3vBmrW_fp2GW4955HQ,6952
 src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
 src/utils.py,sha256=Ow5_Jals88o8mbZ1BoHfZpHZoCfig_UQb5aalH-mpWE,17278
-data_science_document_ai-1.48.0.dist-info/METADATA,sha256=TEE-nCF6lxV4DdmQIhbij2ZXGrVwM0wBSLJxJPdUJLE,2152
-data_science_document_ai-1.48.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-data_science_document_ai-1.48.0.dist-info/RECORD,,
+data_science_document_ai-1.50.0.dist-info/METADATA,sha256=_e_1gpRkzKH5RvZ7yKUfbMNtvWH5Hc-0xS1Gyeq4FUQ,2152
+data_science_document_ai-1.50.0.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+data_science_document_ai-1.50.0.dist-info/RECORD,,

src/postprocessing/postprocess_partner_invoice.py

@@ -103,9 +103,18 @@ def post_process_bundeskasse(aggregated_data):
             )
 
         # Check if the deferredDutyPayer is forto
-        deferredDutyPayer = line_item.get("deferredDutyPayer", {})
-        lower = deferredDutyPayer.get("documentValue", "").lower()
-        if any(key in lower for key in ["de789147263644738", "forto"]):
+        KEYWORDS = {"de789147263644738", "forto", "009812"}
+
+        def is_forto_recipient(line_item: dict) -> bool:
+            values_to_check = [
+                line_item.get("deferredDutyPayer", {}).get("documentValue", ""),
+                line_item.get("vatId", {}).get("documentValue", ""),
+            ]
+
+            combined = " ".join(values_to_check).lower()
+            return any(keyword in combined for keyword in KEYWORDS)
+
+        if is_forto_recipient(line_item):
             is_recipient_forto = True
 
     update_recipient_and_vendor(aggregated_data, is_recipient_forto)

src/prompts/library/bundeskasse/other/placeholders.json

@@ -4,12 +4,12 @@
       "currencyCode": {
         "type": "STRING",
         "nullable": true,
-        "description": "The currency in which the invoice is issued."
+        "description": "The currency in which the invoice is issued. Extract the currency associated with the grand total amount. It is majorly mentioned as EUR, Euro or €."
       },
       "grandTotal": {
         "type": "STRING",
         "nullable": true,
-        "description": "The overall total amount of the invoice."
+        "description": "The overall total amount of the invoice. It can be found with the key words Gesamtabgabenbetrag, Gesamtbetrag, or Zu erstattender Abgabenbetrag"
       },
       "issueDate": {
         "type": "STRING",
@@ -54,7 +54,7 @@
                   "deferredDutyPayer": {
                     "type": "STRING",
                     "nullable": true,
-                    "description": "It can be identified under \"Aufschubenhmer\" for each line item"
+                    "description": "It can be identified under 'Aufschubenhmer' for each line item"
                   },
                   "name": {
                     "type": "STRING",

src/prompts/library/bundeskasse/other/prompt.txt

@@ -31,7 +31,9 @@ Your role is to accurately extract specific entities from these Customs invoices
     - Credit Note Invoice Number is a unique identifier for the credit note, it starts with "ATS" only (e.g., ATS.....).
     - NIZZA is not a credit note invoice number.
 
-- grandTotal in the credit notes can be found under "Zu erstattender Abgabenbetrag".
+- grandTotal:
+    - It can be found with the key words Gesamtabgabenbetrag, Gesamtbetragin. In credit notes, it can be found under "Zu erstattender Abgabenbetrag".
+    - grandTotal value is always or mostly mentioned in EUR currency as it is issued by German Customs.
 
 - serviceDate can also be referred to as "Zollanmeldung" or "Eingangdatum" in the invoice.
 - issueDate can also be referred to as "Einfuhrabgabenbescheid" in the invoice. issueDate and serviceDate can be same in some cases.

src/setup.py

@@ -1,11 +1,8 @@
 """Contains project setup parameters and initialization functions."""
 import json
-
-# import streamlit as st
 import os
 import random
 import time
-from pathlib import Path
 
 import toml
 import vertexai
@@ -18,7 +15,7 @@ from src.constants import project_parameters
 from src.constants_sandbox import project_parameters_sandbox
 
 # Parent repos are imported without .
-from src.io import download_dir_from_bucket, get_bq_client, get_storage_client, logger
+from src.io import get_bq_client, get_storage_client, logger
 from src.llm import LlmClient
 
 
@@ -133,15 +130,7 @@ def setup_params(args=None):
         assert params.keys() & yaml_content.keys() == set()
         params.update(yaml_content)
 
-    # Get models meta data from cloud
-    client = get_storage_client(params)
-    bucket = client.bucket(params["doc_ai_bucket_name"])
-    downloaded_meta = download_dir_from_bucket(
-        bucket, params["g_model_data_folder"], Path(params["local_model_data_folder"])
-    )
-    if not downloaded_meta:
-        logger.info(f"Could not load models metadata from cloud.")
-
+    # Set up LLM clients
     params["LlmClient"] = LlmClient(
         openai_key=os.getenv("OPENAI_KEY"), parameters=params["gemini_params"]
     )
@@ -149,7 +138,8 @@ def setup_params(args=None):
         openai_key=os.getenv("OPENAI_KEY"), parameters=params["gemini_flash_params"]
     )
 
-    params["lookup_data"] = setup_lookup_data(params, bucket)
+    # Load lookup data from GCS bucket
+    setup_lookup_data(params)
 
     return params
 
@@ -182,10 +172,13 @@ def setup_vertexai(params):
     )
 
 
-def setup_lookup_data(params, bucket):
+def setup_lookup_data(params):
     """
     Loads JSON mapping data from given GCP Bucket.
     """
+    client = get_storage_client(params)
+    bucket = client.bucket(params["doc_ai_bucket_name"])
+
     data = dict()
 
     input_path_item_code = (
@@ -208,4 +201,4 @@ def setup_lookup_data(params, bucket):
         input_path_reverse_charge
     )
 
-    return data
+    params["lookup_data"] = data