data-science-document-ai 1.43.1__py3-none-any.whl → 1.43.3__py3-none-any.whl

{data_science_document_ai-1.43.1.dist-info → data_science_document_ai-1.43.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-science-document-ai
-Version: 1.43.1
+Version: 1.43.3
 Summary: "Document AI repo for data science"
 Author: Naomi Nguyen
 Author-email: naomi.nguyen@forto.com

{data_science_document_ai-1.43.1.dist-info → data_science_document_ai-1.43.3.dist-info}/RECORD

@@ -1,13 +1,13 @@
-src/constants.py,sha256=TF_UblovdXZnKIb1lnyJwUqQncJCbzBVihoelI6foSU,3579
+src/constants.py,sha256=rpYIecVLIBLh98YrJ8e5gdvM0bqrXJZWIKgFkUSn69g,3513
 src/constants_sandbox.py,sha256=Iu6HdjCoNSmOX0AwoL9qUQkhq_ZnIN5U9e-Q2UfNuGc,547
 src/docai.py,sha256=dHuR0ehVjUi1CnoNvdp_yxJtpU_HFXqAZ61ywdz7BEo,5655
-src/docai_processor_config.yaml,sha256=qOMmCIORpLQ_D-ytvejXxFvER0e0uGYuzPVdZBGv4Pc,2105
+src/docai_processor_config.yaml,sha256=81NUGs-u8UFJm6mc0ZOeeNQlhe9h0f35GhjTcwErvTA,1717
 src/excel_processing.py,sha256=gzP7QFCp4-n0FTevhWmXm-2UoDF0w0y5v39gsby0IV8,3135
 src/io.py,sha256=tOJpMyI-mP1AaXKG4UFudH47MHWzjWBgVahFJUcjGfs,4749
 src/llm.py,sha256=OE4IEIqcM-hYK9U7e0x1rAfcqdpeo4iXPHBp64L5Qz0,8199
 src/log_setup.py,sha256=RhHnpXqcl-ii4EJzRt47CF2R-Q3YPF68tepg_Kg7tkw,2895
-src/pdf_processing.py,sha256=M0RUi20j481FcvMJ3xifavMNsPHqMFpbRo9w8hMpmL8,16970
-src/postprocessing/common.py,sha256=wvlYI1S75r0q5xp9Yll89nOVWtwDd7hV4Sf0MIButA0,22150
+src/pdf_processing.py,sha256=sr41bSMbH-WwWQ9uF5WKnsrkXJzCDirziv3TaS8hoPQ,17164
+src/postprocessing/common.py,sha256=b0VpxM-levZi_8H5a9gDNjx-67W6F7xRFUDa7CJJGgQ,22214
 src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
 src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
 src/postprocessing/postprocess_partner_invoice.py,sha256=koGR7dN37FqJcepdzkrzNBHuBBUuCp_3CrteScASqyE,10590
@@ -27,6 +27,7 @@ src/prompts/library/bookingConfirmation/yangming/placeholders.json,sha256=IpM9nm
 src/prompts/library/bookingConfirmation/yangming/prompt.txt,sha256=fYKfusDajDFw0v54-nv2iAqUSp2yCeOzc6G7AFe-h2w,3226
 src/prompts/library/bundeskasse/other/placeholders.json,sha256=1ll8AI58F2zRDSwQq_r0gxQdxlQB521l5CuiJ-8G6us,4068
 src/prompts/library/bundeskasse/other/prompt.txt,sha256=WV4D3ellIcB2cVmsZXCpbbHOShYY8VN_iZrYOuyoqzw,2937
+src/prompts/library/commercialInvoice/other/placeholders.json,sha256=zUK2mg9MnHiEQRYF6VgTiUiq68WGy5f7_4qL63CWyR0,4700
 src/prompts/library/commercialInvoice/other/prompt.txt,sha256=6sowYMzrKvgmTDpDnAzkeG4OqA44e6-8aUKWRKNziBY,2699
 src/prompts/library/customsAssessment/other/prompt.txt,sha256=XSqWa3k9LM7dTiJtX8AKTp_0x5Z0pCNRKNUWaywwBlY,2191
 src/prompts/library/customsInvoice/other/placeholders.json,sha256=BnWYtl4sPooTHb_EHRIlrPawBrfHI8_QVas8zytbqyY,12172
@@ -40,6 +41,7 @@ src/prompts/library/draftMbl/other/prompt.txt,sha256=pj-kgPV51upLhDppSKfhc2s5ylg
 src/prompts/library/finalMbL/hapag-lloyd/prompt.txt,sha256=RhxEJ4eWikAQiE40cuPsssnzizge6AJYFTSJLGUmz_U,2326
 src/prompts/library/finalMbL/maersk/prompt.txt,sha256=4neW6buJirgoS84iDsy9ZcfQTaMeOFt92Emba01mzJA,2192
 src/prompts/library/finalMbL/other/prompt.txt,sha256=pj-kgPV51upLhDppSKfhc2s5ylgr06l4IxrkFYjE9uM,2241
+src/prompts/library/packingList/other/placeholders.json,sha256=cGUUvEFoi4Lm0BAiyD29KbNFbUgzO1s7eit_qK3F0ig,4478
 src/prompts/library/packingList/other/prompt.txt,sha256=6Q9d0KBG6YWmNtzFivvmtQmitaUE2jytfwwc5YwsUgQ,2872
 src/prompts/library/partnerInvoice/other/placeholders.json,sha256=NX6ADT4gxLpP90uoNCYDbmfBvROxxVWRKK0lRFy1n9s,10897
 src/prompts/library/partnerInvoice/other/prompt.txt,sha256=fGUtMYWvhedmSiv9xShRv0cHXmEws1D9pQmZP1E2gl0,7806
@@ -52,6 +54,6 @@ src/prompts/prompt_library.py,sha256=jPxybNPPGH7mzonqtAOqmw5WcT-RtbGP0pvMqqP22hg
 src/setup.py,sha256=M-p5c8M9ejKcSZ9N86VtmtPc4TYLxe1_4_dxf6jpfVc,7262
 src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
 src/utils.py,sha256=cTF2A12jugKjXxGlNXEZQtfgcsIoaTtaU7zhVOOvXXA,16634
-data_science_document_ai-1.43.1.dist-info/METADATA,sha256=qHuRdJZSrKPDOQvynxf69ssK3jjM-pnBmGyheEPt0Xw,2152
-data_science_document_ai-1.43.1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-data_science_document_ai-1.43.1.dist-info/RECORD,,
+data_science_document_ai-1.43.3.dist-info/METADATA,sha256=6WQCGhLAMXOWEdTyPax7z0teZpgl-poGtI3o3X_P164,2152
+data_science_document_ai-1.43.3.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+data_science_document_ai-1.43.3.dist-info/RECORD,,

src/constants.py

@@ -50,8 +50,6 @@ project_parameters = {
     "model_selector": {
         "stable": {
             "bookingConfirmation": 1,
-            "packingList": 0,
-            "commercialInvoice": 0,
             "finalMbL": 0,
             "draftMbl": 0,
             "arrivalNotice": 0,

src/docai_processor_config.yaml

@@ -13,20 +13,6 @@ model_config:
           author: "igor.tonko@forto.com"
           created_date: ""
 
-    packingList:
-      - id: "d967005bd9d45aeb"
-        details:
-          display_name: "doc_cap_packingList"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
-    commercialInvoice:
-      - id: "7d37236207f75758"
-        details:
-          display_name: "doc_cap_commercialInvoice"
-          author: "kumar.rajendrababu@forto.com"
-          created_date: ""
-
     finalMbL:
       - id: "1eda2f22d64b1b89"
         details:

src/pdf_processing.py

@@ -36,7 +36,6 @@ from src.utils import (
     get_pdf_page_count,
     get_processor_name,
     run_background_tasks,
-    transform_schema_strings,
     validate_based_on_schema,
 )
 
@@ -207,7 +206,14 @@ async def process_file_w_llm(params, file_content, input_doc_type, llm_client):
     # get the schema placeholder from the Doc AI and generate the response structure
     response_schema = (
         prompt_library.library[input_doc_type]["other"]["placeholders"]
-        if input_doc_type in ["partnerInvoice", "customsInvoice", "bundeskasse"]
+        if input_doc_type
+        in [
+            "partnerInvoice",
+            "customsInvoice",
+            "bundeskasse",
+            "commercialInvoice",
+            "packingList",
+        ]
         else generate_schema_structure(params, input_doc_type)
     )
 
@@ -338,10 +344,10 @@ async def extract_data_by_doctype(
     if_use_llm,
     isBetaTest=False,
 ):
-    # Select LLM client (Using 2.5 Flash model for Bundeskasse)
+    # Select LLM client (Using 2.5 Pro model only for PI and customsInvoice)
     llm_client = (
         params["LlmClient_Flash"]
-        if input_doc_type == "bundeskasse"
+        if input_doc_type not in ["customsInvoice", "partnerInvoice"]
         else params["LlmClient"]
     )
 
@@ -393,6 +399,7 @@ async def data_extraction_manual_flow(
     meta,
     processor_client,
     schema_client,
+    use_default_logging=False,
 ):
     """
     Process a PDF file and extract data from it.
@@ -480,7 +487,9 @@ async def data_extraction_manual_flow(
     logger.info(f"Time taken to process the document: {round(elapsed_time, 4)} seconds")
 
     # Schedule background tasks without using FastAPI's BackgroundTasks
-    if os.getenv("CLUSTER") != "ode":  # skip data export to bigquery in ODE environment
+    if (
+        os.getenv("CLUSTER") != "ode"
+    ) & use_default_logging:  # skip data export to bigquery in ODE environment
         asyncio.create_task(
             run_background_tasks(
                 params,

src/postprocessing/common.py

@@ -429,7 +429,10 @@ async def format_label(entity_k, entity_value, document_type_code, params):
         except ValueError as e:
             logger.info(f"ParserError: {e}")
 
-    elif entity_key in ["invoicenumber", "creditnoteinvoicenumber"]:
+    elif (
+        entity_key in ["invoicenumber", "creditnoteinvoicenumber"]
+        and document_type_code == "bundeskasse"
+    ):
         formatted_value = clean_invoice_number(entity_value)
 
     elif entity_key in ("shipmentid", "partnerreference"):

src/prompts/library/commercialInvoice/other/placeholders.json

@@ -0,0 +1,125 @@
+{
+  "type": "OBJECT",
+  "properties": {
+    "consignee": {
+      "type": "string",
+      "nullable": true,
+      "description": "The receiver or buyer of the goods."
+    },
+    "currency": {
+      "type": "string",
+      "nullable": true,
+      "description": "The currency of the totalAmount."
+    },
+    "grossWeight": {
+      "type": "string",
+      "nullable": true,
+      "description": "The total gross weight of all the goods. Usually mentioned as G.W or GW or Gross Weight, etc.."
+    },
+    "incoterm": {
+      "type": "string",
+      "nullable": true,
+      "description": "An Incoterm is a 3 letter standardized trade term defining the responsibilities of buyers and sellers in international shipping and logistics. For example, FOB, CFR, DAP, CIF, etc..."
+    },
+    "invoiceDate": {
+      "type": "string",
+      "nullable": true,
+      "description": "A date that the invoice was created or issued."
+    },
+    "invoiceNumber": {
+      "type": "string",
+      "nullable": true,
+      "description": "The invoice number of the commercial invoice document."
+    },
+    "measurement": {
+      "type": "string",
+      "nullable": true,
+      "description": "The volume of the goods. Usually, it is measured in \"Cubic Meter (cbm)\" or dimensions. But volume in \"cbm\" is preferred."
+    },
+    "netWeight": {
+      "type": "string",
+      "nullable": true,
+      "description": "The total net weight of all the goods. Usually, mentioned as N.W or NW or Net Weight, etc.."
+    },
+    "shipper": {
+      "type": "string",
+      "nullable": true,
+      "description": "The seller or shipper of the goods."
+    },
+    "totalAmount": {
+      "type": "string",
+      "nullable": true,
+      "description": "The total amount of all the goods mentioned in the invoice."
+    },
+    "skus": {
+      "type": "ARRAY",
+      "items": {
+        "type": "OBJECT",
+        "properties": {
+          "amount": {
+            "type": "string",
+            "nullable": true,
+            "description": "Amount of the goods."
+          },
+          "containerNumber": {
+            "type": "string",
+            "nullable": true,
+            "description": "Container Number consists of 4 capital letters followed by 7 digits. Example: TEMU7972458. Usually mentioned as Container Number, CONTAINER NO. Containers, or Container / Truck No"
+          },
+          "currency": {
+            "type": "string",
+            "nullable": true,
+            "description": "The currency of the Amount. Usually mentioned in USD, EURO, CNY, $, or any other currency units and symbols."
+          },
+          "goodsDescription": {
+            "type": "string",
+            "nullable": true,
+            "description": "Description of the goods."
+          },
+          "grossWeight": {
+            "type": "string",
+            "nullable": true,
+            "description": "The gross weight of an individual product/goods. Usually, mentioned as G.W or GW or Gross Weight, etc.."
+          },
+          "hsCode": {
+            "type": "string",
+            "nullable": true,
+            "description": "The harmonized system code of a goods."
+          },
+          "materialNumber": {
+            "type": "string",
+            "nullable": true,
+            "description": "Material number of the product or goods."
+          },
+          "netWeight": {
+            "type": "string",
+            "nullable": true,
+            "description": "The net weight of an individual product/goods. Usually, mentioned as N.W or NW or Net Weight, etc.."
+          },
+          "packagingQuantity": {
+                "type": "string",
+                "nullable": true,
+                "description": "The quantity of the goods. Usually, the quantity is in pallets, PLT, cartons, CTNS, pieces, PCS, packages, boxes, etc. Please prioritize the packaging types based on their size, as follows: Pallets (PLT) >> Cartons (CTNS) >> Pieces (PCS). Extract the Larger packaging types that will have a lower count."
+            },
+          "packageType": {
+            "type": "string",
+            "nullable": true,
+            "description": "The packaging type is the unit of packagingQuantity. Example; pallets, PLT, cartons, CTNS, pieces, PCS, packages, etc. Sometimes, the packaging type is available in the column name of the packagingQuantity."
+          },
+          "poNumber": {
+            "type": "string",
+            "nullable": true,
+            "description": "Purchase order of the goods."
+          },
+          "skuNumber": {
+            "type": "string",
+            "nullable": true,
+            "description": "SKU number of the goods."
+          }
+        },
+        "required": []
+      }
+    }
+  },
+  "required": []
+}

src/prompts/library/packingList/other/placeholders.json

@@ -0,0 +1,98 @@
+{
+  "type": "OBJECT",
+  "properties": {
+    "buyer": {
+      "type": "string",
+      "nullable": true,
+      "description": "The receiver or buyer of the goods."},
+    "grossWeight": {
+      "type": "string",
+      "nullable": true,
+      "description": "The total gross weight of all the goods. Usually mentioned as G.W or GW or Gross Weight, etc.."},
+    "invoiceNumber": {
+      "type": "string",
+      "nullable": true,
+      "description": "The invoice number"},
+    "netWeight": {
+      "type": "string",
+      "nullable": true,
+      "description": "The total net weight of all the goods. Usually mentioned as N.W or NW or Net Weight, etc.."},
+    "seller": {
+      "type": "string",
+      "nullable": true,
+      "description": "The seller or shipper of the goods."},
+    "skuData": {
+      "type": "ARRAY",
+      "items": {
+        "type": "OBJECT",
+        "properties": {
+          "containerNumber": {
+            "type": "string",
+            "nullable": true,
+            "description": "Container Number consists of 4 capital letters followed by 7 digits. Example: TEMU7972458. Usually mentioned as Container Number, CONTAINER NO. or Containers"},
+          "grossWeight": {
+            "type": "string",
+            "nullable": true,
+            "description": "The gross weight of the goods. Usually mentioned as G.W or GW or Gross Weight, etc.."},
+          "hsCode": {
+            "type": "string",
+            "nullable": true,
+            "description": "The harmonized system code of a goods."},
+          "measurements": {
+            "type": "string",
+            "nullable": true,
+            "description": "The volume of the goods. Usually, it is measured in 'Cubic Meter (cbm)' or dimensions. But volume in 'cbm' is preferred."},
+          "netWeight": {
+            "type": "string",
+            "nullable": true,
+            "description": "The net weight of the goods. Usually mentioned as N.W or NW or Net Weight, etc.."},
+          "packagingType": {
+            "type": "string",
+            "nullable": true,
+            "description": "The packaging type is the unit of quantityShipped. Example; pallets, PLT, cartons, CTNS, pieces, PCS, packages, etc. Sometimes, the packaging type is available in the column name of the quantityShipped."},
+          "poNumber": {
+            "type": "string",
+            "nullable": true,
+            "description": "Purchase order of the goods."},
+          "poPosition": {
+            "type": "string",
+            "nullable": true,
+            "description": "PO position refers to the specific item or line associated with a Purchase Order (PO). It represents the position or line number in the PO that corresponds to the items being shipped."},
+          "quantityShipped": {
+            "type": "string",
+            "nullable": true,
+            "description": "The quantity of the goods. Usually quantity is in pallets, PLT, cartons, CTNS, pieces, PCS, packages, boxes, etc. Please prioritize the packaging types based on their size, as follows: Pallets (PLT) >> Cartons (CTNS) >> Pieces (PCS). Extract the Larger packaging types that will have a lower count."},
+          "sealNumber": {
+            "type": "string",
+            "nullable": true,
+            "description": "A unique number associated with the container number"},
+          "skuDescription": {
+            "type": "string",
+            "nullable": true,
+            "description": "Description of the goods."},
+          "skuNumbers": {
+            "type": "string",
+            "nullable": true,
+            "description": "SKU number of the goods."}
+        },
+        "required": [
+          "skuNumbers",
+          "quantityShipped",
+          "skuDescription",
+          "grossWeight",
+          "netWeight",
+          "packagingType"
+        ]
+      }
+    },
+    "totalPackagingType": {
+      "type": "string",
+      "nullable": true,
+      "description": "The packaging type of all the goods associated with the totalQuantityShipped. It is the unit of totalQuantityShipped. Usually pallets, PLT, cartons, CTNS, pieces, PCS, packages, etc."},
+    "totalQuantityShipped": {
+      "type": "string",
+      "nullable": true,
+      "description": "The total quantity of the goods. Usually quantity is in pallets, cartons, pieces, packages, boxes, etc. Please prioritize the packaging types based on their size, as follows: Pallets >> Cartons >> Pieces. Larger packaging types will have a lower count."}
+  },
+  "required": []
+}