data-science-document-ai 1.40.3__py3-none-any.whl → 1.51.0__py3-none-any.whl

src/setup.py

@@ -1,11 +1,8 @@
 """Contains project setup parameters and initialization functions."""
 import json
-
-# import streamlit as st
 import os
 import random
 import time
-from pathlib import Path
 
 import toml
 import vertexai
@@ -18,7 +15,7 @@ from src.constants import project_parameters
 from src.constants_sandbox import project_parameters_sandbox
 
 # Parent repos are imported without .
-from src.io import download_dir_from_bucket, get_storage_client, logger
+from src.io import get_bq_client, get_storage_client, logger
 from src.llm import LlmClient
 
 
@@ -118,6 +115,10 @@ def setup_params(args=None):
 
     params = setup_docai_client_and_path(params)
 
+    # Set up BigQuery client for logging
+    bq_client, _ = get_bq_client(params)
+    params["bq_client"] = bq_client
+
     # Set up Vertex AI for text embeddings
     setup_vertexai(params)
 
@@ -129,15 +130,7 @@ def setup_params(args=None):
         assert params.keys() & yaml_content.keys() == set()
         params.update(yaml_content)
 
-    # Get models meta data from cloud
-    client = get_storage_client(params)
-    bucket = client.bucket(params["doc_ai_bucket_name"])
-    downloaded_meta = download_dir_from_bucket(
-        bucket, params["g_model_data_folder"], Path(params["local_model_data_folder"])
-    )
-    if not downloaded_meta:
-        logger.info(f"Could not load models metadata from cloud.")
-
+    # Set up LLM clients
     params["LlmClient"] = LlmClient(
         openai_key=os.getenv("OPENAI_KEY"), parameters=params["gemini_params"]
     )
@@ -145,7 +138,8 @@ def setup_params(args=None):
         openai_key=os.getenv("OPENAI_KEY"), parameters=params["gemini_flash_params"]
     )
 
-    params["lookup_data"] = setup_lookup_data(params, bucket)
+    # Load lookup data from GCS bucket
+    setup_lookup_data(params)
 
     return params
 
@@ -178,10 +172,13 @@ def setup_vertexai(params):
     )
 
 
-def setup_lookup_data(params, bucket):
+def setup_lookup_data(params):
     """
     Loads JSON mapping data from given GCP Bucket.
     """
+    client = get_storage_client(params)
+    bucket = client.bucket(params["doc_ai_bucket_name"])
+
     data = dict()
 
     input_path_item_code = (
@@ -204,4 +201,4 @@ def setup_lookup_data(params, bucket):
         input_path_reverse_charge
     )
 
-    return data
+    params["lookup_data"] = data

src/utils.py

@@ -6,38 +6,29 @@ import json
 import os
 import pickle
 from datetime import datetime
-from typing import Literal
+from typing import Any, Dict, List, Literal, Optional
 
+import httpx
+import numpy as np
 import openpyxl
 import pandas as pd
-import requests
 from google.cloud import documentai_v1beta3 as docu_ai_beta
-from PyPDF2 import PdfReader, PdfWriter
+from pypdf import PdfReader, PdfWriter
 
-from src.io import get_bq_client, get_storage_client, logger
+from src.io import bq_logs, get_storage_client, logger
 
 
-def bq_logs(data_to_insert, params):
-    """Insert logs into Google BigQuery.
+def get_pdf_page_count(pdf_bytes):
+    """Get the number of pages in a PDF document efficiently.
 
     Args:
-        data_to_insert (list): The data to insert into BigQuery.
-        params (dict): The parameters dictionary.
+        pdf_bytes (bytes): The PDF content as bytes.
+
+    Returns:
+        int: The number of pages in the PDF.
     """
-    # Get the BigQuery client
-    bq_client, config = get_bq_client(params)
-    # Get the table string
-    table_string = f"{params['g_ai_project_name']}.{params['g_ai_gbq_db_schema']}.{params['g_ai_gbq_db_table_out']}"
-
-    logger.info(f"Log table: {table_string}")
-    # Insert the rows into the table
-    insert_logs = bq_client.insert_rows_json(table_string, data_to_insert)
-
-    # Check if there were any errors inserting the rows
-    if not insert_logs:
-        logger.info("New rows have been added.")
-    else:
-        logger.info("Errors occurred while inserting rows: ", insert_logs)
+    reader = PdfReader(io.BytesIO(pdf_bytes))
+    return len(reader.pages)
 
 
 async def get_data_set_schema_from_docai(
@@ -139,7 +130,12 @@ def store_json_in_gcs(
         bucket = storage_client.bucket(params.get("doc_ai_bucket_name"))
         full_object_name = folder_path + document_id
         blob = bucket.blob(full_object_name)
-        blob.upload_from_string(json_data, content_type="application/json")
+
+        # Convert dict to JSON string if needed
+        json_string = (
+            json.dumps(json_data) if isinstance(json_data, dict) else json_data
+        )
+        blob.upload_from_string(json_string, content_type="application/json")
 
         logger.info(
             f"JSON object stored successfully in gs://{params.get('doc_ai_bucket_name')}/{full_object_name}"  # noqa
@@ -159,6 +155,7 @@ async def run_background_tasks(
     processor_version,
     mime_type,
     elapsed_time=None,
+    page_count=None,
 ):
     """
     Run background tasks asynchronously.
@@ -172,6 +169,7 @@ async def run_background_tasks(
         processor_version: The processor version used to extract the data.
         mime_type: The MIME type of the document.
         elapsed_time: The time taken to process the document.
+        page_count (int, optional): The number of pages in the document.
 
     Returns:
         None
@@ -180,13 +178,8 @@ async def run_background_tasks(
 
     await loop.run_in_executor(None, store_json_in_gcs, params, doc_id, store_data)
 
-    # Keep the page count as 1 for Excel files.
-    page_count = 1
-    # calculate the number of pages processed for PDFs
-    try:
-        if mime_type == "application/pdf":
-            page_count = len(json.loads(store_data.encode("utf-8"))["pages"])
-    except AttributeError:
+    # Use the passed page_count or default to 0 if not provided
+    if page_count is None:
         page_count = 0
 
     # Log the request in BigQuery
@@ -367,9 +360,9 @@ def extract_top_pages(pdf_bytes, num_pages=4):
     return output.getvalue()
 
 
-def get_tms_mappings(
-    input_list: list[str], embedding_type: str, llm_ports: list[str] = None
-):
+async def get_tms_mappings(
+    input_list: List[str], embedding_type: str, llm_ports: Optional[List[str]] = None
+) -> Dict[str, Any]:
     """Get TMS mappings for the given values.
 
     Args:
@@ -379,36 +372,155 @@ def get_tms_mappings(
         llm_ports (list[str], optional): List of LLM ports to use. Defaults to None.
 
     Returns:
-        dict: A dictionary with the mapping results.
+        dict or string: A dictionary or a string with the mapping results.
     """
-    # To test the API locally, port-forward the embedding service in the sandbox to 8080:80
-    # If you want to launch uvicorn from the tms-embedding repo, then use --port 8080 in the config file
     base_url = (
         "http://0.0.0.0:8080/"
         if os.getenv("CLUSTER") is None
         else "http://tms-mappings.api.svc.cluster.local./"
     )
 
+    # Ensure clean inputs
+    if not input_list:
+        return {}
+
     # Ensure input_list is a list
     if not isinstance(input_list, list):
         input_list = [input_list]
 
     # Always send a dict with named keys
     payload = {embedding_type: input_list}
+
     if llm_ports:
         payload["llm_ports"] = llm_ports if isinstance(llm_ports, list) else [llm_ports]
 
     # Make the POST request to the TMS mappings API
-    url = f"{base_url}/{embedding_type}"
-    response = requests.post(url=url, json=payload)
+    url = f"{base_url}{embedding_type}"
 
-    if response.status_code != 200:
-        logger.error(
-            f"Error from TMS mappings API: {response.status_code} - {response.text}"
-        )
+    # Use a timeout so the code doesn't hang forever
+    timeout = httpx.Timeout(60.0, connect=10.0)
 
-    formatted_values = (
-        response.json().get("response", {}).get("data", {}).get(input_list[0], None)
+    async with httpx.AsyncClient(timeout=timeout) as client:
+        try:
+            response = await client.post(url, json=payload)
+            response.raise_for_status()
+
+            # Structure expected: {"response": {"data": {"desc1": "code1", "desc2": "code2"}}}
+            return response.json().get("response", {}).get("data", {})
+
+        except httpx.HTTPStatusError as exc:
+            logger.error(
+                f"Error from TMS mappings API: {exc.response.status_code} - {exc.response.text}"
+            )
+            return {}
+
+
+async def batch_fetch_all_mappings(container_types, terminals, depots):
+    """Batch fetch all mappings for container types, terminals, and depots."""
+    # run batch calls concurrently
+    results = await asyncio.gather(
+        get_tms_mappings(list(container_types), "container_types"),
+        get_tms_mappings(list(terminals), "terminals"),
+        get_tms_mappings(list(depots), "depots"),
     )
 
-    return formatted_values
+    batch_container_map, batch_terminal_map, batch_depot_map = results
+
+    # Convert lists of tuples to dicts if necessary
+    return (
+        dict(batch_container_map or {}),
+        dict(batch_terminal_map or {}),
+        dict(batch_depot_map or {}),
+    )
+
+
+def transform_schema_strings(schema):
+    """
+    Recursively transforms a schema dictionary, replacing all "type": "STRING"
+    definitions with a new object containing "value" and "page_number" fields.
+    It preserves 'nullable' and 'description' fields by moving them to the
+    new 'value' property.
+
+    Args:
+        schema (dict): The input schema dictionary.
+
+    Returns:
+        dict: The transformed schema dictionary.
+    """
+    if not isinstance(schema, dict):
+        return schema
+
+    schema_type = schema.get("type")
+    if not schema_type:
+        return schema
+
+    # Base case: STRING → OBJECT (only if not already transformed)
+    if schema_type.upper() == "STRING":
+        return {
+            "type": "OBJECT",
+            "properties": {
+                "value": {
+                    "type": "STRING",
+                    "nullable": schema.get("nullable", False),
+                    "description": schema.get("description", ""),
+                },
+                "page_number": {
+                    "type": "STRING",
+                    "description": "Number of a page where the value was found in the document starting from 0.",
+                },
+            },
+            "required": [],
+        }
+
+    # Skip already transformed OBJECT (has both 'value' & 'page_number')
+    if (
+        schema_type.upper() == "OBJECT"
+        and "properties" in schema
+        and {"value", "page_number"}.issubset(schema["properties"].keys())
+    ):
+        return schema
+
+    # Recursive case for OBJECT
+    if schema_type.upper() == "OBJECT" and "properties" in schema:
+        new_schema = schema.copy()
+        new_schema["properties"] = {
+            k: transform_schema_strings(v) for k, v in schema["properties"].items()
+        }
+        return new_schema
+
+    # Recursive case for ARRAY
+    if schema_type.upper() == "ARRAY" and "items" in schema:
+        new_schema = schema.copy()
+        new_schema["items"] = transform_schema_strings(schema["items"])
+        return new_schema
+
+    return schema
+
+
+def estimate_page_count(sheet):
+    """Assuming a page is 10 columns x 50 rows."""
+    if hasattr(sheet, "shape"):
+        pg_cnt = sheet.shape[0] * sheet.shape[1]
+    elif hasattr(sheet, "max_row"):
+        pg_cnt = sheet.max_column * sheet.max_row
+    else:
+        return None
+    return np.ceil(pg_cnt / 500)
+
+
+def split_pdf_into_chunks(file_content: bytes, chunk_size: int = 1):
+    """Split PDF into smaller page chunks."""
+    pdf = PdfReader(io.BytesIO(file_content))
+    total_pages = len(pdf.pages)
+
+    # TODO: update the chunk_size based on doc length. However, it breaks the page number extraction logic.
+    for i in range(0, total_pages, chunk_size):
+        writer = PdfWriter()
+        for j in range(i, min(i + chunk_size, total_pages)):
+            writer.add_page(pdf.pages[j])
+
+        buffer = io.BytesIO()
+        writer.write(buffer)
+        buffer.seek(0)
+
+        yield buffer.getvalue()

data_science_document_ai-1.40.3.dist-info/RECORD

@@ -1,59 +0,0 @@
-src/constants.py,sha256=TF_UblovdXZnKIb1lnyJwUqQncJCbzBVihoelI6foSU,3579
-src/constants_sandbox.py,sha256=Iu6HdjCoNSmOX0AwoL9qUQkhq_ZnIN5U9e-Q2UfNuGc,547
-src/docai.py,sha256=AepGdF3ZuSGkujLpewX393FgOBMy-e4sEudiGKho5EA,5280
-src/docai_processor_config.yaml,sha256=qOMmCIORpLQ_D-ytvejXxFvER0e0uGYuzPVdZBGv4Pc,2105
-src/excel_processing.py,sha256=ZUlZ5zgWObmQfAWHoSrEEITKwr-xXxuOiPC3qDnGjtQ,2459
-src/io.py,sha256=IXz4wWqiHa9mnHNgtrC6X9M2lItYp9eu6rHCThUIh5c,3585
-src/llm.py,sha256=aEK3rL8XvY7CakvkOJQmcHpEKwZRd8PPrLrzHiO-GFk,7827
-src/log_setup.py,sha256=RhHnpXqcl-ii4EJzRt47CF2R-Q3YPF68tepg_Kg7tkw,2895
-src/pdf_processing.py,sha256=S_eTsgaDIIr3SCrEmaQZyc7TDJlRI0GCuP0P9EGF1Xc,15385
-src/postprocessing/common.py,sha256=ll7VMEJ_51OeczcV8Uw-aVrufV3kd3kNLCmss3kt0Do,19291
-src/postprocessing/postprocess_booking_confirmation.py,sha256=nK32eDiBNbauyQz0oCa9eraysku8aqzrcoRFoWVumDU,4827
-src/postprocessing/postprocess_commercial_invoice.py,sha256=3I8ijluTZcOs_sMnFZxfkAPle0UFQ239EMuvZfDZVPg,1028
-src/postprocessing/postprocess_partner_invoice.py,sha256=cM4te4qjOI_bXyrF8Zhb6X7eNf5aMKoRaPCFfqFv-98,11538
-src/prompts/library/bookingConfirmation/evergreen/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/evergreen/prompt.txt,sha256=qlBMFDHy-gwr2PVeuHrfMEg_8Ibdym243DnaCgINa7g,2614
-src/prompts/library/bookingConfirmation/hapag-lloyd/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/hapag-lloyd/prompt.txt,sha256=sg11U3lIhhS36BsimX7IOzR7Pez_9gScdNmJna2pPuw,3355
-src/prompts/library/bookingConfirmation/maersk/placeholders.json,sha256=PKWXySGAls6A8tujbSjokYp4ldc3c0DmSP2ITKYiUF8,1405
-src/prompts/library/bookingConfirmation/maersk/prompt.txt,sha256=-00tzWzXtQnXX3EPtaCBM39leCoLa4FB52_t7Z3eoQk,3148
-src/prompts/library/bookingConfirmation/msc/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/msc/prompt.txt,sha256=9wdbLofnp5s1acD19jCmQuw__HMcVq1yr4vIJNJlKVM,4630
-src/prompts/library/bookingConfirmation/oocl/placeholders.json,sha256=NnXjMiEsTCzTDWs2WY7BIMo2p4_98-DL3v1r7x-FL3A,1403
-src/prompts/library/bookingConfirmation/oocl/prompt.txt,sha256=aGowVvOgl4w6TjX5O2RtD4QOiWC1JnXiWgg0t0chThU,2060
-src/prompts/library/bookingConfirmation/other/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/other/prompt.txt,sha256=XOrq5Ns0nl8lDI9VvoOEbIMbOQdv8mcM8HqP8-eIjc4,3304
-src/prompts/library/bookingConfirmation/yangming/placeholders.json,sha256=Re2wBgZoaJ5yImUUAwZOZxFcKXHxi83TCZwTuqd2v2k,1405
-src/prompts/library/bookingConfirmation/yangming/prompt.txt,sha256=BSFy-6zDlAmOH2uZjsp-zZkR_Uy3RS5sGtdv9wysiSI,3151
-src/prompts/library/bundeskasse/other/placeholders.json,sha256=vc_m1yZP__ZMsdvnIvkgKmwF1q_ZMnKuWeIRNWour3w,4054
-src/prompts/library/bundeskasse/other/prompt.txt,sha256=GrrLH8lSuTcWmKUPUgBob0-bS_a61jTFSf09xxcE-nU,2890
-src/prompts/library/commercialInvoice/other/prompt.txt,sha256=fYUF7btc48Uqv4mJH5BgJdY4JVwj9I1xKX_HRBIrN7M,2590
-src/prompts/library/customsAssessment/other/placeholders.json,sha256=5nSGsMbpfKrpKoYImcTto_RlOvPCHyld2RlwU0Zbbqw,361
-src/prompts/library/customsAssessment/other/prompt.txt,sha256=wgJ8PYM0PKXiIKSljhFXEFBQ23GRs2E2DE9lVwHDvBU,2116
-src/prompts/library/customsInvoice/other/placeholders.json,sha256=vDji41DWA1ViKgMGwSBt0vu3wBCACrJhprKb8vG8DwY,12153
-src/prompts/library/customsInvoice/other/prompt.txt,sha256=P9t3Ga0KITSM1vmQWiX3UJ0pBPmndfJj_rZrvrf7utA,9593
-src/prompts/library/deliveryOrder/other/placeholders.json,sha256=6b_6OVsxT7bjFnV_v0OZkGEy-GN5K4AjL0ATzuoLdOU,1286
-src/prompts/library/deliveryOrder/other/prompt.txt,sha256=MVSS5AhkiWT17G9X4xk_AgKzYElagvWjLPCMr_ZhmOs,2393
-src/prompts/library/draftMbl/hapag-lloyd/prompt.txt,sha256=0k1xLW4zWaenCSNQJxXMXenIwI-eYmGgpxnAAcM3HOg,2251
-src/prompts/library/draftMbl/maersk/prompt.txt,sha256=GxaIYlksORvD2uAbodRx_9JFJXD4XbDaVFYtpN9uzxc,2050
-src/prompts/library/draftMbl/other/placeholders.json,sha256=wIN06_NWsESDyNEDfOLPi3F2Vq-XPa4O3U32A32s-_Q,1736
-src/prompts/library/draftMbl/other/prompt.txt,sha256=gqbPm1joXKDUss0wU6vMc-269sx-fYWh90gWuNKOBQc,2166
-src/prompts/library/finalMbL/hapag-lloyd/prompt.txt,sha256=0k1xLW4zWaenCSNQJxXMXenIwI-eYmGgpxnAAcM3HOg,2251
-src/prompts/library/finalMbL/maersk/prompt.txt,sha256=GxaIYlksORvD2uAbodRx_9JFJXD4XbDaVFYtpN9uzxc,2050
-src/prompts/library/finalMbL/other/placeholders.json,sha256=K_yJYhQo2DnZV_Rg6xXjo6sHkSGB-SMO4IQnY47V43w,1735
-src/prompts/library/finalMbL/other/prompt.txt,sha256=gqbPm1joXKDUss0wU6vMc-269sx-fYWh90gWuNKOBQc,2166
-src/prompts/library/packingList/other/prompt.txt,sha256=Qw16n7_48GGFYWz2vRepNowZCX1UPXKetEZ1UqFXPdY,2764
-src/prompts/library/partnerInvoice/other/placeholders.json,sha256=y_S_iAr-hHFzrPJgquZEtR66ETMvdWp1rtwCLrLPQUU,12139
-src/prompts/library/partnerInvoice/other/prompt.txt,sha256=hHzI5Vq3BABBWPMsv72nlahfYjgq08OSpflDDhV0NX8,9360
-src/prompts/library/postprocessing/port_code/placeholders.json,sha256=2TiXf3zSzrglOMPtDOlCntIa5RSvyZQAKG2-IgrCY5A,22
-src/prompts/library/postprocessing/port_code/prompt_port_code.txt,sha256=--1wunSqEr2ox958lEhjO-0JFBfOLzA3qfKYIzG_Iok,884
-src/prompts/library/preprocessing/carrier/placeholders.json,sha256=1UmrQNqBEsjLIpOO-a39Az6bQ_g1lxDGlwqZFU3IEt0,408
-src/prompts/library/preprocessing/carrier/prompt.txt,sha256=NLvRZQCZ6aWC1yTr7Q93jK5z7Vi_b4HBaiFYYnIsO-w,134
-src/prompts/library/shippingInstruction/other/prompt.txt,sha256=fyC24ig4FyRNnLuQM69s4ZVajsK-LHIl2dvaaEXr-6Q,1327
-src/prompts/prompt_library.py,sha256=VJWHeXN-s501C2GiidIIvQQuZdU6T1R27hE2dKBiI40,2555
-src/setup.py,sha256=kPSZosrICfaGZeDaajr40Ha7Ok4XK4fo_uq35Omiwr0,7128
-src/tms.py,sha256=UXbIo1QE--hIX6NZi5Qyp2R_CP338syrY9pCTPrfgnE,1741
-src/utils.py,sha256=-1Yq_5ExZlFQRUPRsQHiBD3TthNSiPVPp46Dvdb9Kf0,13830
-data_science_document_ai-1.40.3.dist-info/METADATA,sha256=ym7EzwlZAar6Qvx0GgwVQM44p30sw74-nrPl7Liyg_8,2153
-data_science_document_ai-1.40.3.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-data_science_document_ai-1.40.3.dist-info/RECORD,,

src/prompts/library/draftMbl/hapag-lloyd/prompt.txt

@@ -1,44 +0,0 @@
-You are a document entity extraction specialist. Given a document, the explained datapoint need to extract.
-
-blNumber: Bill of Lading number.
-voyage: The journey or route code taken by the vessel.
-portOfLoading: The port where cargo is loaded.
-portOfDischarge:  The port where cargo is unloaded.
-bookingNumber: A unique identifier for the booking.
-containers:
-    containerType: Type of the shipping container, usually related to it's size.
-    grossWeight: Total weight of the cargo, including the tare weight of the container.
-    measurements: Dimensions of the cargo (length, width, height) for freight calculations.
-    packageQuantity: package quantity.
-    packageType: Type of packaging used (e.g., cartons, pallets, barrels).
-    containerNumber: Unique ID for tracking the shipping container.
-    sealNumber: Number of the container's seal.
-vessel: The name of the vessel.
-
-
-Your task is to extract the text value of the following entities:
-
-Keywords for datapoints:
-- blNumber: Bill of Lading number, bill of landing no., swb-no., b/l no.
-- voyage: voyage, voy. no, voyage-no.
-- portOfLoading: port of loading, pol, from.]
-- portOfDischarge: port of discharge, pod, delivery, to
-- bookingNumber: Our reference, booking no., carrier reference
-- containers:
-    - containerType: x 40' container
-    - grossWeight: gross weight
-    - measurements: Dimensions of the cargo (length, width, height) for freight calculations
-    - packageQuantity: package quantity, number and kind of packages
-    - packageType: Type of packaging used (e.g., cartons, pallets, barrels), number and kind of packages, description of goods
-    - containerNumber: container number, cntr. nos., it is a combination of 4 letters and 7 digits separated by space right above 'SEAL'
-    - sealNumber: seal number, seal nos., shipper seal, seal.
-- vessel: vessel
-
-
-You must apply the following rules:
-- The JSON schema must be followed during the extraction.
-- The values must only include text found in the document
-- Do not normalize any entity value.
-- If 'sealNumber' is not found don't add it to the result.
-- Validate the JSON make sure it is a valid JSON ! No extra text, no missing comma!
-- Add an escape character (backwards slash) in from of all quotes in values

src/prompts/library/draftMbl/maersk/prompt.txt

@@ -1,17 +0,0 @@
-Extract the following information from the sea waybill document.
-
- **blNumber:** Find the value labeled as "B/L No.".
- **voyage:** Get the "Voyage No." value.
- **portOfLoading:** Find the value in the "Port of Loading" field.
- **portOfDischarge:** Extract the text from the "Port of Discharge" field.
- **bookingNumber:** Look for the value associated with "Booking No.".
- **containers:**
-    The document may contain multiple containers listed within the section "PARTICULARS FURNISHED BY SHIPPER" under the line starting with "Kind of Packages; Description of goods; Marks and Numbers; Container No./Seal No.".  Look for container information that starts with a line that includes "Container Said to Contain" and continues until the next instance of "Container Said to Contain" or the end of the section. For each container, extract the following:
-    * **containerType:** Extract the container type information. It is usually a combination of numbers, the word "DRY", and may include additional characters. It is found on the same line as the container number.
-    * **grossWeight:** Find the value corresponding to the "gross weight" of the container. It is usually represented in KGS and is found on the same line as the container number.
-    * **measurements:** Find the value corresponding to the "measurement" of the container. It is usually represented in CBM and is found on the same line as the container number.
-    * **packageQuantity:** Extract the "package quantity" information. It is usually a whole number and precedes the text "PACKAGE".  All container information will be on the same line as the "package quantity".
-    * **packageType:** Extract the value from the "Kind of Packages" field.
-    * **containerNumber:**  Find the container number. It starts with "MRKU" and is followed by a sequence of digits. It is found on the same line as the text "Container Said to Contain".
-    * **sealNumber:** Get the "Shipper Seal" value. It follows after the text "Shipper Seal :".
- **vessel:** Extract the text from the field "Vessel".

src/prompts/library/finalMbL/hapag-lloyd/prompt.txt

@@ -1,44 +0,0 @@
-You are a document entity extraction specialist. Given a document, the explained datapoint need to extract.
-
-blNumber: Bill of Lading number.
-voyage: The journey or route code taken by the vessel.
-portOfLoading: The port where cargo is loaded.
-portOfDischarge:  The port where cargo is unloaded.
-bookingNumber: A unique identifier for the booking.
-containers:
-    containerType: Type of the shipping container, usually related to it's size.
-    grossWeight: Total weight of the cargo, including the tare weight of the container.
-    measurements: Dimensions of the cargo (length, width, height) for freight calculations.
-    packageQuantity: package quantity.
-    packageType: Type of packaging used (e.g., cartons, pallets, barrels).
-    containerNumber: Unique ID for tracking the shipping container.
-    sealNumber: Number of the container's seal.
-vessel: The name of the vessel.
-
-
-Your task is to extract the text value of the following entities:
-
-Keywords for datapoints:
-- blNumber: Bill of Lading number, bill of landing no., swb-no., b/l no.
-- voyage: voyage, voy. no, voyage-no.
-- portOfLoading: port of loading, pol, from.]
-- portOfDischarge: port of discharge, pod, delivery, to
-- bookingNumber: Our reference, booking no., carrier reference
-- containers:
-    - containerType: x 40' container
-    - grossWeight: gross weight
-    - measurements: Dimensions of the cargo (length, width, height) for freight calculations
-    - packageQuantity: package quantity, number and kind of packages
-    - packageType: Type of packaging used (e.g., cartons, pallets, barrels), number and kind of packages, description of goods
-    - containerNumber: container number, cntr. nos., it is a combination of 4 letters and 7 digits separated by space right above 'SEAL'
-    - sealNumber: seal number, seal nos., shipper seal, seal.
-- vessel: vessel
-
-
-You must apply the following rules:
-- The JSON schema must be followed during the extraction.
-- The values must only include text found in the document
-- Do not normalize any entity value.
-- If 'sealNumber' is not found don't add it to the result.
-- Validate the JSON make sure it is a valid JSON ! No extra text, no missing comma!
-- Add an escape character (backwards slash) in from of all quotes in values

src/prompts/library/finalMbL/maersk/prompt.txt

@@ -1,17 +0,0 @@
-Extract the following information from the sea waybill document.
-
- **blNumber:** Find the value labeled as "B/L No.".
- **voyage:** Get the "Voyage No." value.
- **portOfLoading:** Find the value in the "Port of Loading" field.
- **portOfDischarge:** Extract the text from the "Port of Discharge" field.
- **bookingNumber:** Look for the value associated with "Booking No.".
- **containers:**
-    The document may contain multiple containers listed within the section "PARTICULARS FURNISHED BY SHIPPER" under the line starting with "Kind of Packages; Description of goods; Marks and Numbers; Container No./Seal No.".  Look for container information that starts with a line that includes "Container Said to Contain" and continues until the next instance of "Container Said to Contain" or the end of the section. For each container, extract the following:
-    * **containerType:** Extract the container type information. It is usually a combination of numbers, the word "DRY", and may include additional characters. It is found on the same line as the container number.
-    * **grossWeight:** Find the value corresponding to the "gross weight" of the container. It is usually represented in KGS and is found on the same line as the container number.
-    * **measurements:** Find the value corresponding to the "measurement" of the container. It is usually represented in CBM and is found on the same line as the container number.
-    * **packageQuantity:** Extract the "package quantity" information. It is usually a whole number and precedes the text "PACKAGE".  All container information will be on the same line as the "package quantity".
-    * **packageType:** Extract the value from the "Kind of Packages" field.
-    * **containerNumber:**  Find the container number. It starts with "MRKU" and is followed by a sequence of digits. It is found on the same line as the text "Container Said to Contain".
-    * **sealNumber:** Get the "Shipper Seal" value. It follows after the text "Shipper Seal :".
- **vessel:** Extract the text from the field "Vessel".