data-science-document-ai 1.56.1__py3-none-any.whl → 1.58.0__py3-none-any.whl

src/prompts/library/bookingConfirmation/other/prompt.txt

@@ -1,58 +1,57 @@
-You are a document entity extraction specialist. Given a document, the explained datapoint need to extract.
-
-bookingNumber: A unique identifier for the booking.
-cyCutOff: The deadline for cargo to be delivered to the Container Yard.
-gateInReference: A reference code for cargo entering the terminal.
-gateInTerminal: The specific terminal where cargo is gated in.
-mblNumber: The Master Bill of Lading number. Mostly comes after BL no., B/L no. etc
-pickUpReference: A reference code for cargo pickup.
-pickUpTerminal: The specific terminal for cargo pickup.
-siCutOff: The deadline for submitting shipping instructions.
-vgmCutOff: The deadline for submitting the Verified Gross Mass of the cargo.
-transportLegs:
-    eta: The estimated time of arrival for a specific leg.
-    etd: The estimated time of departure for a specific leg.
-    imoNumber: The International Maritime Organization number for a specific leg.
-    portOfDischarge: The port where cargo is unloaded for a specific leg.
-    portOfLoading: The port where cargo is loaded for a specific leg.
-    vesselName: The name of the vessel for a specific leg.
-    voyage: The journey or route taken by the vessel for a specific leg.
-
-your task is to extract the text value of the following entities and page numbers starting from 0 where the value was found in the document:
-SCHEMA_PLACEHOLDER
-
-Further explanation for the transportLegs part as follows:
-- There is at least one leg in each document
-- There may be multiple legs between the initial and final destination
-- If there are multiple eta, etd, vesselName, ports, there is a higher chance multiple legs occurs
-- Some documents not following an order between legs
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
+
+<TASK> Your task is to extract data from Booking Confirmation documents as per the given response schema structure. <TASK>
+
+<CONTEXT>
+The Freight Forwarding company receives Booking Confirmation from Carrier (Shipping Lines) partners.
+These Booking Confirmations contain various details related to booking, container pick up and drop off depot details, vessel details, as well as other transport Legs data.
+They may be written in different languages such as English, German, Vietnamese, Chinese, and other European languages, and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these Booking Confirmations to support efficient processing and accurate record-keeping.
+
+
+To provide context on the journey of a containers for both Export and Import shipments,
+For Export shipment: An empty container is picked up from a depot (pickupDepotCode) using a pickUpReference and goods loaded into it at a warehouse. Then the loaded container / cargo is transported back to a Container Yard or gateInTerminal before the cyCutOff date for further shipping processes.
+For Import Shipment: The loaded container / cargo arrives at a port of discharge then picked up at pickUpTerminal using pickUpReference. After delivery, an empty container is returned to a depot (dropOffDepotCode).
+<CONTEXT>
+
+<INSTRUCTIONS>
+- Populate fields as defined in the response schema.
+- Use the data field description to understand the context of the data.
+
+- gateInTerminal: The specific terminal where cargo is gated in. It can be found as Export terminal delivery address, PORT OF LOADING (after the slash '/').
+- gateInReference: A reference code for cargo entering the terminal. If not mentioned explicitly and gateInTerminal is extracted, then use bookingNumber as gateInReference.
+- pickUpTerminal: The specific terminal for cargo pickup. It can be found as Import pick up address(es),  PORT OF DISCHARGE (after the slash '/').
+- pickUpReference: A reference code for cargo pickup. If not mentioned explicitly and pickUpTerminal is extracted, then use bookingNumber as pickUpReference.
+
+- cyCutOff: The deadline for cargo to be delivered to the Container Yard. It can be referred to as FCL delivery cut-off, CY CUT OFF, CY Closing - Latest Return Container Date, Cargo Cut-off deadline
+- siCutOff: The deadline for submitting shipping instructions. It can be referred to as Shipping Instruction closing, SI Cut Off, Shipping Instruction deadline, INTENDED SI CUT-OFF
+- vgmCutOff: The deadline for submitting the Verified Gross Mass of the cargo. It can be referred to as VGM cut-off, VGM Submission Deadline, Verified Gross Mass deadline
+
+- carrierName and carrierAddress:
+    - Extract the name and address of the carrier who is the main parent company in the document.
+
+- transportLegs: Multiple Transport Legs entries may exist, capture all instances under "transportLegs". Make sure the order of the legs are important.
+    - eta: The estimated time of arrival for a specific leg.
+    - etd: The estimated time of departure for a specific leg.
+    - imoNumber: The International Maritime Organization number for a specific leg.
+    - portOfDischarge: The port where cargo is unloaded for a specific leg.
+    - portOfLoading: The port where cargo is loaded for a specific leg.
+    - vesselName: The name of the vessel for a specific leg.
+    - voyage: The journey or route taken by the vessel for a specific leg.
+
+- Containers: Need to extract Depot details per Container Type. Multiple Containers entries may exist, capture all instances under "Containers".
+    - containerType: The type of container (e.g., 20FT, 40FT, 20ft, 40ft, 40HC, 20DC, etc...).
+    - pickupDepotCode: The code of the depot where the empty container is picked up.
+    - dropOffDepotCode: The code of the depot where the empty container is dropped off.
+
+IMPORTANT explanation for the transportLegs part as follows:
+- There is at least one leg in each document.
 - 'eta' must be equal or later than 'etd'!
-- portOfLoading and portOfDischarge are name of the Ports. You can rely on the port names from all over the world.
-- portOfLoading and portOfDischarge distinctly denotes the name of the ports. If you find abbreviation of the port use it, if not you can use the full name of the port
-- Abbrevations most likely to be in the paranthesis like follows (DEHAM).
-
-Possible keywords for datapoints:
-- bookingNumber: Our Reference, Booking No., BOOKING NUMBER
-- cyCutOff: FCL delivery cut-off, CY CUT OFF, CY Closing - Latest Return Container Date, Cargo Cut-off deadline
-- gateInReference: Our Reference
-- gateInTerminal: Export terminal delivery address, PORT OF LOADING (after the slash '/')
-- mblNumber: BL/SWB No(s)., CS Reference Number
-- pickUpReference: Export door positioning address(es), Empty Container Depot and Location interception, S/C
-- pickUpTerminal: PORT OF DISCHARGE (after the slash '/')
-- siCutOff: shipping instruction closing, SI Cut Off, Shipping Instruction deadline, INTENDED SI CUT-OFF
-- vgmCutOff: VGM cut-off, VGM Submission Deadline, Verified Gross Mass deadline
-- eta: eta, ETA
-- etd: etd, ETD
-- imoNumber: IMO No, IMO number
-- portOfDischarge: to, PORT OF DISCHARGE
-- portOfLoading: from, PORT OF LOADING
-- vesselName: vessel, INTENDED VESSEL/VOYAGE
-- voyage: Voy. no, INTENDED VESSEL/VOYAGE
-
-You must apply the following rules:
-
-- The JSON schema must be followed during the extraction.
-- The values must only include text found in the document
-- Do not normalize any entity value.
-- If an entity is not found in the document, keep it empty or np.Nan.
-- Validate the JSON make sure its a valid JSON ! No extra text, no missing comma!
+- Multiple legs are possible. When there are multiple legs,
+    - Sequential Sorting: You must manually re-order legs based on etd then eta, regardless of their order in the source text.
+    - The Connectivity Rule: For any sequence of legs, the Destination (Port of Discharge) of the previous leg must match the Origin (Port of Loading) of the following leg.
+    - Transhipment Handling: Treat any mentioned "Transhipment Port" as the bridge between two legs (Discharge for Leg A / Loading for Leg B).
+    - Timeline Integrity: Ensure a "No Time Travel" policy: The eta of a previous leg must be earlier than or equal to the etd of the following leg.
+    - Naming Convention: Look for Port Names followed by abbreviations in parentheses, e.g., "Port Name (ABCDE)".
+
+<INSTRUCTIONS>

src/prompts/library/bookingConfirmation/yangming/placeholders.json

@@ -1,32 +1,160 @@
 {
   "type": "OBJECT",
   "properties": {
-    "cfsCutOff": {"type": "STRING", "nullable": true, "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"},
-    "bookingNumber": {"type": "STRING", "nullable": true},
-    "cyCutOff": {"type": "STRING", "nullable": true},
-    "gateInReference": {"type": "STRING", "nullable": true},
-    "gateInTerminal": {"type": "STRING", "nullable": true},
-    "mblNumber": {"type": "STRING", "nullable": true},
-    "pickUpReference": {"type": "STRING", "nullable": true},
-    "pickUpTerminal": {"type": "STRING", "nullable": true},
-    "siCutOff": {"type": "STRING", "nullable": true},
-    "vgmCutOff": {"type": "STRING", "nullable": true},
+    "bookingNumber": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "A unique identifier assigned to the shipment booking, used for tracking and reference. They are often referred to as 'Booking Number', 'Booking No.', 'Booking Ref.', 'Booking Reference', 'Booking ID', 'carrier's reference' or 'Order Ref'."
+    },
+    "contractNumber": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "It's a contract number between the carrier and Forto Logistics SE & Co KG."
+    },
+    "pickUpTerminalCode": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The specific terminal for cargo pickup during the import shipment."
+    },
+    "gateInTerminalCode": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The specific terminal where cargo is gated in especially Export terminal delivery address"
+    },
+    "serviceCode": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The Shipping service code associated with the booking confirmation."
+    },
+    "performaDate": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "the date considered to apply the rates and charges specified in the booking confirmation"
+    },
+    "cfsCutOff": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "the date by which an LCL (Less than Container Load) shipment needs to be checked in to a CFS (Container Freight Station) to meet its scheduled sailing"
+    },
+    "cyCutOff": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The date by which the cargo to be delivered to the Container Yard. It can be found with keys FCL delivery cut-off, CY CUT OFF, CY Closing."
+    },
+    "gateInReference": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "A reference code for cargo entering the terminal to drop the loaded cargo for Export. Sometimes it can be 'Our Reference'."
+    },
+    "mblNumber": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "Bill of Lading number (B/L NO.), a document issued by the carrier."
+    },
+    "pickUpReference": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "A reference code for cargo pickup during the import shipment. Sometimes it can be 'Our Reference'"
+    },
+    "siCutOff": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The deadline date for submitting the Shipping Instructions (SI) to the carrier. It can be found with keys SI DEADLINE, SI DUE, SI CUT OFF, B/L INSTRUCTION DEADLINE."
+    },
+    "vgmCutOff": {
+      "type": "STRING",
+      "nullable": true,
+        "description": "The deadline date for submitting the Verified Gross Mass (VGM) to the carrier. It can be found with keys VGM DEADLINE, VGM DUE, VGM CUT OFF."
+    },
+    "containers": {
+      "type": "ARRAY",
+      "items": {
+        "type": "OBJECT",
+        "properties": {
+          "containerType": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The size / type of the container, such as 20ft, 40ft, 40HC, 20DC etc."
+          },
+          "pickUpDepotCode": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The depot code where the empty container will be picked up."
+          },
+          "dropOffDepotCode": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The depot code where the empty container will be dropped off."
+          }
+        }
+      },
+      "required": [
+            "containerType",
+            "pickupDepotCode",
+            "dropoffDepotCode"
+          ]
+    },
     "transportLegs": {
       "type": "ARRAY",
       "items": {
         "type": "OBJECT",
         "properties": {
-            "eta": {"type": "STRING", "nullable": true},
-            "etd": {"type": "STRING", "nullable": true},
-            "imoNumber": {"type": "STRING", "nullable": true},
-            "portOfDischarge": {"type": "STRING", "nullable": true},
-            "portOfLoading": {"type": "STRING", "nullable": true},
-            "vesselName": {"type": "STRING", "nullable": true},
-            "voyage": {"type": "STRING", "nullable": true}
-          },
-        "required": []
-      }
+          "eta": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "Estimated Time of Arrival (ETA) is the expected date when the shipment will arrive at its destination."
+          },
+          "etd": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "Estimated Time of Departure (ETD) is the expected date when the shipment will leave the origin port."
+          },
+          "imoNumber": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The International Maritime Organization number for a specific leg. It can be found as IMO No, IMO number."
+          },
+          "portOfDischarge": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The port where the goods are discharged from the vessel. This is the destination port for the shipment."
+          },
+          "portOfLoading": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The port where the goods are loaded onto the vessel. This is the origin port for the shipment."
+          },
+          "vesselName": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The name of the vessel carrying the shipment. It can be found at vessel, INTENDED VESSEL/VOYAGE"
+          },
+          "voyage": {
+            "type": "STRING",
+            "nullable": true,
+            "description": "The journey or route taken by the vessel for a specific leg. It can be found at Voy. no, INTENDED VESSEL/VOYAGE"
+          }
+        }
+      },
+      "required": [
+          "eta",
+          "etd",
+          "portOfDischarge",
+          "portOfLoading",
+          "vesselName",
+          "voyage"
+        ]
+    },
+    "carrierAddress": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The address of the carrier who provides service and issued the document."
+    },
+    "carrierName": {
+      "type": "STRING",
+      "nullable": true,
+      "description": "The name of the carrier who issued the document."
     }
   },
-  "required": []
+  "required": ["bookingNumber", "transportLegs", "containers", "cyCutOff", "vgmCutOff", "siCutOff"]
 }

src/prompts/library/bookingConfirmation/yangming/prompt.txt

@@ -1,4 +1,14 @@
-You are a document entity extraction specialist. Given a document, the explained datapoint need to extract.
+<PERSONA> You are an efficient document entity data extraction specialist working for a Freight Forwarding company. <PERSONA>
+
+<TASK> Your task is to extract data from Booking Confirmation documents as per the given response schema structure. <TASK>
+
+<CONTEXT>
+The Freight Forwarding company receives Booking Confirmation from Yangming Carrier (Shipping Lines) partners.
+These Booking Confirmations contain various details related to booking, container pick up and drop off depot details, vessel details, as well as other transport Legs data.
+They may be written in different languages such as English, German, Vietnamese, Chinese, and other European languages, and can appear in a variety of formats and layouts.
+Your role is to accurately extract specific entities from these Booking Confirmations to support efficient processing and accurate record-keeping.
+<CONTEXT>
+
 
 bookingNumber: A unique identifier for the booking.
 cyCutOff: The deadline for cargo to be delivered to the Container Yard.

src/setup.py

@@ -113,8 +113,6 @@ def setup_params(args=None):
     # Directories and paths
     os.makedirs(params["folder_data"], exist_ok=True)
 
-    params = setup_docai_client_and_path(params)
-
     # Set up BigQuery client for logging
     bq_client, _ = get_bq_client(params)
     params["bq_client"] = bq_client
@@ -122,13 +120,17 @@ def setup_params(args=None):
     # Set up Vertex AI for text embeddings
     setup_vertexai(params)
 
-    # Load models from YAML file
-    current_dir = os.path.dirname(__file__)
-    file_path = os.path.join(current_dir, "docai_processor_config.yaml")
-    with open(file_path) as file:
-        yaml_content = yaml.safe_load(file)
-        assert params.keys() & yaml_content.keys() == set()
-        params.update(yaml_content)
+    if params.get("if_use_docai"):
+        # Set up Document AI client and processor paths
+        params = setup_docai_client_and_path(params)
+
+        # Load models from YAML file
+        current_dir = os.path.dirname(__file__)
+        file_path = os.path.join(current_dir, "docai_processor_config.yaml")
+        with open(file_path) as file:
+            yaml_content = yaml.safe_load(file)
+            assert params.keys() & yaml_content.keys() == set()
+            params.update(yaml_content)
 
     # Set up LLM clients
     params["LlmClient"] = LlmClient(

src/utils.py

@@ -361,7 +361,10 @@ def extract_top_pages(pdf_bytes, num_pages=4):
 
 
 async def get_tms_mappings(
-    input_list: List[str], embedding_type: str, llm_ports: Optional[List[str]] = None
+    input_list: List[str],
+    embedding_type: str,
+    llm_ports: Optional[List[str]] = None,
+    input_key: str = None,
 ) -> Dict[str, Any]:
     """Get TMS mappings for the given values.
 
@@ -370,6 +373,7 @@ async def get_tms_mappings(
         embedding_type (str): Type of embedding to use
          (e.g., "container_types", "ports", "depots", "lineitems", "terminals").
         llm_ports (list[str], optional): List of LLM ports to use. Defaults to None.
+        input_key (str, optional): Key to use for input list in payload. Defaults to None.
 
     Returns:
         dict or string: A dictionary or a string with the mapping results.
@@ -389,7 +393,7 @@ async def get_tms_mappings(
         input_list = [input_list]
 
     # Always send a dict with named keys
-    payload = {embedding_type: input_list}
+    payload = {input_key or embedding_type: input_list}
 
     if llm_ports:
         payload["llm_ports"] = llm_ports if isinstance(llm_ports, list) else [llm_ports]