llm-ie 1.2.3__py3-none-any.whl → 1.3.0__py3-none-any.whl

llm_ie/__init__.py

@@ -1,12 +1,12 @@
 from .data_types import LLMInformationExtractionFrame, LLMInformationExtractionDocument
 from .engines import BasicLLMConfig, ReasoningLLMConfig, Qwen3LLMConfig, OpenAIReasoningLLMConfig 
-from .engines import LlamaCppInferenceEngine, OllamaInferenceEngine, HuggingFaceHubInferenceEngine, VLLMInferenceEngine, OpenRouterInferenceEngine, OpenAIInferenceEngine, AzureOpenAIInferenceEngine, LiteLLMInferenceEngine
-from .extractors import DirectFrameExtractor, ReviewFrameExtractor, BasicFrameExtractor, BasicReviewFrameExtractor, SentenceFrameExtractor, SentenceReviewFrameExtractor, AttributeExtractor, BinaryRelationExtractor, MultiClassRelationExtractor
-from .chunkers import UnitChunker, WholeDocumentUnitChunker, SentenceUnitChunker, TextLineUnitChunker, SeparatorUnitChunker, ContextChunker, NoContextChunker, WholeDocumentContextChunker, SlideWindowContextChunker
+from .engines import LlamaCppInferenceEngine, OllamaInferenceEngine, HuggingFaceHubInferenceEngine, VLLMInferenceEngine, SGLangInferenceEngine, OpenRouterInferenceEngine, OpenAIInferenceEngine, AzureOpenAIInferenceEngine, LiteLLMInferenceEngine
+from .extractors import StructExtractor, BasicStructExtractor, DirectFrameExtractor, ReviewFrameExtractor, BasicFrameExtractor, BasicReviewFrameExtractor, SentenceFrameExtractor, SentenceReviewFrameExtractor, AttributeExtractor, BinaryRelationExtractor, MultiClassRelationExtractor
+from .chunkers import UnitChunker, WholeDocumentUnitChunker, SentenceUnitChunker, TextLineUnitChunker, SeparatorUnitChunker, LLMUnitChunker, ContextChunker, NoContextChunker, WholeDocumentContextChunker, SlideWindowContextChunker
 from .prompt_editor import PromptEditor
 
 __all__ = ["LLMInformationExtractionFrame", "LLMInformationExtractionDocument",
-           "BasicLLMConfig", "ReasoningLLMConfig", "Qwen3LLMConfig", "OpenAIReasoningLLMConfig", "LlamaCppInferenceEngine", "OllamaInferenceEngine", "HuggingFaceHubInferenceEngine", "VLLMInferenceEngine", "OpenRouterInferenceEngine", "OpenAIInferenceEngine", "AzureOpenAIInferenceEngine", "LiteLLMInferenceEngine",
-           "DirectFrameExtractor", "ReviewFrameExtractor", "BasicFrameExtractor", "BasicReviewFrameExtractor", "SentenceFrameExtractor", "SentenceReviewFrameExtractor", "AttributeExtractor", "BinaryRelationExtractor", "MultiClassRelationExtractor",
-           "UnitChunker", "WholeDocumentUnitChunker", "SentenceUnitChunker", "TextLineUnitChunker", "SeparatorUnitChunker", "ContextChunker", "NoContextChunker", "WholeDocumentContextChunker", "SlideWindowContextChunker",
+           "BasicLLMConfig", "ReasoningLLMConfig", "Qwen3LLMConfig", "OpenAIReasoningLLMConfig", "LlamaCppInferenceEngine", "OllamaInferenceEngine", "HuggingFaceHubInferenceEngine", "VLLMInferenceEngine", "SGLangInferenceEngine", "OpenRouterInferenceEngine", "OpenAIInferenceEngine", "AzureOpenAIInferenceEngine", "LiteLLMInferenceEngine",
+           "StructExtractor", "BasicStructExtractor", "DirectFrameExtractor", "ReviewFrameExtractor", "BasicFrameExtractor", "BasicReviewFrameExtractor", "SentenceFrameExtractor", "SentenceReviewFrameExtractor", "AttributeExtractor", "BinaryRelationExtractor", "MultiClassRelationExtractor",
+           "UnitChunker", "WholeDocumentUnitChunker", "SentenceUnitChunker", "TextLineUnitChunker", "SeparatorUnitChunker", "LLMUnitChunker", "ContextChunker", "NoContextChunker", "WholeDocumentContextChunker", "SlideWindowContextChunker",
            "PromptEditor"]

llm_ie/asset/default_prompts/LLMUnitChunker_user_prompt.txt

@@ -0,0 +1,129 @@
+### Task description
+You are a helpful assistant that breaks down a text document into semantic units or chunks. Each chunk should represent a coherent section of the text, such as a paragraph, subsection, or topic.
+
+### Schema definition
+You will output a JSON array of objects. Each object should have the following fields:
+    - "title": Generate a brief title summarizing the content of the chunk.
+    - "anchor_text": the first line of text in the chunk used to locate it in the original document. Must be an exact match.
+        - if there is a title or heading for the chunk, use that as the anchor_text.
+        - if there is no title or heading, use the first sentence of the chunk as the anchor_text.
+
+```JSON
+[
+    {
+        "title": "<your title here>",
+        "anchor_text": "<the anchor text of the chunk here>"
+    },
+    {
+        "title": "<your title here>",
+        "anchor_text": "<the anchor text of the chunk here>"
+    }
+]
+```
+
+### Examples
+
+**Input**:
+"# Clinical Note
+
+**Patient Name**: Michael Green  
+**Medical Record Number**: 1122334455  
+**Date of Visit**: January 5, 2025  
+**Provider**: Dr. Emily Carter, MD  
+
+## Reason for Visit  
+Follow-up for poorly controlled type 2 diabetes and complaints of occasional dizziness and blurred vision.
+
+## Summary of Visit  
+Michael Green, a 62-year-old male with a known history of type 2 diabetes, hypertension, and obesity, presents for follow-up regarding his glycemic control. Despite recent adjustments to his treatment plan, his glucose readings have remained elevated, averaging 180-220 mg/dL. He reports occasional episodes of dizziness and blurred vision, particularly in the morning before meals. He denies chest pain, palpitations, or recent falls. He reports compliance with his medication regimen but admits to difficulty following a consistent low-carbohydrate diet.
+
+Michael has been using a glucose meter to monitor his blood sugar levels and logs them daily. His last hemoglobin A1c, performed three months ago, was 9.2%. He reports no recent hospitalizations, infections, or significant stressors.
+
+## Notable History  
+- **Chronic Conditions**:  
+  - Type 2 diabetes mellitus, diagnosed 10 years ago.  
+  - Hypertension, well-controlled on medication.  
+  - Hyperlipidemia, on statin therapy.  
+- **Past Surgical History**:  
+  - Knee arthroscopy for a meniscal tear, age 50.  
+- **Family History**:  
+  - Mother: Deceased at 75, complications from diabetes.  
+  - Father: Deceased at 70, myocardial infarction."
+
+**Output**:
+```JSON
+[
+    {
+        "title": "Patient Information",
+        "anchor_text": "# Clinical Note"
+    },
+    {
+        "title": "Reason for Visit",
+        "anchor_text": "## Reason for Visit"
+    },
+    {
+        "title": "Summary of Visit",
+        "anchor_text": "## Summary of Visit"
+    },
+    {
+        "title": "Notable History",
+        "anchor_text": "## Notable History"
+    }
+]
+```
+
+**Input**:
+"In the [**Hospital1 18**] ED, 35.3 102 133/58 100%AC 500x20, 5, 1.0 with an
+ABG 7.16/66/162. He had a CTH which was unremarkable. He then
+had a CTA chest, afterwhich he went into PEA arrest.
+Rescucitation last approximately 10-15 minutes with multiple
+rounds of epi and bicarb, with ROSC. He was then admitted to the
+MICU for further management.
+.
+Currently, the patient is intubated, sedated, and parlyzed.
+
+Past Medical History:
+Asthma
+Dilated cardiomyopathy
+Multiple admissions for dyspnea this winter ([**1-26**]).
+Anxiety/depression
+CKD
+HLD
+Obesity
+HTN
+
+Social History:
+Unknown
+
+Family History:
+Unknown"
+
+**Output**:
+```JSON
+[
+    {
+        "title": "Patient Presentation and Initial Management",
+        "anchor_text": "In the [**Hospital1 18**] ED, 35.3 102 133/58 100%AC 500x20, 5, 1.0 with an"
+    },
+    {
+        "title": "Current Status of the Patient",
+        "anchor_text": "Currently, the patient is intubated, sedated, and parlyzed."
+    },
+    {
+        "title": "Past Medical History",
+        "anchor_text": "Past Medical History:"
+    },
+    {
+        "title": "Social History",
+        "anchor_text": "Social History:"
+    },
+    {
+        "title": "Family History",
+        "anchor_text": "Family History:"
+    }
+]
+```
+
+### Document text
+
+"{{document_text}}"

llm_ie/asset/prompt_guide/AttributeExtractor_prompt_guide.txt

@@ -7,7 +7,7 @@ Prompt Template Design:
    List the attributes to extract, and provide clear definitions for each one. 
 
 3. Output Format Definition:  
-   The output should be a JSON list, where each attribute be a key. The values could be any structure (e.g., str, int, List[str]).
+   The output should be a JSON, where each attribute be a key. The values could be any structure (e.g., str, int, List[str]).
 
 4. Optional: Hints:  
    Provide itemized hints for the information extractors to guide the extraction process. Remind the prompted agent to be truthful. Emphasize that the prompted agent is supposed to perform the task instead of writting code or instruct other agents to do it.
@@ -37,7 +37,7 @@ Example:
     Your output should follow the JSON format:
     {"Date": "<MM/DD/YYYY>", "Status": "<status>"}
 
-    I am only interested in the content between []. Do not explain your answer. 
+    I am only interested in the content between {}. Do not explain your answer. 
 
     ### Hints
     - If the date is not complete, use the first available date in the context. For example, if the date is 01/2023, you should return 01/01/2023.

llm_ie/asset/prompt_guide/StructExtractor_prompt_guide.txt

@@ -0,0 +1,53 @@
+Prompt Template Design:
+
+1. Task Description:  
+   Provide a detailed description of the task, including the background and the type of task (e.g., structured data extraction task).
+
+2. Schema Definition:  
+   List the key-value pairs to extract, and provide clear definitions for each one. 
+
+3. Output Format Definition:  
+   The output should be a JSON. The values could be any structure (e.g., str, int, List[str]).
+
+4. Optional: Hints:  
+   Provide itemized hints for the information extractors to guide the extraction process. Remind the prompted agent to be truthful. Emphasize that the prompted agent is supposed to perform the task instead of writting code or instruct other agents to do it.
+
+5. Optional: Examples:  
+   Include examples in the format:  
+    Input: ...  
+    Output: ...
+
+6. Context:  
+   The template must include a placeholder {{input}} for the document or chunk. 
+   
+
+Example:
+
+    ### Task description
+    This is an structured data extraction task. Given a medical report, you need to extract structured patient data from it.
+
+    ### Schema definition
+        "PatientName" which is the name of the patient,
+        "Age" which is the age of the patient in years,
+        "MRN" which is the medical record number of the patient. 
+
+    ### Output format definition
+    Your output should follow the JSON format:
+    ```json
+   {
+      "PatientName": "<patient_name>",
+      "Age": <age_in_years>,
+      "MRN": "<medical_record_number>"
+   }
+   ```
+    I am only interested in the content between {}. Do not explain your answer. 
+
+    ### Hints
+    - Make sure to extract the exact patient name as it appears in the report.
+    - You are suppose to perform the extraction task instead of writting code or instruct other agents to do it.
+    - If some values are not available, you should return "not specified".
+
+    ### Context
+    The text below is from the medical report:
+
+    "{{input}}"

llm_ie/chunkers.py

@@ -1,8 +1,11 @@
 import abc
-from typing import Set, List, Dict, Tuple, Union, Callable
+from typing import List
 import asyncio
 import uuid
+import importlib.resources
+from llm_ie.utils import extract_json, apply_prompt_template
 from llm_ie.data_types import FrameExtractionUnit
+from llm_ie.engines import InferenceEngine
 
 
 class UnitChunker(abc.ABC):
@@ -74,13 +77,14 @@ class SeparatorUnitChunker(UnitChunker):
         text : str
             The document text.
         """
+        doc_id = doc_id if doc_id is not None else str(uuid.uuid4())
         paragraphs = text.split(self.sep)
         paragraph_units = []
         start = 0
         for paragraph in paragraphs:
             end = start + len(paragraph)
             paragraph_units.append(FrameExtractionUnit(
-                doc_id=doc_id if doc_id is not None else str(uuid.uuid4()),
+                doc_id=doc_id,
                 start=start,
                 end=end,
                 text=paragraph
@@ -104,10 +108,11 @@ class SentenceUnitChunker(UnitChunker):
         text : str
             The document text.
         """
+        doc_id = doc_id if doc_id is not None else str(uuid.uuid4())
         sentences = []
         for start, end in self.PunktSentenceTokenizer().span_tokenize(text):
             sentences.append(FrameExtractionUnit(
-                doc_id=doc_id if doc_id is not None else str(uuid.uuid4()),
+                doc_id=doc_id,
                 start=start,
                 end=end,
                 text=text[start:end]
@@ -129,13 +134,14 @@ class TextLineUnitChunker(UnitChunker):
         text : str
             The document text.
         """
+        doc_id = doc_id if doc_id is not None else str(uuid.uuid4())
         lines = text.split('\n')
         line_units = []
         start = 0
         for line in lines:
             end = start + len(line)
             line_units.append(FrameExtractionUnit(
-                doc_id=doc_id if doc_id is not None else str(uuid.uuid4()),
+                doc_id=doc_id,
                 start=start,
                 end=end,
                 text=line
@@ -143,6 +149,100 @@ class TextLineUnitChunker(UnitChunker):
             start = end + 1 
         return line_units
     
+class LLMUnitChunker(UnitChunker):
+    def __init__(self, inference_engine:InferenceEngine, prompt_template:str=None, system_prompt:str=None):
+        """
+        This class prompt an LLM for document segmentation (e.g., sections, paragraphs).
+
+        Parameters:
+        ----------
+        inference_engine : InferenceEngine
+            the LLM inferencing engine object.
+        prompt_template : str
+            the prompt template that defines how to chunk the document. Must define a JSON schema with 
+            ```json
+            [
+                {
+                    "title": "<your title here>",
+                    "anchor_text": "<the anchor text of the chunk here>"
+                },
+                {
+                    "title": "<your title here>",
+                    "anchor_text": "<the anchor text of the chunk here>"
+                }
+            ]
+            ```
+        system_prompt : str, optional
+            The system prompt.
+        """
+        self.inference_engine = inference_engine
+
+        if prompt_template is None:
+            file_path = importlib.resources.files('llm_ie.asset.default_prompts').joinpath("LLMUnitChunker_user_prompt.txt")
+            with open(file_path, 'r', encoding="utf-8") as f:
+                self.prompt_template = f.read()
+        else:
+            self.prompt_template = prompt_template
+        
+        self.system_prompt = system_prompt
+
+    def chunk(self, text, doc_id=None) -> List[FrameExtractionUnit]:
+        """
+        Parameters:
+        -----------
+        text : str
+            the document text.
+        doc_id : str, optional
+            the document id.
+        """
+        doc_id = doc_id if doc_id is not None else str(uuid.uuid4())
+        user_prompt = apply_prompt_template(prompt_template=self.prompt_template, text_content=text)
+        messages = []
+        if self.system_prompt is not None:
+            messages.append({'role': 'system', 'content': self.system_prompt})
+        messages.append({'role': 'user', 'content': user_prompt})
+
+        gen_text = self.inference_engine.chat(messages=messages)
+
+        header_list = extract_json(gen_text=gen_text["response"])
+        units = []
+        start = 0
+        prev_end = 0
+        for header in header_list:
+            if "anchor_text" not in header:
+                Warning.warn(f"Missing anchor_text in header: {header}. Skipping this header.")
+                continue
+            if not isinstance(header["anchor_text"], str):
+                Warning.warn(f"Invalid anchor_text: {header['anchor_text']}. Skipping this header.")
+                continue
+
+            start = prev_end
+            # find the first instandce of the leading sentence in the rest of the text
+            end = text.find(header["anchor_text"], start)
+            # if not found, skip this header
+            if end == -1:
+                continue
+            # if start == end (empty text), skip this header
+            if start == end:
+                continue
+            # create a frame extraction unit
+            units.append(FrameExtractionUnit(
+                doc_id=doc_id,
+                start=start,
+                end=end,
+                text=text[start:end]
+            ))
+            prev_end = end
+        # add the last section
+        if prev_end < len(text):
+            units.append(FrameExtractionUnit(
+                doc_id=doc_id,
+                start=prev_end,
+                end=len(text),
+                text=text[prev_end:]
+            ))
+        return units
+
 
 class ContextChunker(abc.ABC):
     def __init__(self):

llm_ie/data_types.py

@@ -141,7 +141,7 @@ class LLMInformationExtractionFrame:
 
 
 class LLMInformationExtractionDocument:
-    def __init__(self, doc_id:str=None, filename:str=None, text:str=None, 
+    def __init__(self, doc_id:str=None, filename:str=None, text:str=None, struct:Dict=None,
                  frames:List[LLMInformationExtractionFrame]=None, relations:List[Dict[str,str]]=None):
         """
         This class holds LLM-extracted frames, handles save/ load.
@@ -154,6 +154,8 @@ class LLMInformationExtractionDocument:
             the directory to a yaml file of a saved LLMInformationExtractionDocument
         text : str, Optional
             document text
+        struct : Dict, Optional
+            a dictionary of unanchored structure information
         frames : List[LLMInformationExtractionFrame], Optional
             a list of LLMInformationExtractionFrame
         relations : List[Dict[str,str]], Optional
@@ -168,12 +170,28 @@ class LLMInformationExtractionDocument:
                 llm_ie = json.load(json_file)
             if 'doc_id' in llm_ie.keys():
                 self.doc_id = llm_ie['doc_id']
+            else: 
+                raise ValueError("doc_id key not found in the file.")
+            
             if 'text' in llm_ie.keys():
                 self.text = llm_ie['text']
+            else:
+                raise ValueError("text key not found in the file.")
+            
+            if 'struct' in llm_ie.keys():
+                self.struct = llm_ie['struct']
+            else:
+                self.struct = {}
+
             if 'frames' in llm_ie.keys():
                 self.frames = [LLMInformationExtractionFrame.from_dict(d) for d in llm_ie['frames']]
+            else:
+                self.frames = []
+
             if 'relations' in llm_ie.keys():
                 self.relations = llm_ie['relations']
+            else:
+                self.relations = []
 
         # create object from raw inputs
         else:
@@ -181,9 +199,15 @@ class LLMInformationExtractionDocument:
                 raise TypeError("doc_id must be a string.")
             self.doc_id = doc_id
             self.text = text
+            self.struct = struct.copy() if struct is not None else {}
             self.frames = frames.copy() if frames is not None else []
             self.relations = relations.copy() if relations is not None else []
 
+    def has_struct(self) -> bool:
+        """
+        This method checks if there is any unanchored structure information.
+        """
+        return bool(self.struct)
 
     def has_frame(self) -> bool:
         """
@@ -228,6 +252,18 @@ class LLMInformationExtractionDocument:
 
         return None
 
+    def set_struct(self, struct:Dict):
+        """
+        This method sets the unanchored structure information.
+
+        Parameters
+        ----------
+        struct : Dict
+            a dictionary of unanchored structure information
+        """
+        if not isinstance(struct, Dict):
+            raise TypeError("struct must be a dictionary.")
+        self.struct = struct.copy()
     
     def add_frame(self, frame:LLMInformationExtractionFrame, valid_mode:str=None, create_id:bool=False) -> bool:
         """
@@ -326,10 +362,12 @@ class LLMInformationExtractionDocument:
 
     def __repr__(self, N_top_chars:int=100) -> str:
         text_to_print = self.text[0:N_top_chars]
+        struct_key_count = len(self.struct.keys())
         frame_count = len(self.frames)
         relation_count = len(self.relations)
         return ''.join((f'LLMInformationExtractionDocument(doc_id: "{self.doc_id}"\n',
                         f'text: "{text_to_print}...",\n',
+                        f'struct keys: {struct_key_count}\n',
                         f'frames: {frame_count}\n',
                         f'relations: {relation_count}'))
 
@@ -338,6 +376,7 @@ class LLMInformationExtractionDocument:
         with open(filename, 'w') as json_file:
             json.dump({'doc_id': self.doc_id, 
                         'text': self.text, 
+                        'struct': self.struct,
                         'frames': [frame.to_dict() for frame in self.frames],
                         'relations': self.relations}, 
                         json_file, indent=4)
@@ -346,16 +385,22 @@ class LLMInformationExtractionDocument:
 
     def _viz_preprocess(self) -> Tuple:
         """
-        This method preprocesses the entities and relations for visualization.
+        This method preprocesses the struct, entities and relations for visualization.
         """
         if importlib.util.find_spec("ie_viz") is None:
-            raise ImportError("ie_viz not found. Please install ie_viz (```pip install ie-viz```).")
+            raise ImportError("ie_viz not found. Please install ie_viz (```pip install -U ie-viz```).")
 
+        # Struct
+        if self.has_struct():
+            struct = self.struct
+        else:
+            struct = {}
+        # Entities
         if self.has_frame():
             entities = [{"entity_id": frame.frame_id, "start": frame.start, "end": frame.end, "attr": frame.attr} for frame in self.frames]
         else:
-            raise ValueError("No frames in the document.")
-        
+            entities = None
+        # Relations
         if self.has_relation():
             relations = []
             for relation in self.relations:
@@ -364,7 +409,7 @@ class LLMInformationExtractionDocument:
         else:
             relations = None
 
-        return entities, relations
+        return struct, entities, relations
 
 
     def viz_serve(self, host: str = '0.0.0.0', port: int = 5000, theme:str = "light", title:str="Frames Visualization",
@@ -388,29 +433,20 @@ class LLMInformationExtractionDocument:
             The function to be used for mapping the entity attributes to colors. When provided, the color_attr_key and 
             theme will be overwritten. The function must take an entity dictionary as input and return a color string (hex).
         """
-        entities, relations = self._viz_preprocess()
+        struct, entities, relations = self._viz_preprocess()
         from ie_viz import serve
 
-        try:
-            serve(text=self.text,
-                    entities=entities,
-                    relations=relations,
-                    host=host,
-                    port=port,
-                    theme=theme,
-                    title=title,
-                    color_attr_key=color_attr_key,
-                    color_map_func=color_map_func)
-        except TypeError:
-            warnings.warn("The version of ie_viz is not the latest. Please update to the latest version (pip install --upgrade ie-viz) for complete features.", UserWarning)
-            serve(text=self.text,
-                    entities=entities,
-                    relations=relations,
-                    host=host,
-                    port=port,
-                    theme=theme,
-                    color_attr_key=color_attr_key,
-                    color_map_func=color_map_func)
+        serve(text=self.text,
+              struct=struct,
+              entities=entities,
+              relations=relations,
+              host=host,
+              port=port,
+              theme=theme,
+              title=title,
+              color_attr_key=color_attr_key,
+              color_map_func=color_map_func)
+
     
     def viz_render(self, theme:str = "light", color_attr_key:str=None, color_map_func:Callable=None,
                    title:str="Frames Visualization") -> str:
@@ -429,22 +465,14 @@ class LLMInformationExtractionDocument:
         title : str, Optional
             the title of the HTML.
         """
-        entities, relations = self._viz_preprocess()
+        struct, entities, relations = self._viz_preprocess()
         from ie_viz import render
 
-        try:
-            return render(text=self.text,
-                        entities=entities,
-                        relations=relations,
-                        theme=theme,
-                        title=title,
-                        color_attr_key=color_attr_key,
-                        color_map_func=color_map_func)
-        except TypeError:
-                warnings.warn("The version of ie_viz is not the latest. Please update to the latest version (pip install --upgrade ie-viz) for complete features.", UserWarning)
-                return render(text=self.text,
-                        entities=entities,
-                        relations=relations,
-                        theme=theme,
-                        color_attr_key=color_attr_key,
-                        color_map_func=color_map_func)
+        return render(text=self.text,
+                      struct=struct,
+                      entities=entities,
+                      relations=relations,
+                      theme=theme,
+                      title=title,
+                      color_attr_key=color_attr_key,
+                      color_map_func=color_map_func)

llm_ie/engines.py

@@ -1060,6 +1060,50 @@ class VLLMInferenceEngine(OpenAICompatibleInferenceEngine):
         return {"reasoning": getattr(response.choices[0].message, "reasoning_content", ""),
                 "response": getattr(response.choices[0].message, "content", "")}
         
+class SGLangInferenceEngine(OpenAICompatibleInferenceEngine):
+    def __init__(self, model:str, api_key:str="", base_url:str="http://localhost:30000/v1", config:LLMConfig=None, **kwrs):
+        """
+        SGLang OpenAI compatible API inference engine.
+        https://docs.sglang.ai/basic_usage/openai_api.html
+
+        Parameters:
+        ----------
+        model_name : str
+            model name as shown in the vLLM server
+        api_key : str, Optional
+            the API key for the vLLM server.
+        base_url : str, Optional
+            the base url for the vLLM server. 
+        config : LLMConfig
+            the LLM configuration.
+        """
+        super().__init__(model, api_key, base_url, config, **kwrs)
+
+
+    def _format_response(self, response: Any) -> Dict[str, str]:
+        """
+        This method format the response from OpenAI API to a dict with keys "type" and "data".
+
+        Parameters:
+        ----------
+        response : Any
+            the response from OpenAI-compatible API. Could be a dict, generator, or object.
+        """
+        if isinstance(response, self.ChatCompletionChunk):
+            if hasattr(response.choices[0].delta, "reasoning_content") and getattr(response.choices[0].delta, "reasoning_content") is not None:
+                chunk_text = getattr(response.choices[0].delta, "reasoning_content", "")
+                if chunk_text is None:
+                    chunk_text = ""
+                return {"type": "reasoning", "data": chunk_text}
+            else:
+                chunk_text = getattr(response.choices[0].delta, "content", "")
+                if chunk_text is None:
+                    chunk_text = ""
+                return {"type": "response", "data": chunk_text}
+
+        return {"reasoning": getattr(response.choices[0].message, "reasoning_content", ""),
+                "response": getattr(response.choices[0].message, "content", "")}
+    
 
 class OpenRouterInferenceEngine(OpenAICompatibleInferenceEngine):
     def __init__(self, model:str, api_key:str=None, base_url:str="https://openrouter.ai/api/v1", config:LLMConfig=None, **kwrs):