llm-ie 0.1.7__py3-none-any.whl → 0.2.0__py3-none-any.whl

llm_ie/asset/prompt_guide/BinaryRelationExtractor_prompt_guide.txt

@@ -0,0 +1,38 @@
+Prompt template design:
+    1. Task description (mention binary relation extraction and ROI)
+    2. Schema definition (defines relation)
+    3. Output format definition (must use the key "Relation")
+    4. Hints
+    5. Input placeholders (must include "roi_text", "frame_1", and "frame_2" placeholders)
+
+
+Example:
+
+    # Task description
+    This is a binary relation extraction task. Given a region of interest (ROI) text and two entities from a medical note, indicate the relation existence between the two entities.
+
+    # Schema definition
+        True: if there is a relationship between a medication name (one of the entities) and its strength or frequency (the other entity).
+        False: Otherwise.
+
+    # Output format definition
+    Your output should follow the JSON format:
+    {"Relation": "<True or False>"}
+
+    I am only interested in the content between []. Do not explain your answer. 
+
+    # Hints
+        1. Your input always contains one medication entity and 1) one strength entity or 2) one frequency entity.
+        2. Pay attention to the medication entity and see if the strength or frequency is for it.
+        3. If the strength or frequency is for another medication, output False. 
+        4. If the strength or frequency is for the same medication but at a different location (span), output False.
+
+    # Input placeholders
+    ROI Text with the two entities annotated with <entity_1> and <entity_2>:
+    "{{roi_text}}"
+
+    Entity 1 full information:
+    {{frame_1}}
+
+    Entity 2 full information:
+    {{frame_2}}

llm_ie/asset/prompt_guide/MultiClassRelationExtractor_prompt_guide.txt

@@ -0,0 +1,46 @@
+Prompt template design:
+    1. Task description (mention multi-class relation extraction and ROI)
+    2. Schema definition (defines relation types)
+    3. Output format definition (must use the key "RelationType")
+    4. Input placeholders (must include "roi_text", "frame_1", and "frame_2" placeholders)
+
+
+Example:
+
+    # Task description
+    This is a multi-class relation extraction task. Given a region of interest (ROI) text and two frames from a medical note, classify the relation types between the two frames. 
+
+    # Schema definition
+        Strength-Drug: this is a relationship between the drug strength and its name. 
+        Dosage-Drug: this is a relationship between the drug dosage and its name.
+        Duration-Drug: this is a relationship between a drug duration and its name.
+        Frequency-Drug: this is a relationship between a drug frequency and its name.
+        Form-Drug: this is a relationship between a drug form and its name.
+        Route-Drug: this is a relationship between the route of administration for a drug and its name.
+        Reason-Drug: this is a relationship between the reason for which a drug was administered (e.g., symptoms, diseases, etc.) and a drug name.
+        ADE-Drug: this is a relationship between an adverse drug event (ADE) and a drug name.
+
+    # Output format definition
+    Choose one of the relation types listed below or choose "No Relation":
+    {{pos_rel_types}}
+
+    Your output should follow the JSON format:
+    {"RelationType": "<relation type or No Relation>"}
+
+    I am only interested in the content between []. Do not explain your answer. 
+
+    # Hints
+        1. Your input always contains one medication entity and 1) one strength entity or 2) one frequency entity.
+        2. Pay attention to the medication entity and see if the strength or frequency is for it.
+        3. If the strength or frequency is for another medication, output "No Relation". 
+        4. If the strength or frequency is for the same medication but at a different location (span), output "No Relation".
+
+    # Input placeholders
+    ROI Text with the two entities annotated with <entity_1> and <entity_2>:
+    "{{roi_text}}"
+
+    Entity 1 full information:
+    {{frame_1}}
+
+    Entity 2 full information:
+    {{frame_2}}

llm_ie/data_types.py

@@ -1,5 +1,5 @@
-from typing import List, Dict
-import yaml
+from typing import List, Dict, Iterable
+import json
 
 
 class LLMInformationExtractionFrame:
@@ -22,7 +22,8 @@ class LLMInformationExtractionFrame:
         attr : Dict[str,str], Optional
             dict of attributes
         """
-        assert isinstance(frame_id, str), "frame_id must be a string."
+        if not isinstance(frame_id, str):
+            raise TypeError("frame_id must be a string.")
         self.frame_id = frame_id
         self.start = start
         self.end = end
@@ -78,7 +79,8 @@ class LLMInformationExtractionFrame:
 
 
 class LLMInformationExtractionDocument:
-    def __init__(self, doc_id:str=None, filename:str=None, text:str=None, frames:List[LLMInformationExtractionFrame]=None):
+    def __init__(self, doc_id:str=None, filename:str=None, text:str=None, 
+                 frames:List[LLMInformationExtractionFrame]=None, relations:List[Dict[str,str]]=None):
         """
         This class holds LLM-extracted frames, handles save/ load.
 
@@ -92,30 +94,79 @@ class LLMInformationExtractionDocument:
             document text
         frames : List[LLMInformationExtractionFrame], Optional
             a list of LLMInformationExtractionFrame
+        relations : List[Dict[str,str]], Optional
+            a list of dictionary of {"frame_1", "frame_2", "relation"}. 
+            If binary relation (no relation type), there is no "relation" key. 
         """
-        assert doc_id or filename, "Either doc_id (create from raw inputs) or filename (create from file) must be provided."
+        if doc_id is None and filename is None:
+            raise ValueError("Either doc_id (create from raw inputs) or filename (create from file) must be provided.")
         # if create object from file
         if filename:
-            with open(filename) as yaml_file:
-                llm_ie = yaml.safe_load(yaml_file)
+            with open(filename) as json_file:
+                llm_ie = json.load(json_file)
             if 'doc_id' in llm_ie.keys():
                 self.doc_id = llm_ie['doc_id']
             if 'text' in llm_ie.keys():
                 self.text = llm_ie['text']
             if 'frames' in llm_ie.keys():
                 self.frames = [LLMInformationExtractionFrame.from_dict(d) for d in llm_ie['frames']]
+            if 'relations' in llm_ie.keys():
+                self.relations = llm_ie['relations']
 
         # create object from raw inputs
         else:
-            assert isinstance(doc_id, str), "doc_id must be a string."
+            if not isinstance(doc_id, str):
+                raise TypeError("doc_id must be a string.")
             self.doc_id = doc_id
             self.text = text
             self.frames = frames.copy() if frames is not None else []
+            self.relations = relations.copy() if relations is not None else []
 
 
     def has_frame(self) -> bool:
+        """
+        This method checks if there is any frames.
+        """
         return bool(self.frames)
     
+    def has_relation(self) -> bool:
+        """
+        This method checks if there is any relations.
+        """
+        return bool(self.relations)
+    
+    def has_duplicate_frame_ids(self) -> bool:
+        """
+        This method checks for duplicate frame ids.
+        """
+        frame_id_set = set()
+        for frame in self.frames:
+            if frame.frame_id in frame_id_set:
+                return True
+            frame_id_set.add(frame.frame_id)
+
+        return False
+    
+    def get_frame_by_id(self, frame_id:str) -> LLMInformationExtractionFrame:
+        """
+        This method use frame_id to search for a frame. 
+        If there are redundent frame_ids, the first will be returned
+
+        Parameters:
+        -----------
+        frame_id : str
+            frame id to retrieve
+
+        Returns : LLMInformationExtractionFrame
+            a frame (if found) or None (not found).
+        """
+        for frame in self.frames:
+            if frame.frame_id == frame_id:
+                return frame
+
+        return None
+
+    
     def add_frame(self, frame:LLMInformationExtractionFrame, valid_mode:str=None, create_id:bool=False) -> bool:
         """
         This method add a new frame to the frames (list).
@@ -132,7 +183,11 @@ class LLMInformationExtractionDocument:
         create_id : bool, Optional
             Assign a sequential frame ID.
         """
-        assert valid_mode in {None, "span", "attr"}, 'valid_mode must be one of {None, "span", "attr"}'
+        if not isinstance(frame, LLMInformationExtractionFrame):
+            raise TypeError(f"Expect frame to be LLMInformationExtractionFrame, received {type(frame)} instead.")
+
+        if valid_mode not in {None, "span", "attr"}:
+            raise ValueError(f'Expect valid_mode to be one of {{None, "span", "attr"}}, received {valid_mode}')
 
         if valid_mode == "span":
             for exist_frame in self.frames:
@@ -153,18 +208,76 @@ class LLMInformationExtractionDocument:
         return True
 
 
+    def add_frames(self, frames:List[LLMInformationExtractionFrame], valid_mode:str=None, create_id:bool=False):
+        """
+        This method adds a list of frames.
+        """
+        if not isinstance(frames, Iterable):
+            raise TypeError("frames must be a list or Interable.")
+        
+        for frame in frames:
+            self.add_frame(frame=frame, valid_mode=valid_mode, create_id=create_id)
+
+
+    def add_relation(self, relation:Dict[str,str]) -> bool:
+        """
+        This method add a relation to the relations (list).
+
+        Parameters:
+        -----------
+        relation : Dict[str,str]
+            the relation to add. Must be a dict with {"frame_1", "frame_2", ("relation")}. 
+            Could have an optional "relation" key for relation type. 
+
+        Returns : bool
+            sucess addition.
+        """
+        if not isinstance(relation, Dict):
+            raise TypeError(f"Expect relation to be a Dict, received {type(relation)} instead.")
+
+        required_keys = {"frame_1", "frame_2"}
+        if not required_keys.issubset(relation.keys()):
+            raise ValueError('relation missing "frame_1" or "frame_2" keys.')
+        
+        allowed_keys = {"frame_1", "frame_2", "relation"}
+        if not set(relation.keys()).issubset(allowed_keys):
+            raise ValueError('Only keys {"frame_1", "frame_2", "relation"} are allowed.')
+        
+        if not self.get_frame_by_id(relation["frame_1"]):
+            raise ValueError(f'frame_id: {relation["frame_1"]} not found in frames.')
+        
+        if not self.get_frame_by_id(relation["frame_2"]):
+            raise ValueError(f'frame_id: {relation["frame_2"]} not found in frames.')
+
+        self.relations.append(relation)
+        return True
+
+    def add_relations(self, relations:List[Dict[str,str]]):
+        """
+        This method adds a list of relations.
+        """
+        if not isinstance(relations, Iterable):
+            raise TypeError("relations must be a list or Interable.")
+        for relation in relations:
+            self.add_relation(relation)
+
+
     def __repr__(self, N_top_chars:int=100) -> str:
         text_to_print = self.text[0:N_top_chars]
         frame_count = len(self.frames)
-        return ''.join((f'LLMInformationExtractionDocument(doc_id="{self.doc_id}...")\n',
-                        f'text="{text_to_print}...",\n',
-                        f'frames={frame_count}'))
+        relation_count = len(self.relations)
+        return ''.join((f'LLMInformationExtractionDocument(doc_id: "{self.doc_id}"\n',
+                        f'text: "{text_to_print}...",\n',
+                        f'frames: {frame_count}\n',
+                        f'relations: {relation_count}'))
+
 
     def save(self, filename:str):
-        with open(filename, 'w') as yaml_file:
-            yaml.safe_dump({'doc_id':self.doc_id, 
-                            'text':self.text, 
-                            'frames':[frame.to_dict() for frame in self.frames]}, 
-                            yaml_file, sort_keys=False)
-            yaml_file.flush()
+        with open(filename, 'w') as json_file:
+            json.dump({'doc_id': self.doc_id, 
+                        'text': self.text, 
+                        'frames': [frame.to_dict() for frame in self.frames],
+                        'relations': self.relations}, 
+                        json_file, indent=4)
+            json_file.flush()
             

llm_ie/extractors.py

@@ -1,16 +1,19 @@
 import abc
 import re
 import json
+import inspect
 import importlib.resources
-from typing import List, Dict, Tuple, Union
-from llm_ie.data_types import LLMInformationExtractionFrame
+import warnings
+import itertools
+from typing import List, Dict, Tuple, Union, Callable
+from llm_ie.data_types import LLMInformationExtractionFrame, LLMInformationExtractionDocument
 from llm_ie.engines import InferenceEngine
 
 
-class FrameExtractor:
+class Extractor:
     def __init__(self, inference_engine:InferenceEngine, prompt_template:str, system_prompt:str=None, **kwrs):
         """
-        This is the abstract class for frame extraction.
+        This is the abstract class for (frame and relation) extractors.
         Input LLM inference engine, system prompt (optional), prompt template (with instruction, few-shot examples).
 
         Parameters
@@ -26,12 +29,17 @@ class FrameExtractor:
         self.prompt_template = prompt_template
         self.system_prompt = system_prompt
 
+
     @classmethod
     def get_prompt_guide(cls) -> str:
+        """
+        This method returns the pre-defined prompt guideline for the extractor from the package asset.
+        """
         file_path = importlib.resources.files('llm_ie.asset.prompt_guide').joinpath(f"{cls.__name__}_prompt_guide.txt")
         with open(file_path, 'r') as f:
             return f.read()
 
+
     def _get_user_prompt(self, text_content:Union[str, Dict[str,str]]) -> str:
         """
         This method applies text_content to prompt_template and returns a prompt.
@@ -49,18 +57,19 @@ class FrameExtractor:
         pattern = re.compile(r'{{(.*?)}}')
         if isinstance(text_content, str):
             matches = pattern.findall(self.prompt_template)
-            assert len(matches) == 1, \
-                "When text_content is str, the prompt template must has only 1 placeholder {{<placeholder name>}}."
-            prompt = pattern.sub(text_content, self.prompt_template)
+            if len(matches) != 1:
+                raise ValueError("When text_content is str, the prompt template must has exactly 1 placeholder {{<placeholder name>}}.")
+            text = re.sub(r'\\', r'\\\\', text_content)
+            prompt = pattern.sub(text, self.prompt_template)
 
         elif isinstance(text_content, dict):
             placeholders = pattern.findall(self.prompt_template)
-            assert len(placeholders) == len(text_content), \
-                f"Expect text_content ({len(text_content)}) and prompt template placeholder ({len(placeholders)}) to have equal size."
-            assert all([k in placeholders for k, _ in text_content.items()]), \
-                f"All keys in text_content ({text_content.keys()}) must match placeholders in prompt template ({placeholders})."
+            if len(placeholders) != len(text_content):
+                raise ValueError(f"Expect text_content ({len(text_content)}) and prompt template placeholder ({len(placeholders)}) to have equal size.")
+            if not all([k in placeholders for k, _ in text_content.items()]):
+                raise ValueError(f"All keys in text_content ({text_content.keys()}) must match placeholders in prompt template ({placeholders}).")
 
-            prompt = pattern.sub(lambda match: text_content[match.group(1)], self.prompt_template)
+            prompt = pattern.sub(lambda match: re.sub(r'\\', r'\\\\', text_content[match.group(1)]), self.prompt_template)
 
         return prompt
     
@@ -79,6 +88,27 @@ class FrameExtractor:
         return out
     
 
+class FrameExtractor(Extractor):
+    def __init__(self, inference_engine:InferenceEngine, prompt_template:str, system_prompt:str=None, **kwrs):
+        """
+        This is the abstract class for frame extraction.
+        Input LLM inference engine, system prompt (optional), prompt template (with instruction, few-shot examples).
+
+        Parameters
+        ----------
+        inference_engine : InferenceEngine
+            the LLM inferencing engine object. Must implements the chat() method.
+        prompt_template : str
+            prompt template with "{{<placeholder name>}}" placeholder.
+        system_prompt : str, Optional
+            system prompt.
+        """
+        super().__init__(inference_engine=inference_engine,
+                         prompt_template=prompt_template,
+                         system_prompt=system_prompt,
+                         **kwrs)
+    
+
     def _find_entity_spans(self, text: str, entities: List[str], case_sensitive:bool=False) -> List[Tuple[int]]:
         """
         This function inputs a text and a list of entity text, 
@@ -290,7 +320,8 @@ class ReviewFrameExtractor(BasicFrameExtractor):
         super().__init__(inference_engine=inference_engine, prompt_template=prompt_template, 
                          system_prompt=system_prompt, **kwrs)
         self.review_prompt = review_prompt
-        assert review_mode in {"addition", "revision"}, 'review_mode must be one of {"addition", "revision"}.'
+        if review_mode not in {"addition", "revision"}: 
+            raise ValueError('review_mode must be one of {"addition", "revision"}.')
         self.review_mode = review_mode
 
 
@@ -528,3 +559,395 @@ class SentenceFrameExtractor(FrameExtractor):
                                 attr={k: v for k, v in ent.items() if k != entity_key and v != ""})
                     frame_list.append(frame)
         return frame_list
+
+
+class RelationExtractor(Extractor):
+    def __init__(self, inference_engine:InferenceEngine, prompt_template:str, system_prompt:str=None, **kwrs):
+        """
+        This is the abstract class for relation extraction.
+        Input LLM inference engine, system prompt (optional), prompt template (with instruction, few-shot examples).
+
+        Parameters
+        ----------
+        inference_engine : InferenceEngine
+            the LLM inferencing engine object. Must implements the chat() method.
+        prompt_template : str
+            prompt template with "{{<placeholder name>}}" placeholder.
+        system_prompt : str, Optional
+            system prompt.
+        """
+        super().__init__(inference_engine=inference_engine,
+                         prompt_template=prompt_template,
+                         system_prompt=system_prompt,
+                         **kwrs)
+        
+    def _get_ROI(self, frame_1:LLMInformationExtractionFrame, frame_2:LLMInformationExtractionFrame, 
+                 text:str, buffer_size:int=100) -> str:
+        """
+        This method returns the Region of Interest (ROI) that covers the two frames. Leaves a buffer_size of characters before and after.
+        The returned text has the two frames inline annotated with <entity_1>, <entity_2>.
+
+        Parameters:
+        -----------
+        frame_1 : LLMInformationExtractionFrame
+            a frame
+        frame_2 : LLMInformationExtractionFrame
+            the other frame
+        text : str
+            the entire document text
+        buffer_size : int, Optional
+            the number of characters before and after the two frames in the ROI text.
+
+        Return : str
+            the ROI text with the two frames inline annotated with <entity_1>, <entity_2>.
+        """
+        left_frame, right_frame = sorted([frame_1, frame_2], key=lambda f: f.start)
+        left_frame_name = "entity_1" if left_frame == frame_1 else "entity_2"
+        right_frame_name = "entity_1" if right_frame == frame_1 else "entity_2"
+
+        start = max(left_frame.start - buffer_size, 0)
+        end = min(right_frame.end + buffer_size, len(text))
+        roi = text[start:end]
+
+        roi_annotated = roi[0:left_frame.start - start] + \
+                f'<{left_frame_name}>' + \
+                roi[left_frame.start - start:left_frame.end - start] + \
+                f"</{left_frame_name}>" + \
+                roi[left_frame.end - start:right_frame.start - start] + \
+                f'<{right_frame_name}>' + \
+                roi[right_frame.start - start:right_frame.end - start] + \
+                f"</{right_frame_name}>" + \
+                roi[right_frame.end - start:end - start]
+
+        if start > 0:
+            roi_annotated = "..." + roi_annotated
+        if end < len(text):
+            roi_annotated = roi_annotated + "..."
+        return roi_annotated
+    
+
+    @abc.abstractmethod
+    def extract_relations(self, doc:LLMInformationExtractionDocument, buffer_size:int=100, max_new_tokens:int=128, 
+                         temperature:float=0.0, stream:bool=False, **kwrs) -> List[Dict]:
+        """
+        This method considers all combinations of two frames. 
+
+        Parameters:
+        -----------
+        doc : LLMInformationExtractionDocument
+            a document with frames.
+        buffer_size : int, Optional
+            the number of characters before and after the two frames in the ROI text.
+        max_new_tokens : str, Optional
+            the max number of new tokens LLM should generate. 
+        temperature : float, Optional
+            the temperature for token sampling.
+        stream : bool, Optional
+            if True, LLM generated text will be printed in terminal in real-time. 
+
+        Return : List[Dict]
+            a list of dict with {"frame_1", "frame_2"} for all relations.
+        """
+        return NotImplemented
+    
+
+class BinaryRelationExtractor(RelationExtractor):
+    def __init__(self, inference_engine:InferenceEngine, prompt_template:str, possible_relation_func: Callable, 
+                 system_prompt:str=None, **kwrs):
+        """
+        This class extracts binary (yes/no) relations between two entities.
+        Input LLM inference engine, system prompt (optional), prompt template (with instruction, few-shot examples).
+
+        Parameters
+        ----------
+        inference_engine : InferenceEngine
+            the LLM inferencing engine object. Must implements the chat() method.
+        prompt_template : str
+            prompt template with "{{<placeholder name>}}" placeholder.
+        possible_relation_func : Callable, Optional
+            a function that inputs 2 frames and returns a bool indicating possible relations between them.
+        system_prompt : str, Optional
+            system prompt.
+        """
+        super().__init__(inference_engine=inference_engine,
+                         prompt_template=prompt_template,
+                         system_prompt=system_prompt,
+                         **kwrs)
+        
+        if possible_relation_func:
+            # Check if possible_relation_func is a function
+            if not callable(possible_relation_func):
+                raise TypeError(f"Expect possible_relation_func as a function, received {type(possible_relation_func)} instead.")
+            
+            sig = inspect.signature(possible_relation_func)
+            # Check if frame_1, frame_2 are in input parameters
+            if len(sig.parameters) != 2:
+                raise ValueError("The possible_relation_func must have exactly frame_1 and frame_2 as parameters.")
+            if "frame_1" not in sig.parameters.keys():
+                raise ValueError("The possible_relation_func is missing frame_1 as a parameter.")
+            if "frame_2" not in sig.parameters.keys():
+                raise ValueError("The possible_relation_func is missing frame_2 as a parameter.")
+            # Check if output is a bool
+            if sig.return_annotation != bool:
+                raise ValueError(f"Expect possible_relation_func to output a bool, current type hint suggests {sig.return_annotation} instead.")
+
+            self.possible_relation_func = possible_relation_func
+
+
+    def _extract_relation(self, frame_1:LLMInformationExtractionFrame, frame_2:LLMInformationExtractionFrame, 
+                      text:str, buffer_size:int=100, max_new_tokens:int=128, temperature:float=0.0, stream:bool=False, **kwrs) -> bool:
+        """
+        This method inputs two frames and a ROI text, extracts the binary relation.
+
+        Parameters:
+        -----------
+        frame_1 : LLMInformationExtractionFrame
+            a frame
+        frame_2 : LLMInformationExtractionFrame
+            the other frame
+        text : str
+            the entire document text
+        buffer_size : int, Optional
+            the number of characters before and after the two frames in the ROI text.
+        max_new_tokens : str, Optional
+            the max number of new tokens LLM should generate. 
+        temperature : float, Optional
+            the temperature for token sampling.
+        stream : bool, Optional
+            if True, LLM generated text will be printed in terminal in real-time. 
+
+        Return : bool
+            a relation indicator
+        """
+        roi_text = self._get_ROI(frame_1, frame_2, text, buffer_size=buffer_size)
+        if stream:
+            print(f"\n\nROI text: \n{roi_text}\n")
+            print("Extraction:")
+
+        messages = []
+        if self.system_prompt:
+            messages.append({'role': 'system', 'content': self.system_prompt})
+
+        messages.append({'role': 'user', 'content': self._get_user_prompt(text_content={"roi_text":roi_text, 
+                                                                                        "frame_1": str(frame_1.to_dict()),
+                                                                                        "frame_2": str(frame_2.to_dict())}
+                                                                                        )})
+        response = self.inference_engine.chat(
+                    messages=messages,
+                    max_new_tokens=max_new_tokens,
+                    temperature=temperature,
+                    stream=stream,
+                    **kwrs
+                )
+        
+        rel_json = self._extract_json(response)
+        if len(rel_json) > 0:
+            if "Relation" in rel_json[0]:
+                rel = rel_json[0]["Relation"]
+                if isinstance(rel, bool):
+                    return rel
+                elif isinstance(rel, str) and rel in {"True", "False"}:
+                    return eval(rel)
+                else:
+                    warnings.warn('Extractor output JSON "Relation" key does not have bool or {"True", "False"} as value.' + \
+                                  'Following default, relation = False.', RuntimeWarning)
+            else:
+                warnings.warn('Extractor output JSON without "Relation" key. Following default, relation = False.', RuntimeWarning)
+        else:
+            warnings.warn("Extractor did not output a JSON. Following default, relation = False.", RuntimeWarning)
+
+        return False
+    
+    
+    def extract_relations(self, doc:LLMInformationExtractionDocument, buffer_size:int=100, max_new_tokens:int=128, 
+                         temperature:float=0.0, stream:bool=False, **kwrs) -> List[Dict]:
+        """
+        This method considers all combinations of two frames. Use the possible_relation_func to filter impossible pairs.
+
+        Parameters:
+        -----------
+        doc : LLMInformationExtractionDocument
+            a document with frames.
+        buffer_size : int, Optional
+            the number of characters before and after the two frames in the ROI text.
+        max_new_tokens : str, Optional
+            the max number of new tokens LLM should generate. 
+        temperature : float, Optional
+            the temperature for token sampling.
+        stream : bool, Optional
+            if True, LLM generated text will be printed in terminal in real-time. 
+
+        Return : List[Dict]
+            a list of dict with {"frame_1", "frame_2"} for all relations.
+        """
+        if not doc.has_frame():
+            raise ValueError("Input document must have frames.")
+
+        if doc.has_duplicate_frame_ids():
+            raise ValueError("All frame_ids in the input document must be unique.")
+
+        pairs = itertools.combinations(doc.frames, 2)
+        rel_pair_list = []
+        for frame_1, frame_2 in pairs:
+            pos_rel = self.possible_relation_func(frame_1, frame_2)
+            if pos_rel:
+                rel = self._extract_relation(frame_1=frame_1, frame_2=frame_2, text=doc.text, buffer_size=buffer_size, 
+                                         max_new_tokens=max_new_tokens, temperature=temperature, stream=stream, **kwrs)
+                if rel:
+                    rel_pair_list.append({'frame_1':frame_1.frame_id, 'frame_2':frame_2.frame_id})
+
+        return rel_pair_list
+
+
+
+class MultiClassRelationExtractor(RelationExtractor):
+    def __init__(self, inference_engine:InferenceEngine, prompt_template:str, possible_relation_types_func: Callable, 
+                 system_prompt:str=None, **kwrs):
+        """
+        This class extracts relations with relation types.
+        Input LLM inference engine, system prompt (optional), prompt template (with instruction, few-shot examples).
+
+        Parameters
+        ----------
+        inference_engine : InferenceEngine
+            the LLM inferencing engine object. Must implements the chat() method.
+        prompt_template : str
+            prompt template with "{{<placeholder name>}}" placeholder.
+        possible_relation_types_func : Callable
+            a function that inputs 2 frames and returns a List of possible relation types between them. 
+            If the two frames must not have relations, this function should return an empty list [].
+        system_prompt : str, Optional
+            system prompt.
+        """
+        super().__init__(inference_engine=inference_engine,
+                         prompt_template=prompt_template,
+                         system_prompt=system_prompt,
+                         **kwrs)
+        
+        if possible_relation_types_func:
+            # Check if possible_relation_types_func is a function
+            if not callable(possible_relation_types_func):
+                raise TypeError(f"Expect possible_relation_types_func as a function, received {type(possible_relation_types_func)} instead.")
+            
+            sig = inspect.signature(possible_relation_types_func)
+            # Check if frame_1, frame_2 are in input parameters
+            if len(sig.parameters) != 2:
+                raise ValueError("The possible_relation_types_func must have exactly frame_1 and frame_2 as parameters.")
+            if "frame_1" not in sig.parameters.keys():
+                raise ValueError("The possible_relation_types_func is missing frame_1 as a parameter.")
+            if "frame_2" not in sig.parameters.keys():
+                raise ValueError("The possible_relation_types_func is missing frame_2 as a parameter.")
+            # Check if output is a List
+            if sig.return_annotation not in {inspect._empty, List, List[str]}:
+                raise ValueError(f"Expect possible_relation_types_func to output a List of string, current type hint suggests {sig.return_annotation} instead.")
+
+            self.possible_relation_types_func = possible_relation_types_func
+
+        
+    def _extract_relation(self, frame_1:LLMInformationExtractionFrame, frame_2:LLMInformationExtractionFrame, 
+                      pos_rel_types:List[str], text:str, buffer_size:int=100, max_new_tokens:int=128, temperature:float=0.0, stream:bool=False, **kwrs) -> str:
+        """
+        This method inputs two frames and a ROI text, extracts the relation.
+
+        Parameters:
+        -----------
+        frame_1 : LLMInformationExtractionFrame
+            a frame
+        frame_2 : LLMInformationExtractionFrame
+            the other frame
+        pos_rel_types : List[str]
+            possible relation types.
+        text : str
+            the entire document text
+        buffer_size : int, Optional
+            the number of characters before and after the two frames in the ROI text.
+        max_new_tokens : str, Optional
+            the max number of new tokens LLM should generate. 
+        temperature : float, Optional
+            the temperature for token sampling.
+        stream : bool, Optional
+            if True, LLM generated text will be printed in terminal in real-time. 
+
+        Return : str
+            a relation type 
+        """
+        roi_text = self._get_ROI(frame_1, frame_2, text, buffer_size=buffer_size)
+        if stream:
+            print(f"\n\nROI text: \n{roi_text}\n")
+            print("Extraction:")
+
+        messages = []
+        if self.system_prompt:
+            messages.append({'role': 'system', 'content': self.system_prompt})
+
+        messages.append({'role': 'user', 'content': self._get_user_prompt(text_content={"roi_text":roi_text, 
+                                                                                        "frame_1": str(frame_1.to_dict()),
+                                                                                        "frame_2": str(frame_2.to_dict()),
+                                                                                        "pos_rel_types":str(pos_rel_types)})})
+        response = self.inference_engine.chat(
+                    messages=messages,
+                    max_new_tokens=max_new_tokens,
+                    temperature=temperature,
+                    stream=stream,
+                    **kwrs
+                )
+        
+        rel_json = self._extract_json(response)
+        if len(rel_json) > 0:
+            if "RelationType" in rel_json[0]:
+                rel = rel_json[0]["RelationType"]
+                if rel in pos_rel_types:
+                    return rel_json[0]["RelationType"]
+                else:
+                    warnings.warn(f'Extracted relation type "{rel}", which is not in the return of possible_relation_types_func: {pos_rel_types}.'+ \
+                                  'Following default, relation = "No Relation".', RuntimeWarning)
+            
+            else:
+                warnings.warn('Extractor output JSON without "RelationType" key. Following default, relation = "No Relation".', RuntimeWarning)
+        
+        else:
+            warnings.warn('Extractor did not output a JSON. Following default, relation = "No Relation".', RuntimeWarning)
+
+        return "No Relation"
+
+
+    def extract_relations(self, doc:LLMInformationExtractionDocument, buffer_size:int=100, max_new_tokens:int=128, 
+                         temperature:float=0.0, stream:bool=False, **kwrs) -> List[Dict]:
+        """
+        This method considers all combinations of two frames. Use the possible_relation_types_func to filter impossible pairs 
+        and to provide possible relation types between two frames. 
+
+        Parameters:
+        -----------
+        doc : LLMInformationExtractionDocument
+            a document with frames.
+        buffer_size : int, Optional
+            the number of characters before and after the two frames in the ROI text.
+        max_new_tokens : str, Optional
+            the max number of new tokens LLM should generate. 
+        temperature : float, Optional
+            the temperature for token sampling.
+        stream : bool, Optional
+            if True, LLM generated text will be printed in terminal in real-time. 
+
+        Return : List[Dict]
+            a list of dict with {"frame_1", "frame_2", "relation"} for all relations.
+        """
+        if not doc.has_frame():
+            raise ValueError("Input document must have frames.")
+
+        if doc.has_duplicate_frame_ids():
+            raise ValueError("All frame_ids in the input document must be unique.")
+
+        pairs = itertools.combinations(doc.frames, 2)
+        rel_pair_list = []
+        for frame_1, frame_2 in pairs:
+            pos_rel_types = self.possible_relation_types_func(frame_1, frame_2)
+            if pos_rel_types:
+                rel = self._extract_relation(frame_1=frame_1, frame_2=frame_2, pos_rel_types=pos_rel_types, text=doc.text, 
+                                         buffer_size=buffer_size, max_new_tokens=max_new_tokens, temperature=temperature, stream=stream, **kwrs)
+            
+                if rel != "No Relation":
+                    rel_pair_list.append({'frame_1':frame_1.frame_id, 'frame_2':frame_2.frame_id, "relation":rel})
+
+        return rel_pair_list

{llm_ie-0.1.7.dist-info → llm_ie-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: llm-ie
-Version: 0.1.7
+Version: 0.2.0
 Summary: An LLM-powered tool that transforms everyday language into robust information extraction pipelines.
 License: MIT
 Author: Enshuo (David) Hsu
@@ -25,11 +25,14 @@ An LLM-powered tool that transforms everyday language into robust information ex
 - [Prerequisite](#prerequisite)
 - [Installation](#installation)
 - [Quick Start](#quick-start)
+- [Examples](#examples)
 - [User Guide](#user-guide)
     - [LLM Inference Engine](#llm-inference-engine)
     - [Prompt Template](#prompt-template)
     - [Prompt Editor](#prompt-editor)
     - [Extractor](#extractor)
+        - [FrameExtractor](#frameextractor)
+        - [RelationExtractor](#relationextractor)
 
 ## Overview
 LLM-IE is a toolkit that provides robust information extraction utilities for frame-based information extraction. Since prompt design has a significant impact on generative information extraction with LLMs, it also provides a built-in LLM editor to help with prompt writing. The flowchart below demonstrates the workflow starting from a casual language request.
@@ -206,6 +209,10 @@ for frame in frames:
 doc.save("<your filename>.llmie")
 ```
 
+## Examples
+  - [Write prompt templates with AI editors](demo/prompt_template_writing.ipynb)
+  - [NER + RE for Drug, Strength, Frequency](demo/medication_relation_extraction.ipynb)
+
 ## User Guide
 This package is comprised of some key classes:
 - LLM Inference Engine
@@ -547,12 +554,25 @@ Recommendations:
 After a few iterations of revision, we will have a high-quality prompt template for the information extraction pipeline. 
 
 ### Extractor
-An extractor implements a prompting method for information extraction. The ```BasicFrameExtractor``` directly prompts LLM to generate a list of dictionaries. Each dictionary is then post-processed into a frame. The ```ReviewFrameExtractor``` is based on the ```BasicFrameExtractor``` but adds a review step after the initial extraction to boost sensitivity and improve performance. ```SentenceFrameExtractor``` gives LLM the entire document upfront as a reference, then prompts LLM sentence by sentence and collects per-sentence outputs. To learn about an extractor, use the class method ```get_prompt_guide()``` to print out the prompt guide. 
+An extractor implements a prompting method for information extraction. There are two extractor families: ```FrameExtractor``` and ```RelationExtractor```. 
+The ```FrameExtractor``` extracts named entities and entity attributes ("frame"). The ```RelationExtractor``` extracts the relation (and relation types) between frames. 
+
+#### FrameExtractor
+The ```BasicFrameExtractor``` directly prompts LLM to generate a list of dictionaries. Each dictionary is then post-processed into a frame. The ```ReviewFrameExtractor``` is based on the ```BasicFrameExtractor``` but adds a review step after the initial extraction to boost sensitivity and improve performance. ```SentenceFrameExtractor``` gives LLM the entire document upfront as a reference, then prompts LLM sentence by sentence and collects per-sentence outputs. To learn about an extractor, use the class method ```get_prompt_guide()``` to print out the prompt guide. 
 
 <details>
 <summary>BasicFrameExtractor</summary>
 
-The ```BasicFrameExtractor``` directly prompts LLM to generate a list of dictionaries. Each dictionary is then post-processed into a frame.
+The ```BasicFrameExtractor``` directly prompts LLM to generate a list of dictionaries. Each dictionary is then post-processed into a frame. The ```text_content``` holds the input text as a string, or as a dictionary (if prompt template has multiple input placeholders). The ```entity_key``` defines which JSON key should be used as entity text. It must be consistent with the prompt template. 
+
+```python
+from llm_ie.extractors import BasicFrameExtractor
+
+extractor = BasicFrameExtractor(llm, prompt_temp)
+frames = extractor.extract_frames(text_content=text, entity_key="Diagnosis", stream=True)
+```
+
+Use the ```get_prompt_guide()``` method to inspect the prompt template guideline for ```BasicFrameExtractor```. 
 
 ```python
 from llm_ie.extractors import BasicFrameExtractor
@@ -630,15 +650,202 @@ frames = extractor.extract_frames(text_content=text, entity_key="Diagnosis", str
 <details>
 <summary>SentenceFrameExtractor</summary>
 
-The ```SentenceFrameExtractor``` instructs the LLM to extract sentence by sentence. The reason is to ensure the accuracy of frame spans. It also prevents LLMs from overseeing sections/ sentences. Empirically, this extractor results in better sensitivity than the ```BasicFrameExtractor``` in complex tasks. 
+The ```SentenceFrameExtractor``` instructs the LLM to extract sentence by sentence. The reason is to ensure the accuracy of frame spans. It also prevents LLMs from overseeing sections/ sentences. Empirically, this extractor results in better recall than the ```BasicFrameExtractor``` in complex tasks. 
+
+The ```multi_turn``` parameter specifies multi-turn conversation for prompting. If True, sentences and LLM outputs will be appended to the input message and carry-over. If False, only the current sentence is prompted. For LLM inference engines that supports prompt cache (e.g., Llama.Cpp, Ollama), use multi-turn conversation prompting can better utilize the KV caching and results in faster inferencing. But for vLLM with [Automatic Prefix Caching (APC)](https://docs.vllm.ai/en/latest/automatic_prefix_caching/apc.html), multi-turn conversation is not necessary.
 
 ```python
 from llm_ie.extractors import SentenceFrameExtractor
 
 extractor = SentenceFrameExtractor(llm, prompt_temp)
-frames = extractor.extract_frames(text_content=text, entity_key="Diagnosis", stream=True)
+frames = extractor.extract_frames(text_content=text, entity_key="Diagnosis", multi_turn=True, stream=True)
 ```
 </details>
 
+#### RelationExtractor
+Relation extractors prompt LLM with combinations of two frames from a document (```LLMInformationExtractionDocument```) and extract relations.
+The ```BinaryRelationExtractor``` extracts binary relations (yes/no) between two frames. The ```MultiClassRelationExtractor``` extracts relations and assign relation types ("multi-class"). 
+
+An important feature of the relation extractors is that users are required to define a ```possible_relation_func``` or ```possible_relation_types_func``` function for the extractors. The reason is, there are too many possible combinations of two frames (N choose 2 combinations). The ```possible_relation_func``` helps rule out impossible combinations and therefore, reduce the LLM inferencing burden.
+
+<details>
+<summary>BinaryRelationExtractor</summary>
+
+Use the get_prompt_guide() method to inspect the prompt template guideline for BinaryRelationExtractor.
+```python
+from llm_ie.extractors import BinaryRelationExtractor
+
+print(BinaryRelationExtractor.get_prompt_guide())
+```
+
+```
+Prompt template design:
+    1. Task description (mention binary relation extraction and ROI)
+    2. Schema definition (defines relation)
+    3. Output format definition (must use the key "Relation")
+    4. Hints
+    5. Input placeholders (must include "roi_text", "frame_1", and "frame_2" placeholders)
 
 
+Example:
+
+    # Task description
+    This is a binary relation extraction task. Given a region of interest (ROI) text and two entities from a medical note, indicate the relation existence between the two entities.
+
+    # Schema definition
+        True: if there is a relationship between a medication name (one of the entities) and its strength or frequency (the other entity).
+        False: Otherwise.
+
+    # Output format definition
+    Your output should follow the JSON format:
+    {"Relation": "<True or False>"}
+
+    I am only interested in the content between []. Do not explain your answer. 
+
+    # Hints
+        1. Your input always contains one medication entity and 1) one strength entity or 2) one frequency entity.
+        2. Pay attention to the medication entity and see if the strength or frequency is for it.
+        3. If the strength or frequency is for another medication, output False. 
+        4. If the strength or frequency is for the same medication but at a different location (span), output False.
+
+    # Input placeholders
+    ROI Text with the two entities annotated with <entity_1> and <entity_2>:
+    "{{roi_text}}"
+
+    Entity 1 full information:
+    {{frame_1}}
+
+    Entity 2 full information:
+    {{frame_2}}
+```
+
+As an example, we define the ```possible_relation_func``` function:
+  - if the two frames are > 500 characters apart, we assume no relation (False)
+  - if the two frames are "Medication" and "Strength", or "Medication" and "Frequency", there could be relations (True)
+
+```python
+def possible_relation_func(frame_1, frame_2) -> bool:
+    """
+    This function pre-process two frames and outputs a bool indicating whether the two frames could be related.
+    """
+    # if the distance between the two frames are > 500 characters, assume no relation.
+    if abs(frame_1.start - frame_2.start) > 500:
+        return False
+    
+    # if the entity types are "Medication" and "Strength", there could be relations.
+    if (frame_1.attr["entity_type"] == "Medication" and frame_2.attr["entity_type"] == "Strength") or \
+        (frame_2.attr["entity_type"] == "Medication" and frame_1.attr["entity_type"] == "Strength"):
+        return True
+    
+    # if the entity types are "Medication" and "Frequency", there could be relations.
+    if (frame_1.attr["entity_type"] == "Medication" and frame_2.attr["entity_type"] == "Frequency") or \
+        (frame_2.attr["entity_type"] == "Medication" and frame_1.attr["entity_type"] == "Frequency"):
+        return True
+
+    # Otherwise, no relation.
+    return False
+```
+
+In the ```BinaryRelationExtractor``` constructor, we pass in the prompt template and ```possible_relation_func```.
+
+```python
+from llm_ie.extractors import BinaryRelationExtractor
+
+extractor = BinaryRelationExtractor(llm, prompt_template=prompt_template, possible_relation_func=possible_relation_func)
+relations = extractor.extract_relations(doc, stream=True)
+```
+
+</details>
+
+
+<details>
+<summary>MultiClassRelationExtractor</summary>
+
+The main difference from ```BinaryRelationExtractor``` is that the ```MultiClassRelationExtractor``` allows specifying relation types. The prompt template guideline has an additional placeholder for possible relation types ```{{pos_rel_types}}```. 
+
+```python
+print(MultiClassRelationExtractor.get_prompt_guide())
+```
+
+```
+Prompt template design:
+    1. Task description (mention multi-class relation extraction and ROI)
+    2. Schema definition (defines relation types)
+    3. Output format definition (must use the key "RelationType")
+    4. Input placeholders (must include "roi_text", "frame_1", and "frame_2" placeholders)
+
+
+Example:
+
+    # Task description
+    This is a multi-class relation extraction task. Given a region of interest (ROI) text and two frames from a medical note, classify the relation types between the two frames. 
+
+    # Schema definition
+        Strength-Drug: this is a relationship between the drug strength and its name. 
+        Dosage-Drug: this is a relationship between the drug dosage and its name.
+        Duration-Drug: this is a relationship between a drug duration and its name.
+        Frequency-Drug: this is a relationship between a drug frequency and its name.
+        Form-Drug: this is a relationship between a drug form and its name.
+        Route-Drug: this is a relationship between the route of administration for a drug and its name.
+        Reason-Drug: this is a relationship between the reason for which a drug was administered (e.g., symptoms, diseases, etc.) and a drug name.
+        ADE-Drug: this is a relationship between an adverse drug event (ADE) and a drug name.
+
+    # Output format definition
+    Choose one of the relation types listed below or choose "No Relation":
+    {{pos_rel_types}}
+
+    Your output should follow the JSON format:
+    {"RelationType": "<relation type or No Relation>"}
+
+    I am only interested in the content between []. Do not explain your answer. 
+
+    # Hints
+        1. Your input always contains one medication entity and 1) one strength entity or 2) one frequency entity.
+        2. Pay attention to the medication entity and see if the strength or frequency is for it.
+        3. If the strength or frequency is for another medication, output "No Relation". 
+        4. If the strength or frequency is for the same medication but at a different location (span), output "No Relation".
+
+    # Input placeholders
+    ROI Text with the two entities annotated with <entity_1> and <entity_2>:
+    "{{roi_text}}"
+
+    Entity 1 full information:
+    {{frame_1}}
+
+    Entity 2 full information:
+    {{frame_2}}
+```
+
+As an example, we define the ```possible_relation_types_func``` :
+  - if the two frames are > 500 characters apart, we assume "No Relation" (output [])
+  - if the two frames are "Medication" and "Strength", the only possible relation types are "Strength-Drug" or "No Relation"
+  - if the two frames are "Medication" and "Frequency", the only possible relation types are "Frequency-Drug" or "No Relation"
+
+```python 
+def possible_relation_types_func(frame_1, frame_2) -> List[str]:
+    # If the two frames are > 500 characters apart, we assume "No Relation"
+    if abs(frame_1.start - frame_2.start) > 500:
+        return []
+    
+    # If the two frames are "Medication" and "Strength", the only possible relation types are "Strength-Drug" or "No Relation"
+    if (frame_1.attr["entity_type"] == "Medication" and frame_2.attr["entity_type"] == "Strength") or \
+        (frame_2.attr["entity_type"] == "Medication" and frame_1.attr["entity_type"] == "Strength"):
+        return ['Strength-Drug']
+    
+    # If the two frames are "Medication" and "Frequency", the only possible relation types are "Frequency-Drug" or "No Relation"
+    if (frame_1.attr["entity_type"] == "Medication" and frame_2.attr["entity_type"] == "Frequency") or \
+        (frame_2.attr["entity_type"] == "Medication" and frame_1.attr["entity_type"] == "Frequency"):
+        return ['Frequency-Drug']
+
+    return []
+```
+
+
+```python
+from llm_ie.extractors import MultiClassRelationExtractor
+
+extractor = MultiClassRelationExtractor(llm, prompt_template=re_prompt_template, possible_relation_types_func=possible_relation_types_func)
+relations = extractor.extract_relations(doc, stream=True)
+```
+
+</details>

{llm_ie-0.1.7.dist-info → llm_ie-0.2.0.dist-info}/RECORD

@@ -2,12 +2,14 @@ llm_ie/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 llm_ie/asset/PromptEditor_prompts/comment.txt,sha256=C_lxx-dlOlFJ__jkHKosZ8HsNAeV1aowh2B36nIipBY,159
 llm_ie/asset/PromptEditor_prompts/rewrite.txt,sha256=bYLOix7DUBlcWv-Q0JZ5kDnZ9OEXBt_AGDN0TydLB8o,191
 llm_ie/asset/prompt_guide/BasicFrameExtractor_prompt_guide.txt,sha256=XbnU8byLGGUA3A3lT0bb2Hw-ggzhcqD3ZuKzduod2ww,1944
+llm_ie/asset/prompt_guide/BinaryRelationExtractor_prompt_guide.txt,sha256=z9Xg0fdFbVVwnTYcUTcAUvEIWhF075W8qGxN-Vj7xdo,1548
+llm_ie/asset/prompt_guide/MultiClassRelationExtractor_prompt_guide.txt,sha256=D5DphUHw8SUERUVdcIjUynuTmYJa6-PwBlF7FzxNsvQ,2276
 llm_ie/asset/prompt_guide/ReviewFrameExtractor_prompt_guide.txt,sha256=XbnU8byLGGUA3A3lT0bb2Hw-ggzhcqD3ZuKzduod2ww,1944
 llm_ie/asset/prompt_guide/SentenceFrameExtractor_prompt_guide.txt,sha256=8nj9OLPJMtr9Soi5JU3Xk-HC7pKNoI54xA_A4u7I5j4,2620
-llm_ie/data_types.py,sha256=MnpyXFviFWhxeC5mqbaPdAxGx6vV_PhnUIFfUamq3D8,6687
+llm_ie/data_types.py,sha256=2RKP4wXDuku-Tn4s8uzzUFavG1fZ2e47SaY8oL57LsI,10923
 llm_ie/engines.py,sha256=m9ytGUX61jEy9SmVHbb90mrfGMAwC6dV-v7Jke1U7Ho,9296
-llm_ie/extractors.py,sha256=PfcUhmU_LfVFIfI5v3C7DzGAFF0xEPDdLUnwKHYnUyg,24125
+llm_ie/extractors.py,sha256=i0m8uFaKXiVY1ucjvzbUFbV1slPYfZ3EGOZrolnFVHA,44079
 llm_ie/prompt_editor.py,sha256=dbu7A3O7O7Iw2v-xCgrTFH1-wTLAGf4SHDqdeS-He2Q,1869
-llm_ie-0.1.7.dist-info/METADATA,sha256=wMAToHKL1K3hZP-xONBdEw3sy56stKYThfn3NqbfZ34,29712
-llm_ie-0.1.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-llm_ie-0.1.7.dist-info/RECORD,,
+llm_ie-0.2.0.dist-info/METADATA,sha256=9CPC3OAd2J0nROZ7z8DI7lvGKOO2H2uAnVQv_YDFItg,40052
+llm_ie-0.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+llm_ie-0.2.0.dist-info/RECORD,,