hjxdl 0.1.13__py3-none-any.whl → 0.1.15__py3-none-any.whl

hdl/jupyfuncs/llm/extract.py

@@ -0,0 +1,123 @@
+import pdfplumber
+import pytesseract
+from PIL import Image
+import pandas as pd
+import io
+from spire.doc import Document
+from spire.doc.common import *
+# from ..path.glob import (
+#     get_current_dir,
+#     get_files
+# )
+
+
+class DocExtractor():
+    def __init__(
+        self,
+        doc_files: list,
+        lang: str = "chi_sim"
+    ) -> None:
+        self.doc_files = doc_files
+        self.lang = lang
+
+    @classmethod
+    def text_from_doc(
+        doc_path
+    ):
+        document = Document()
+        # Load a Word document
+        document.LoadFromFile(doc_path)
+        document_text = document.GetText()
+        return document_text
+
+    @staticmethod
+    def text_from_plain(
+        txt_path
+    ):
+        with open(txt_path, "r") as f:
+            text = f.read()
+        return text
+
+    @staticmethod
+    def extract_text_from_image(
+        image: Image.Image,
+    ) -> str:
+        return pytesseract.image_to_string(image, lang=self.lang)
+
+    @staticmethod
+    def is_within_bbox(
+        bbox1, bbox2
+    ):
+        """Check if bbox1 is within bbox2."""
+        return bbox1[0] >= bbox2[0] and bbox1[1] >= bbox2[1] and bbox1[2] <= bbox2[2] and bbox1[3] <= bbox2[3]
+
+    def text_tables_from_pdf(
+        self,
+        pdf_path,
+        table_from_pic: bool = False
+    ):
+        all_tables = []
+        all_texts = []
+        with pdfplumber.open(pdf_path) as pdf:
+            for page_number, page in enumerate(pdf.pages):
+                tables = page.find_tables()
+                page_text = page.extract_text(x_tolerance=0.1, y_tolerance=0.1) or ''
+                page_text_lines = page_text.split('\n')
+
+                # Extract tables
+                if tables:
+                    for table in tables:
+                        if table and len(table.extract()) > 1:
+                            table_data = table.extract()
+                            df = pd.DataFrame(table_data[1:], columns=table_data[0])
+                            df['Page'] = page_number + 1  # 添加页码信息
+                            all_tables.append(df)
+
+                # Get bounding boxes for tables
+                table_bboxes = [table.bbox for table in tables]
+
+                # Filter out text within table bounding boxes
+                non_table_text = []
+                for char in page.chars:
+                    char_bbox = (char['x0'], char['top'], char['x1'], char['bottom'])
+                    if not any(self.is_within_bbox(char_bbox, table_bbox) for table_bbox in table_bboxes):
+                        non_table_text.append(char['text'])
+                remaining_text = ''.join(non_table_text).strip()
+                if remaining_text:
+                    all_texts.append(remaining_text)
+
+                # Extract tables from images if specified
+                if table_from_pic:
+                    for img in page.images:
+                        try:
+                            x0, top, x1, bottom = img["x0"], img["top"], img["x1"], img["bottom"]
+                            if x0 < 0 or top < 0 or x1 > page.width or bottom > page.height:
+                                print(f"Skipping image with invalid bounds on page {page_number + 1}")
+                                continue
+
+                            cropped_image = page.within_bbox((x0, top, x1, bottom)).to_image()
+                            img_bytes = io.BytesIO()
+                            cropped_image.save(img_bytes, format='PNG')
+                            img_bytes.seek(0)
+                            pil_image = Image.open(img_bytes)
+
+                            ocr_text = self.extract_text_from_image(pil_image, lang=self.lang)
+
+                            table = [line.split() for line in ocr_text.split('\n') if line.strip()]
+
+                            if table:
+                                num_columns = max(len(row) for row in table)
+                                for row in table:
+                                    if len(row) != num_columns:
+                                        row.extend([''] * (num_columns - len(row)))
+
+                                df = pd.DataFrame(table[1:], columns=table[0])
+                                df['Page'] = page_number + 1
+                                all_tables.append(df)
+                        except Exception as e:
+                            print(f"Error processing image on page {page_number + 1}: {e}")
+
+        if all_tables:
+            return all_texts, all_tables
+        else:
+            return all_texts, [pd.DataFrame()]

hdl/jupyfuncs/llm/openapi.py

@@ -0,0 +1,94 @@
+from openai import OpenAI
+# import traceback
+
+
+def chat_oai_stream(
+    base_url="http://127.0.0.1:8000/v1",
+    api_key="dummy_key",
+    model="/data/models/Qwen-7B-Chat-Int4",
+    prompt="Who are you?",
+    *args,
+    **kwargs
+):
+    """Chat with OpenAI's GPT-3 model using the specified parameters.
+
+    Args:
+        base_url (str): The base URL for the OpenAI API. Default is "http://127.0.0.1:8000/v1".
+        api_key (str): The API key for accessing the OpenAI API. Default is "dummy_key".
+        model (str): The model ID to use for the chat. Default is "/data/models/Qwen-7B-Chat-Int4".
+        prompt (str): The initial prompt for the chat conversation.
+
+    Yields:
+        str: The generated content from the chat conversation.
+
+    """
+    client = OpenAI(
+        base_url=base_url,
+        api_key=api_key,
+    )
+    response = client.chat.completions.create(
+        model=model,
+        messages=[{
+            "role": "user",
+            "content": prompt
+        }],
+        stream=True,
+        *args,
+        **kwargs
+    )
+
+    for chunk in response:
+        content = chunk.choices[0].delta.content
+        yield content
+
+
+def chat_oai_invoke(
+    base_url="http://127.0.0.1:8000/v1",
+    api_key="dummy_key",
+    model="/data/models/Qwen-7B-Chat-Int4",
+    prompt="Who are you?",
+):
+    """Invoke OpenAI chat API to generate a response based on the given prompt.
+
+    Args:
+        base_url (str): The base URL of the OpenAI API. Default is "http://127.0.0.1:8000/v1".
+        api_key (str): The API key for accessing the OpenAI API. Default is "dummy_key".
+        model (str): The model to use for generating the response. Default is "/data/models/Qwen-7B-Chat-Int4".
+        prompt (str): The prompt message to start the conversation. Default is "Who are you?".
+
+    Returns:
+        str: The response generated by the OpenAI chat API based on the prompt.
+    """
+    client = OpenAI(
+        base_url=base_url,
+        api_key=api_key,
+    )
+    response = client.chat.completions.create(
+        model=model,
+        messages=[{
+            "role": "user",
+            "content": prompt
+        }],
+        stream=False,
+        *args,
+        **kwargs
+    )
+
+    return response.choices[0].message.content
+
+# client = OpenAI(
+#     base_url="http://127.0.0.1:8000/v1",
+#     api_key="dummy_key"  # 使用虚拟的API Key
+# )
+
+
+# response = client.chat.completions.create(
+#     model="/data/models/Qwen-7B-Chat-Int4",
+#     messages=[{"role": "user", "content": "给我讲个鬼故事"}],
+#     stream=True
+# )
+# for chunk in response:
+#     content = chunk.choices[0].delta.content
+#     if content:
+#         print(content, end='', flush=True)
+# print()

hdl/jupyfuncs/network/proxy.py

@@ -0,0 +1,20 @@
+import requests
+
+
+def get_proxies(
+    pool_server='http://172.20.0.9:5010',
+    https=False
+):
+    http_proxy = requests.get(f"{pool_server}/get/").json().get("proxy")
+    https_proxy = requests.get(f"{pool_server}/get/?type=https").json().get("proxy")
+    if https:
+        proxy_dict = {
+            'http': f'http://{https_proxy}',
+            'https': f'https://{https_proxy}'
+        }
+    else:
+        proxy_dict = {
+            'http': f'http://{http_proxy}',
+            'https': f'https://{http_proxy}'
+        }
+    return proxy_dict

hdl/jupyfuncs/path/glob.py

@@ -0,0 +1,285 @@
+import os
+import typing as t
+import inspect
+import fnmatch
+import linecache
+import time
+import gc
+import psutil
+from os import path as osp
+import pathlib
+import sys
+import importlib
+import subprocess
+import re
+
+import multiprocess as mp
+
+import importlib.resources as pkg_resources
+import json
+
+
+def in_jupyter():
+    """Check if the code is running in a Jupyter notebook.
+    
+        Returns:
+            bool: True if running in Jupyter notebook, False otherwise.
+    """
+
+    which = True if 'ipykernel_launcher.py' in sys.argv[0] else False
+    return which
+
+
+def in_docker():
+    """Check if the code is running inside a Docker container.
+    
+        Returns:
+            bool: True if running inside a Docker container, False otherwise.
+    """
+    return osp.exists('/.dockerenv')
+
+
+def get_files(
+    dir_path,
+    file_types: list = ["txt"]
+):
+    """Get a list of files with specific file extensions in the given directory path.
+    
+    Args:
+        dir_path (str): The path to the target directory.
+    
+    Returns:
+        list: A list of absolute file paths that have file extensions such as .md, .doc, .docx, .pdf, .csv, or .txt.
+    """
+    # args：dir_path，目标文件夹路径
+    file_list = []
+    for filepath, dirnames, filenames in os.walk(dir_path):
+        # os.walk 函数将递归遍历指定文件夹
+        filenames = [f for f in filenames if not f[0] == '.']
+        dirnames[:] = [d for d in dirnames if not d[0] == '.'] 
+        for filename in filenames:
+            # 通过后缀名判断文件类型是否满足要求
+            if filename.endswith(file_types):
+                # 如果满足要求，将其绝对路径加入到结果列表
+                file_list.append(os.path.join(filepath, filename))
+    return file_list
+
+
+def get_dataset_file(filename):
+    """Get dataset file.
+    
+    Args:
+        filename (str): The name of the dataset file.
+    
+    Returns:
+        dict: The data loaded from the dataset file.
+    """
+    with pkg_resources.path('jupyfuncs.datasets', filename) as file_path:
+        with open(file_path, 'r') as f:
+            data = json.load(f)
+    return data
+
+
+def recursive_glob(treeroot, pattern):
+    """Recursively searches for files matching a specified pattern starting from the given directory.
+    
+    Args:
+        treeroot (str): The root directory to start the search from.
+        pattern (str): The pattern to match the files against.
+    
+    Returns:
+        list: A list of file paths that match the specified pattern.
+    """
+    results = []
+    for base, dirs, files in os.walk(treeroot):
+        goodfiles = fnmatch.filter(files, pattern)
+        results.extend(os.path.join(base, f) for f in goodfiles)
+    return results
+
+
+def makedirs(path: str, isfile: bool = False) -> None:
+    """Creates a directory given a path to either a directory or file.
+    If a directory is provided, creates that directory. If a file is provided (i.e. :code:`isfile == True`),
+    creates the parent directory for that file.
+
+
+    Args:
+        path (str): Path to a directory or file. 
+        isfile (bool, optional): Whether the provided path is a directory or file.Defaults to False.
+    """    
+    if isfile:
+        path = os.path.dirname(path)
+    if path != '':
+        os.makedirs(path, exist_ok=True)
+
+
+def get_current_dir():
+    """Return the current directory path."""
+    return os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+
+
+def get_num_lines(file):
+    """Get the number of lines in a file.
+    
+    Args:
+        file (str): The path to the file.
+    
+    Returns:
+        int: The number of lines in the file.
+    """
+    num_lines = subprocess.check_output(
+        ['wc', '-l', file]
+    ).split()[0]
+    return int(num_lines)
+
+
+
+
+def chunkify_file(
+    fname,
+    size=1024 * 1024 * 1000,
+    skiplines=-1
+):
+    """
+    function to divide a large text file into chunks each having size ~= size so that the chunks are line aligned
+
+    Params : 
+        fname : path to the file to be chunked
+        size : size of each chink is ~> this
+        skiplines : number of lines in the begining to skip, -1 means don't skip any lines
+    Returns : 
+        start and end position of chunks in Bytes
+    """
+    chunks = []
+    fileEnd = os.path.getsize(fname)
+    with open(fname, "rb") as f:
+        if(skiplines > 0):
+            for i in range(skiplines):
+                f.readline()
+
+        chunkEnd = f.tell()
+        count = 0
+        while True:
+            chunkStart = chunkEnd
+            f.seek(f.tell() + size, os.SEEK_SET)
+            f.readline()  # make this chunk line aligned
+            chunkEnd = f.tell()
+            chunks.append((chunkStart, chunkEnd - chunkStart, fname))
+            count += 1
+
+            if chunkEnd > fileEnd:
+                break
+    return chunks
+
+
+def parallel_apply_line_by_line_chunk(chunk_data):
+    """
+    function to apply a function to each line in a chunk
+
+    Params :
+        chunk_data : the data for this chunk 
+    Returns :
+        list of the non-None results for this chunk
+    """
+    chunk_start, chunk_size, file_path, func_apply = chunk_data[:4]
+    func_args = chunk_data[4:]
+
+    # t1 = time.time()
+    chunk_res = []
+    with open(file_path, "rb") as f:
+        f.seek(chunk_start)
+        cont = f.read(chunk_size).decode(encoding='utf-8')
+        lines = cont.splitlines()
+
+        for _, line in enumerate(lines):
+            ret = func_apply(line, *func_args)
+            if(ret != None):
+                chunk_res.append(ret)
+    return chunk_res
+
+
+def parallel_apply_line_by_line(
+    input_file_path,
+    chunk_size_factor,
+    num_procs,
+    skiplines,
+    func_apply,
+    func_args,
+    fout=None
+):
+    """
+    function to apply a supplied function line by line in parallel
+
+    Params :
+        input_file_path : path to input file
+        chunk_size_factor : size of 1 chunk in MB
+        num_procs : number of parallel processes to spawn, max used is num of available cores - 1
+        skiplines : number of top lines to skip while processing
+        func_apply : a function which expects a line and outputs None for lines we don't want processed
+        func_args : arguments to function func_apply
+        fout : do we want to output the processed lines to a file
+    Returns :
+        list of the non-None results obtained be processing each line
+    """
+    num_parallel = min(num_procs, psutil.cpu_count()) - 1
+
+    jobs = chunkify_file(input_file_path, 1024 * 1024 * chunk_size_factor, skiplines)
+
+    jobs = [list(x) + [func_apply] + func_args for x in jobs]
+
+    print("Starting the parallel pool for {} jobs ".format(len(jobs)))
+
+    lines_counter = 0
+
+    pool = mp.Pool(num_parallel, maxtasksperchild=1000)  # maxtaskperchild - if not supplied some weird happend and memory blows as the processes keep on lingering
+
+    outputs = []
+    for i in range(0, len(jobs), num_parallel):
+        print("Chunk start = ", i)
+        t1 = time.time()
+        chunk_outputs = pool.map(
+            parallel_apply_line_by_line_chunk,
+            jobs[i: i + num_parallel]
+        )
+
+        for i, subl in enumerate(chunk_outputs):
+            for x in subl:
+                if(fout != None):
+                    print(x, file=fout)
+                else:
+                    outputs.append(x)
+                lines_counter += 1
+        del(chunk_outputs)
+        gc.collect()
+        print("All Done in time ", time.time() - t1)
+
+    print("Total lines we have = {}".format(lines_counter))
+
+    pool.close()
+    pool.terminate()
+    return outputs
+
+
+def get_func_from_dir(score_dir: str) -> t.Tuple[t.Callable, str]:
+    """Get function and mode from directory.
+    
+    Args:
+        score_dir (str): The directory path containing the function file.
+        
+    Returns:
+        Tuple[Callable, str]: A tuple containing the main function and the mode.
+    """
+    if score_dir.endswith('.py'):
+        func_dir = pathlib.Path(score_dir).parent.resolve()
+        file_name = pathlib.Path(score_dir).stem
+    else:
+        func_dir = osp.abspath(score_dir)
+        file_name = "main"
+
+    sys.path.append(func_dir)
+    module = importlib.import_module(file_name) 
+    try:
+        mode = module.MODE
+    except Exception as _:
+        mode = 'batch'
+    return module.main, mode 

hdl/jupyfuncs/path/strings.py

@@ -0,0 +1,65 @@
+import re
+import subprocess
+
+
+def get_n_tokens(
+    paragraph,
+    model: str = None
+):
+    """Get the number of tokens in a paragraph using a specified model.
+    
+    Args:
+        paragraph (str): The input paragraph to tokenize.
+        model (str): The name of the model to use for tokenization. If None, a default CJK tokenization will be used.
+    
+    Returns:
+        int: The number of tokens in the paragraph based on the specified model or default CJK tokenization.
+    """
+    if model is None:
+        cjk_regex = re.compile(u'[\u1100-\uFFFDh]+?')
+        trimed_cjk = cjk_regex.sub( ' a ', paragraph, 0)
+        return len(trimed_cjk.split())
+    else:
+        import tiktoken
+        encoding = tiktoken.encoding_for_model(model)
+        num_tokens = len(encoding.encode(paragraph))
+        return num_tokens
+
+
+def str_from_line(file, line, split=False):
+    """Retrieve a specific line from a file and process it.
+    
+    Args:
+        file (str): The path to the file.
+        line (int): The line number to retrieve (starting from 0).
+        split (bool, optional): If True, split the line by space or tab and return the first element. Defaults to False.
+    
+    Returns:
+        str: The content of the specified line from the file.
+    """
+    smi = subprocess.check_output(
+        # ['sed','-n', f'{str(i+1)}p', file]
+        ["sed", f"{str(line + 1)}q;d", file]
+    )
+    if isinstance(smi, bytes):
+        smi = smi.decode().strip()
+    if split:
+        if ' ' or '\t' in smi:
+            smi = smi.split()[0]
+    return smi
+
+
+def splitted_strs_from_line(
+    file: str,
+    idx: int
+) -> list:
+    """Return a list of strings obtained by splitting the line at the specified index from the given file.
+    
+        Args:
+            file (str): The file path.
+            idx (int): The index of the line to split.
+    
+        Returns:
+            List: A list of strings obtained by splitting the line at the specified index.
+    """
+    return str_from_line(file, idx).split()

hdl/jupyfuncs/show/pbar.py

@@ -0,0 +1,50 @@
+import sys
+from os import path as osp
+from IPython.core.display import HTML
+
+__all__ = [
+    'in_jupyter',
+    'tqdm',
+    'trange',
+    'tnrange',
+    'NO_WHITE',
+]
+
+
+NO_WHITE = HTML("""
+    <style>
+    .jp-OutputArea-prompt:empty {
+    padding: 0;
+    border: 0;
+    }
+    </style>
+    """)
+
+
+def in_jupyter():
+    """Check if the code is running in a Jupyter notebook.
+    
+        Returns:
+            bool: True if running in Jupyter notebook, False otherwise.
+    """
+
+    which = True if 'ipykernel_launcher.py' in sys.argv[0] else False
+    return which
+
+def in_docker():
+    """Check if the code is running inside a Docker container.
+    
+        Returns:
+            bool: True if running inside a Docker container, False otherwise.
+    """
+    return osp.exists('/.dockerenv')
+
+
+if in_jupyter():
+    from tqdm.notebook import tqdm
+    from tqdm.notebook import trange
+    from tqdm.notebook import tnrange
+else:
+    from tqdm import tqdm
+    from tqdm import trange
+    from tqdm import tnrange