jwf-common-python 1.0.0__py3-none-any.whl

jwf_common_python/language_utils/core.py

@@ -0,0 +1,203 @@
+"""纯文本和词表处理，不读写文件，不修改输入，不配置全局日志。"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from collections.abc import Iterable, Iterator
+from urllib.parse import urlsplit
+
+WORD_PATTERN = re.compile(r"[A-Za-z]+")
+
+
+def _clean_words(words: Iterable[str]) -> Iterator[str]:
+    if isinstance(words, (str, bytes)):
+        raise TypeError("词表必须为字符串的可迭代对象，请将多行文本先 splitlines()")
+    for word in words:
+        if not isinstance(word, str):
+            raise TypeError("词表中的元素必须为字符串")
+        word = word.strip()
+        if word:
+            yield word
+
+
+def _text_chunks(text: str | Iterable[str]) -> Iterator[str]:
+    if isinstance(text, str):
+        yield text
+    else:
+        for chunk in text:
+            if not isinstance(chunk, str):
+                raise TypeError("文本内容必须为字符串")
+            yield chunk
+
+
+def filter_words(
+    words: Iterable[str],
+    *,
+    exclude_words: Iterable[str] = (),
+    known_words: Iterable[str] = (),
+    case_sensitive: bool = False,
+    min_length: int = 1,
+    unique: bool = False,
+) -> list[str]:
+    """排除无效/已知词，保留原文拼写和顺序，默认保留重复项。
+
+    默认以 casefold 忽略大小写比较；unique=True 按同一规则去重。
+    空白词忽略，长度按去除首尾空白后的字符串计算。
+    """
+    if (
+        isinstance(min_length, bool)
+        or not isinstance(min_length, int)
+        or min_length < 1
+    ):
+        raise ValueError("min_length 必须为正整数")
+    key = (lambda word: word) if case_sensitive else str.casefold
+    excluded = {
+        key(word)
+        for group in (exclude_words, known_words)
+        for word in _clean_words(group)
+    }
+    seen = set()
+    result = []
+    for word in _clean_words(words):
+        normalized = key(word)
+        if (
+            len(word) >= min_length
+            and normalized not in excluded
+            and (not unique or normalized not in seen)
+        ):
+            result.append(word)
+            if unique:
+                seen.add(normalized)
+    return result
+
+
+def extract_words(text: str | Iterable[str], **options) -> list[str]:
+    """提取英文 A-Z/a-z 片段，保留原分词规则，再通过 filter_words 过滤。
+
+    连字符、撇号、数字等均为分隔符，不进行词典查验或词形还原。
+    """
+    words = (
+        match.group()
+        for chunk in _text_chunks(text)
+        for match in WORD_PATTERN.finditer(chunk)
+    )
+    return filter_words(words, **options)
+
+
+def extract_chapter(
+    text: str, start_title: str, end_title: str | None = None, *, occurrence: int = -1
+) -> str:
+    """按完整标题行提取正文，不包含标题。
+
+    去除标题行首尾空白后匹配；默认选择最后一次出现的起始标题，跳过目录。
+    结束位置为该起始标题之后的第一个结束标题；end_title=None 表示直到文末。
+    occurrence 使用 Python 索引规则，缺失标题或越界时抛出 ValueError。
+    """
+    if not isinstance(text, str):
+        raise TypeError("text 必须为字符串")
+    if not isinstance(start_title, str) or not start_title.strip():
+        raise ValueError("start_title 不能为空")
+    if end_title is not None and (
+        not isinstance(end_title, str) or not end_title.strip()
+    ):
+        raise ValueError("end_title 必须为非空字符串或 None")
+    if isinstance(occurrence, bool) or not isinstance(occurrence, int):
+        raise ValueError("occurrence 必须为整数")
+    lines = text.splitlines()
+    starts = [
+        index for index, line in enumerate(lines) if line.strip() == start_title.strip()
+    ]
+    try:
+        start = starts[occurrence]
+    except IndexError as exc:
+        raise ValueError("找不到指定次数的起始标题") from exc
+    end = len(lines)
+    if end_title is not None:
+        end = next(
+            (
+                index
+                for index in range(start + 1, len(lines))
+                if lines[index].strip() == end_title.strip()
+            ),
+            -1,
+        )
+        if end == -1:
+            raise ValueError("起始标题之后找不到结束标题")
+    return "\n".join(lines[start + 1 : end])
+
+
+def get_chapter_words(
+    text: str,
+    start_title: str,
+    end_title: str | None = None,
+    *,
+    occurrence: int = -1,
+    **options,
+) -> list[str]:
+    return extract_words(
+        extract_chapter(text, start_title, end_title, occurrence=occurrence), **options
+    )
+
+
+def word_frequencies(
+    words: Iterable[str], *, top_n: int | None = None, case_sensitive: bool = False
+) -> list[tuple[str, int]]:
+    """按词频降序返回 (词, 次数)，同频按首次出现顺序；默认合并大小写。
+
+    合并大小写时返回 casefold 后的词形。top_n=None 表示全部，0 返回空列表。
+    """
+    if top_n is not None and (
+        isinstance(top_n, bool) or not isinstance(top_n, int) or top_n < 0
+    ):
+        raise ValueError("top_n 必须为非负整数或 None")
+    cleaned = _clean_words(words)
+    return Counter(
+        word if case_sensitive else word.casefold() for word in cleaned
+    ).most_common(top_n)
+
+
+def format_words(words: Iterable[str], **options) -> str:
+    """按词频排序并返回去重后的逐行词表文本，替代旧写文件函数。"""
+    return "\n".join(word for word, _ in word_frequencies(words, **options))
+
+
+def group_words_by_length(words: Iterable[str]) -> dict[int, list[str]]:
+    """按长度分组，保留各组输入顺序和重复项。"""
+    groups = {}
+    for word in _clean_words(words):
+        groups.setdefault(len(word), []).append(word)
+    return groups
+
+
+def sort_words_by_length(words: Iterable[str], *, descending: bool = True) -> list[str]:
+    """按长度稳定排序，保留同长度的原顺序，不修改输入。"""
+    return sorted(_clean_words(words), key=len, reverse=descending)
+
+
+def is_title_case_word(word: str) -> bool:
+    """保留旧判断规则：Dog 为 True，A、DOG、DoG 和带标点的文本为 False。"""
+    return isinstance(word, str) and re.fullmatch(r"[A-Z][a-z]+", word) is not None
+
+
+def parse_urls(text: str | Iterable[str]) -> list[str]:
+    """逐行读取 HTTP(S) URL，忽略空行和 # 注释，按首次出现顺序去重。"""
+    result = []
+    seen = set()
+    for chunk in _text_chunks(text):
+        for line in chunk.splitlines():
+            url = line.strip()
+            if not url or url.startswith("#"):
+                continue
+            parts = urlsplit(url)
+            _ = parts.port  # 触发端口格式及范围校验。
+            if (
+                parts.scheme not in ("http", "https")
+                or not parts.hostname
+                or any(char.isspace() for char in url)
+            ):
+                raise ValueError(f"无效的 HTTP(S) URL：{url}")
+            if url not in seen:
+                seen.add(url)
+                result.append(url)
+    return result

jwf_common_python/language_utils/files.py

@@ -0,0 +1,52 @@
+"""只在显式调用时读取指定路径，不创建目录或写文件。"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from .core import extract_words, filter_words, parse_urls
+
+
+def read_words(path: str | Path, *, encoding: str = "utf-8", **options) -> list[str]:
+    """读取指定文本文件并取词，options 同 extract_words。"""
+    return extract_words(Path(path).read_text(encoding=encoding), **options)
+
+
+def read_word_lists(
+    paths: Iterable[str | Path],
+    *,
+    encoding: str = "utf-8",
+    case_sensitive: bool = False,
+) -> list[str]:
+    """合并多个词表文件并去重，保留首次出现拼写，修复旧代码只保留最后文件的问题。"""
+    if isinstance(paths, (str, Path)):
+        raise TypeError("paths 必须为路径的可迭代对象，例如 [path]")
+    words = (word for path in paths for word in read_words(path, encoding=encoding))
+    return filter_words(words, unique=True, case_sensitive=case_sensitive)
+
+
+def read_urls(path: str | Path, *, encoding: str = "utf-8") -> list[str]:
+    return parse_urls(Path(path).read_text(encoding=encoding))
+
+
+def generate_file_menu(
+    directory: str | Path, *, prefix: str = "", recursive: bool = True
+) -> tuple[list[str], str]:
+    """列出 .txt 文件，返回排序后的相对路径列表及换行连接的菜单文本。
+
+    prefix 筛选首级文件或目录名，例如旧数据的 wanglu_ 前缀。
+    相对子路径保留，避免不同目录下的同名文件混淆。
+    """
+    root = Path(directory)
+    if not root.exists():
+        raise FileNotFoundError(root)
+    if not root.is_dir():
+        raise NotADirectoryError(root)
+    candidates = root.rglob("*.txt") if recursive else root.glob("*.txt")
+    paths = sorted(
+        path.relative_to(root).as_posix()
+        for path in candidates
+        if path.is_file() and path.relative_to(root).parts[0].startswith(prefix)
+    )
+    return paths, "\n".join(paths)

jwf_common_python/language_utils/web.py

@@ -0,0 +1,117 @@
+"""网页取词，等待全部任务结束后按 URL 输入顺序返回结果。"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable
+from concurrent.futures import ThreadPoolExecutor
+from html.parser import HTMLParser
+
+import requests
+
+from .core import _clean_words, extract_words, filter_words, parse_urls
+
+
+class _TextParser(HTMLParser):
+    HIDDEN_TAGS = {"script", "style", "noscript", "template"}
+    BLOCK_TAGS = {
+        "p",
+        "div",
+        "br",
+        "hr",
+        "li",
+        "ul",
+        "ol",
+        "section",
+        "article",
+        "header",
+        "footer",
+        "main",
+        "nav",
+        "pre",
+        "blockquote",
+        "table",
+        "tr",
+        "td",
+        "th",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+    }
+
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.chunks = []
+        self.hidden_depth = 0
+
+    def handle_starttag(self, tag, attrs):
+        if tag in self.HIDDEN_TAGS:
+            self.hidden_depth += 1
+        elif not self.hidden_depth and tag in self.BLOCK_TAGS:
+            self.chunks.append("\n")
+
+    def handle_endtag(self, tag):
+        if tag in self.HIDDEN_TAGS and self.hidden_depth:
+            self.hidden_depth -= 1
+        elif not self.hidden_depth and tag in self.BLOCK_TAGS:
+            self.chunks.append("\n")
+
+    def handle_data(self, data):
+        if not self.hidden_depth:
+            self.chunks.append(data)
+
+
+def fetch_words_from_pages(
+    urls: str | Iterable[str],
+    *,
+    max_workers: int = 3,
+    timeout: float = 30,
+    exclude_words: Iterable[str] = (),
+    known_words: Iterable[str] = (),
+    min_length: int = 2,
+    unique: bool = True,
+    case_sensitive: bool = False,
+) -> dict[str, list[str]]:
+    """获取 HTML 中的英文片段，移除标签及 script/style/noscript/template 内容。
+
+    不执行 JavaScript；结果不代表已通过词典校验。默认过滤单字母并去重，
+    不更改输入拼写。任何 HTTP/网络错误直接抛出，不返回不完整的成功结果。
+    """
+    if (
+        isinstance(max_workers, bool)
+        or not isinstance(max_workers, int)
+        or max_workers <= 0
+    ):
+        raise ValueError("max_workers 必须为正整数")
+    if (
+        isinstance(timeout, bool)
+        or not isinstance(timeout, (int, float))
+        or not math.isfinite(timeout)
+        or timeout <= 0
+    ):
+        raise ValueError("timeout 必须为有限的正数")
+    targets = parse_urls(urls)
+    excluded = tuple(_clean_words(exclude_words))
+    known = tuple(_clean_words(known_words))
+    options = dict(
+        exclude_words=excluded,
+        known_words=known,
+        min_length=min_length,
+        unique=unique,
+        case_sensitive=case_sensitive,
+    )
+    filter_words([], **options)  # 在发起网络请求前检查取词参数。
+
+    def fetch(url):
+        with requests.get(url, timeout=timeout) as response:
+            response.raise_for_status()
+            parser = _TextParser()
+            parser.feed(response.text)
+            parser.close()
+        return extract_words("".join(parser.chunks), **options)
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        return dict(zip(targets, executor.map(fetch, targets)))

jwf_common_python/llm/__init__.py

@@ -0,0 +1 @@
+# pip install openai

jwf_common_python/llm/aliyun_qwen.py

@@ -0,0 +1,10 @@
+# 百炼是阿里云推出的一站式大模型服务平台
+# 通义千问是阿里云自主研发的大语言模型，属于通义大模型系列的一部分
+# 千问大模型是通义千问的一部分，是基于Transformer架构构建的
+
+
+class LLMAliyunQwen:
+    """阿里云出品的千问大模型"""
+
+    def __init__(self):
+        pass

jwf_common_python/llm/azure_gpt.py

@@ -0,0 +1,66 @@
+import base64
+import os
+
+import requests
+
+# [参考1](https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Ctypescript%2Cpython-new&pivots=rest-api)
+# [参考2](https://learn.microsoft.com/en-us/azure/ai-services/openai/reference)
+# Configuration
+API_KEY = "xxx"
+# IMAGE_PATH = "YOUR_IMAGE_PATH"
+# encoded_image = base64.b64encode(open(IMAGE_PATH, "rb").read()).decode("ascii")
+headers = {
+    "Content-Type": "application/json",
+    "api-key": API_KEY,
+}
+
+# Payload for the request
+payload = {
+    "messages": [
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": "你是一个帮助用户查找信息的 AI 助手。"}],
+        }
+    ],
+    "temperature": 0.7,
+    "top_p": 0.95,
+    "max_tokens": 800,
+}
+
+ENDPOINT = "https://pro-chatgpt-4.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-02-15-preview"
+
+# Send request
+try:
+    response = requests.post(ENDPOINT, headers=headers, json=payload)
+    response.raise_for_status()  # Will raise an HTTPError if the HTTP request returned an unsuccessful status code
+except requests.RequestException as e:
+    raise SystemExit(f"Failed to make the request. Error: {e}")
+
+# Handle the response as needed (e.g., print or process)
+print(response.json())
+
+# 2222
+import os
+
+from openai import AzureOpenAI
+
+client = AzureOpenAI(
+    azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
+    api_key=os.getenv("AZURE_OPENAI_API_KEY"),
+    api_version="2024-02-01",
+)
+
+response = client.chat.completions.create(
+    model="gpt-35-turbo",  # model = "deployment_name".
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Does Azure OpenAI support customer managed keys?"},
+        {
+            "role": "assistant",
+            "content": "Yes, customer managed keys are supported by Azure OpenAI.",
+        },
+        {"role": "user", "content": "Do other Azure AI services support this too?"},
+    ],
+)
+
+print(response.choices[0].message.content)

jwf_common_python/llm/baidu_ernie.py

@@ -0,0 +1,112 @@
+# ------ 百度出品 ------:
+# 千帆是百度推出的一站式大模型平台，支持多种大模型:
+#   - 包括ERNIE模型
+#   - 可以接入其他大模型如Llama2、ChatGLM2等
+# ERNIE（Enhanced Representation through kNowledge IntEgration）是一种深度语义表示模型,基于Transformer架构
+# 文心一言是基于ERNIE模型开发的生成式对话产品，能够进行文学创作、商业文案创作、数理逻辑推理等多种任务
+import datetime
+
+import requests
+
+
+class LLMBaiduErnie:
+    """百度出品的ERNIE模型"""
+
+    def __init__(self, client_id: str, client_secret: str):
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.grant_type = "client_credentials"
+
+    def get_token_info(self) -> dict:
+        """获取百度大模型token
+
+        - access_token: 使用大模型的access_token
+        - expire_time: token过期时间,通常是30天
+        """
+        url = f"https://aip.baidubce.com/oauth/2.0/token?grant_type\
+={self.grant_type}&client_id={self.client_id}&client_secret={self.client_secret}"
+        response = requests.post(url, headers={"Content-Type": "application/json"})
+        if response.status_code != 200:
+            raise Exception(
+                f"百度大模型登录获取token失败: status code: {response.status_code}, response: {response.text}"
+            )
+        res = response.json()
+
+        # 1. 获取access_token
+        access_token = res["access_token"]
+
+        # 当前时间 xxx 秒后到期
+        expires_in = res["expires_in"]
+        # 当前时间
+        current_time = datetime.datetime.now()
+        # 2. 过期时间
+        expire_time = current_time + datetime.timedelta(seconds=expires_in)
+
+        self.token_info = {"access_token": access_token, "expire_time": expire_time}
+        return self.token_info
+
+    def send_model_request(self, text: str, temperature: float = 0.95):
+        """发送模型请求
+        温度越低,生成的文本越保守,
+        温度越高,生成的文本越有创意
+        """
+        # check1
+        if not hasattr(self, "token_info"):
+            # 当 token没有被获取时 重新获取token
+            self.get_token_info()
+
+        # check2
+        access_token = self.token_info["access_token"]
+        _expire_time = self.token_info["expire_time"]
+        if _expire_time <= datetime.datetime.now():
+            # 当 token过期时 重新获取token
+            self.get_token_info()
+            access_token = self.token_info["access_token"]
+
+        # 3. 构造请求参数
+        url = f"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/\
+completions_pro?access_token={access_token}"
+        json_data = {
+            "messages": [{"role": "user", "content": text}],
+            "temperature": temperature,
+            "top_p": 0.8,
+            "penalty_score": 1,
+            "enable_system_memory": False,
+            "disable_search": False,
+            "enable_citation": False,
+        }
+
+        # 4. 发送模型请求
+        response = requests.post(
+            url,
+            headers={"Content-Type": "application/json"},
+            json=json_data,
+        )
+        if response.status_code != 200:
+            raise Exception(
+                f"百度大模型请求失败: status code: {response.status_code}, response: {response.text}"
+            )
+
+        # 5. 返回结果
+        res = response.json()
+        msg = res["result"]
+        return msg
+
+    def is_natural_language(self, text: str) -> bool:
+        """判断文本是自然语言还是非自然语言
+
+        True: 自然语言
+        False: 非自然语言
+        """
+        prompt = f"""请判断以下文本是普通自然语言还是代码、配置文件等非自然语言.
+如果是自然语言那就返回字符串 "Natural Language",如果是非自然语言,那就返回字符串 "Non-Natural Language",但是不要返回包裹字符串的引号.
+需要判断的文本:\n {text} """
+
+        res_status = self.send_model_request(prompt, temperature=0.1)
+        match res_status:
+            case "Natural Language":
+                return True
+            case "Non-Natural Language":
+                return False
+            case _:
+                raise Exception(f"未知的返回值: {res_status}")

jwf_common_python/message_queue/__init__.py

@@ -0,0 +1,7 @@
+"""Kafka 和 RabbitMQ 消息队列；使用对应后端时才加载其可选依赖。"""
+
+from .interface import MessageCallback, MessageQueue
+from .kafka import KafkaQueue
+from .rabbitmq import RabbitmqQueue
+
+__all__ = ["MessageCallback", "MessageQueue", "KafkaQueue", "RabbitmqQueue"]

jwf_common_python/message_queue/interface.py

@@ -0,0 +1,49 @@
+"""消息队列的同步接口。"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import Any
+
+MessageCallback = Callable[[bytes | None], None]
+
+
+class MessageQueue(ABC):
+    """消息以 bytes 传递（Kafka 删除标记为 None）。
+
+    回调正常返回表示处理成功，抛出异常表示失败。
+    消费采用至少一次处理语义，业务回调应能处理重复消息。
+    实例不支持跨线程共享，使用 with 或 close() 释放连接。
+    """
+
+    def __init__(self) -> None:
+        self._closed = False
+
+    def _ensure_open(self) -> None:
+        if self._closed:
+            raise RuntimeError("消息队列已关闭")
+
+    @staticmethod
+    def _validate_item(item: bytes) -> None:
+        if not isinstance(item, bytes):
+            raise TypeError("item 必须为 bytes")
+
+    @abstractmethod
+    def enqueue(self, item: bytes) -> dict[str, Any] | None:
+        """等待服务端确认发送；失败时抛出原始 SDK 异常。"""
+
+    @abstractmethod
+    def consumer(self, callback: MessageCallback) -> None:
+        """阻塞消费；只在 callback 成功后确认消息，异常向调用方传播。"""
+
+    @abstractmethod
+    def close(self) -> None:
+        """释放资源，可重复调用。"""
+
+    def __enter__(self):
+        self._ensure_open()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.close()