jwf-common-python 1.0.0__py3-none-any.whl

jwf_common_python/__init__.py

@@ -0,0 +1,14 @@
+import importlib
+
+class CommonPackageManager:
+    """公共包管理器"""
+
+    def _check_dependencies(self, module_names: list[str]):
+        """检查依赖项是否已安装"""
+        for module_name in module_names:
+            try:
+                setattr(self, module_name, importlib.import_module(module_name))
+            except ImportError:
+                print(
+                    f"Missing required dependencies. use pip install {module_name} to install it."
+                )

jwf_common_python/db_utils/__init__.py

@@ -0,0 +1,6 @@
+"""数据库工具；后端依赖在实例化或使用对应功能时加载。"""
+
+from .clickhouse import ClickhouseOperator
+from .elasticsearch import ElasticsearchHandle
+
+__all__ = ["ClickhouseOperator", "ElasticsearchHandle"]

jwf_common_python/db_utils/clickhouse.py

@@ -0,0 +1,124 @@
+"""ClickHouse 查询和 DataFrame 写入工具。"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pandas import DataFrame
+
+
+def _pandas():
+    try:
+        import pandas
+    except ImportError as exc:
+        raise ImportError("DataFrame 功能需要：pip install pandas numpy") from exc
+    return pandas
+
+
+class ClickhouseOperator:
+    def __init__(
+        self,
+        host: str,
+        port: int | str = 9000,
+        user: str = "default",
+        password: str = "",
+        database: str = "default",
+        *,
+        settings: dict[str, Any] | None = None,
+        **client_options,
+    ) -> None:
+        """使用原生 TCP 协议，client_options 可配置 TLS、连接超时等。
+
+        默认启用 NumPy，与原 DataFrame 写入行为一致。
+        仅执行普通 SQL 时可传 settings={"use_numpy": False}，无需 NumPy/pandas。
+        """
+        if not isinstance(host, str) or not host.strip():
+            raise ValueError("host 不能为空")
+        port = int(port)
+        if not 1 <= port <= 65535:
+            raise ValueError("port 必须在 1 到 65535 之间")
+        try:
+            from clickhouse_driver import Client
+        except ImportError as exc:
+            raise ImportError(
+                "请先安装 ClickHouse 依赖：pip install clickhouse-driver numpy"
+            ) from exc
+
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.database = database
+        self._closed = False
+        self.client = Client(
+            host=host,
+            port=port,
+            user=user,
+            password=password,
+            database=database,
+            settings={"use_numpy": True, **(settings or {})},
+            **client_options,
+        )
+
+    def _ensure_open(self) -> None:
+        if self._closed:
+            raise RuntimeError("ClickHouse 客户端已关闭")
+
+    def execute(self, sql: str, params=None, **options):
+        """执行 SQL 并返回原始结果，params 交由驱动绑定，不拼接参数。"""
+        self._ensure_open()
+        return self.client.execute(sql, params=params, **options)
+
+    def fetch_all_tables(self) -> set[str]:
+        """返回当前数据库的表名，无需创建 DataFrame。"""
+        return {row[0] for row in self.execute("SHOW TABLES")}
+
+    def fetch_specify_sql_data(
+        self, sql: str, replace_nonwords: bool = True, *, params=None, **options
+    ) -> tuple[DataFrame, list[str], int]:
+        """返回 (DataFrame, 列名, 行数)，保留旧接口并增加查询参数绑定。"""
+        self._ensure_open()
+        _pandas()
+        data = self.client.query_dataframe(
+            sql, params=params, replace_nonwords=replace_nonwords, **options
+        )
+        return data, data.columns.to_list(), len(data)
+
+    @staticmethod
+    def convert_datatype_for_ck(data: DataFrame) -> DataFrame:
+        """复制后将 category 列转为字符串，缺失值保留为 None，不修改原数据。"""
+        pd = _pandas()
+        if not isinstance(data, pd.DataFrame):
+            raise TypeError("data 必须为 pandas.DataFrame")
+        result = data.copy(deep=True)
+        for column, dtype in result.dtypes.items():
+            if isinstance(dtype, pd.CategoricalDtype):
+                values = result[column].astype(object).map(str, na_action="ignore")
+                result[column] = values.where(values.notna(), None)
+        return result
+
+    def insert_specify_sql(self, sql_part: str, data: DataFrame, **options) -> int:
+        """按指定 INSERT SQL 写入 DataFrame，返回驱动报告的写入行数。
+
+        不自动转换数据；需要转换 category 时先调用 convert_datatype_for_ck。
+        """
+        self._ensure_open()
+        pd = _pandas()
+        if not isinstance(data, pd.DataFrame):
+            raise TypeError("data 必须为 pandas.DataFrame")
+        return self.client.insert_dataframe(sql_part, data, **options)
+
+    def close(self) -> None:
+        if not self._closed:
+            try:
+                self.client.disconnect()
+            finally:
+                self._closed = True
+
+    def __enter__(self):
+        self._ensure_open()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.close()

jwf_common_python/db_utils/elasticsearch.py

@@ -0,0 +1,196 @@
+"""通用 Elasticsearch 工具，不依赖 Django 或业务模型。"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+
+class ElasticsearchHandle:
+    def __init__(self, hosts, index: str, **client_options) -> None:
+        """显式传入 hosts、索引名；认证、TLS、超时通过 client_options 传入。"""
+        if not hosts:
+            raise ValueError("hosts 不能为空")
+        if not isinstance(index, str) or not index.strip():
+            raise ValueError("index 不能为空")
+        try:
+            from elasticsearch import Elasticsearch
+        except ImportError as exc:
+            raise ImportError(
+                "请先安装 Elasticsearch 依赖：pip install elasticsearch"
+            ) from exc
+
+        self.index = index
+        self._closed = False
+        self.elasticsearch = Elasticsearch(
+            hosts=hosts, **{"request_timeout": 30, **client_options}
+        )
+
+    def _ensure_open(self) -> None:
+        if self._closed:
+            raise RuntimeError("Elasticsearch 客户端已关闭")
+
+    @staticmethod
+    def _document(content: Mapping[str, Any]) -> dict[str, Any]:
+        if not isinstance(content, Mapping):
+            raise TypeError("文档内容必须为 Mapping")
+        return dict(content)
+
+    @staticmethod
+    def _validate_id(doc_id) -> None:
+        if (
+            isinstance(doc_id, bool)
+            or not isinstance(doc_id, (str, int))
+            or (isinstance(doc_id, str) and not doc_id.strip())
+        ):
+            raise ValueError("doc_id 必须为非空字符串或整数")
+
+    @staticmethod
+    def _validate_bulk_options(id_field: str, chunk_size: int) -> None:
+        if not isinstance(id_field, str) or not id_field.strip():
+            raise ValueError("id_field 不能为空")
+        if not isinstance(chunk_size, int) or chunk_size <= 0:
+            raise ValueError("chunk_size 必须为正整数")
+
+    def create_index(self, *, mappings=None, settings=None) -> dict[str, Any]:
+        """创建索引并返回完整响应体；索引已存在时保留 SDK 的冲突异常。"""
+        self._ensure_open()
+        body = self.elasticsearch.indices.create(
+            index=self.index, mappings=mappings, settings=settings
+        ).body
+        if not body.get("acknowledged"):
+            raise RuntimeError("Elasticsearch 未确认索引创建")
+        return body
+
+    def init_data(
+        self,
+        documents: Iterable[Mapping[str, Any]],
+        *,
+        mappings=None,
+        settings=None,
+        id_field: str = "id",
+        chunk_size: int = 500,
+        refresh: bool | str = False,
+    ) -> dict[str, Any]:
+        """索引不存在时创建，然后批量创建传入文档，不查询任何业务数据库。
+
+        已存在的文档 ID 会引发冲突，不覆盖原文档。批量导入不是事务，
+        失败前已成功写入的文档不会回滚。
+        """
+        self._ensure_open()
+        self._validate_bulk_options(id_field, chunk_size)
+        index_response = None
+        if not self.elasticsearch.indices.exists(index=self.index):
+            index_response = self.create_index(mappings=mappings, settings=settings)
+        count, errors = self.bulk_create(
+            documents, id_field=id_field, chunk_size=chunk_size, refresh=refresh
+        )
+        return {
+            "index_response": index_response,
+            "success_count": count,
+            "errors": errors,
+        }
+
+    def bulk_create(
+        self,
+        documents: Iterable[Mapping[str, Any]],
+        *,
+        id_field: str = "id",
+        chunk_size: int = 500,
+        refresh: bool | str = False,
+    ) -> tuple[int, list[dict[str, Any]]]:
+        """分批消费可迭代文档，返回 (成功数, 错误列表)；SDK 失败时抛异常。"""
+        self._ensure_open()
+        self._validate_bulk_options(id_field, chunk_size)
+        from elasticsearch.helpers import bulk
+
+        def actions():
+            for record in documents:
+                document = self._document(record)
+                doc_id = document.get(id_field)
+                self._validate_id(doc_id)
+                yield {
+                    "_op_type": "create",
+                    "_index": self.index,
+                    "_id": doc_id,
+                    "_source": document,
+                }
+
+        return bulk(
+            self.elasticsearch,
+            actions(),
+            chunk_size=chunk_size,
+            raise_on_error=True,
+            raise_on_exception=True,
+            refresh=refresh,
+        )
+
+    def create_data(
+        self, doc_id, content: Mapping[str, Any], **options
+    ) -> dict[str, Any]:
+        self._ensure_open()
+        self._validate_id(doc_id)
+        return self.elasticsearch.create(
+            index=self.index, id=doc_id, document=self._document(content), **options
+        ).body
+
+    def update_data(
+        self, doc_id, content: Mapping[str, Any], **options
+    ) -> dict[str, Any]:
+        """content 为待更新字段字典，不需要再包一层 doc。"""
+        self._ensure_open()
+        self._validate_id(doc_id)
+        return self.elasticsearch.update(
+            index=self.index, id=doc_id, doc=self._document(content), **options
+        ).body
+
+    def delete_doc(self, doc_id, **options) -> dict[str, Any]:
+        self._ensure_open()
+        self._validate_id(doc_id)
+        return self.elasticsearch.delete(index=self.index, id=doc_id, **options).body
+
+    def search(
+        self, query: Mapping[str, Any] | None = None, **options
+    ) -> dict[str, Any]:
+        """执行查询并返回完整响应体，支持 size、from_、sort 等 SDK 参数。"""
+        self._ensure_open()
+        return self.elasticsearch.search(
+            index=self.index,
+            query={"match_all": {}} if query is None else self._document(query),
+            **options,
+        ).body
+
+    def search_doc(
+        self, key: str, *, field: str = "content", size: int = 10, offset: int = 0
+    ) -> dict[str, Any]:
+        """全文匹配指定字段并返回结果，包含高亮、总数等原始字段。"""
+        if not isinstance(key, str) or not key.strip():
+            raise ValueError("key 不能为空")
+        if not isinstance(field, str) or not field.strip():
+            raise ValueError("field 不能为空")
+        if size <= 0 or offset < 0:
+            raise ValueError("size 必须大于 0，offset 不能小于 0")
+        return self.search(
+            {"match": {field: key}},
+            highlight={"fields": {field: {}}},
+            size=size,
+            from_=offset,
+        )
+
+    def show_info(self) -> dict[str, Any]:
+        """保留旧接口：返回索引默认搜索页，不代表遍历所有文档。"""
+        return self.search()
+
+    def close(self) -> None:
+        if not self._closed:
+            try:
+                self.elasticsearch.close()
+            finally:
+                self._closed = True
+
+    def __enter__(self):
+        self._ensure_open()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.close()

jwf_common_python/db_utils/mysql.py

@@ -0,0 +1,64 @@
+from urllib.parse import quote_plus
+
+try:
+    import pandas as pd
+    import pymysql  # noqa
+    from sqlalchemy import create_engine
+except ImportError as exc:
+    raise ImportError(
+        "Couldn't import pandas sqlalchemy. \
+            pip install --upgrade pandas sqlalchemy pymysql"
+    ) from exc
+from pandas import DataFrame, Series
+
+
+class MysqlOperator:
+    def __init__(self, **kwargs) -> None:
+        if not {"host", "port", "user", "password", "database"}.issubset(kwargs.keys()):
+            raise ValueError("host,port,user,password,database 必须提供")
+        self.host = kwargs["host"]
+        self.port = kwargs["port"]
+        self.user = kwargs["user"]
+        self.password = quote_plus(kwargs["password"])
+        self.database = kwargs["database"]
+        _ = f"{self.user}:{self.password}@{self.host}:{self.port}/{self.database}"
+        c = f"mysql+pymysql://{_}"
+        self.engine = create_engine(c)
+
+    def fetch_all_tables(self) -> set[str]:
+        """获取数据库中所有的表名"""
+        sql_query = """SELECT table_name AS tables FROM information_schema.tables
+                        WHERE table_schema = DATABASE();
+                    """
+        df: DataFrame = pd.read_sql(sql_query, self.engine)
+        names: Series[str] = df["tables"]
+        names_set: set[str] = set(names.to_list())
+        return names_set
+
+    def fetch_specify_sql_data(self, sql: str) -> tuple[DataFrame, list[str]]:
+        """执行`指定插叙sql`,获取DataFrame类型的插叙结果
+
+        返回如下: 结果dataframe, 列名组成list
+
+        replace_noneword False 表示不自动转换列名
+        """
+
+        data: DataFrame = pd.read_sql(sql, self.engine)
+        columns: list[str] = data.columns.to_list()
+        # print(type(data), columns, len(data))
+        return (data, columns)
+
+    # def execute_sql(self, sql: str, parameters) -> None:
+    #     """执行sql"""
+    #     with self.engine.connect() as connection:
+    #         connection.execute(sql, parameters)
+
+    # def delete_auto_increment_column(self, df: DataFrame, pk: str = "id") -> DataFrame:
+    #     """删除dataframe中的自增列"""
+    #     df = df.drop(columns=pk)
+    #     return df
+
+    def insert_database_mysql(self, df: DataFrame, table_name: str) -> int:
+        """将dataframe中的数据插入到mysql中"""
+        row: int = df.to_sql(table_name, self.engine, if_exists="append", index=False)
+        return row

jwf_common_python/eudic/__init__.py

@@ -0,0 +1,5 @@
+"""欧路词典生词本工具。"""
+
+from .wordbook import EudicClient
+
+__all__ = ["EudicClient"]

jwf_common_python/eudic/wordbook.py

@@ -0,0 +1,211 @@
+"""欧路 OpenAPI 生词本客户端，不打印、不读写文件、不退出进程。"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+import requests
+
+BASE_URL = "https://api.frdic.com/api/open/v1/studylist"
+MAX_PAGE = 50
+
+
+class _TokenAuth(requests.auth.AuthBase):
+    def __init__(self, token: str) -> None:
+        self._token = token
+
+    def __call__(self, request):
+        request.headers["Authorization"] = self._token
+        return request
+
+
+class EudicClient:
+    def __init__(
+        self, token: str, *, language: str = "en", timeout: float = 30
+    ) -> None:
+        """token 为完整 Authorization 请求头值，例如 NIS 开头的授权字符串。"""
+        if not isinstance(token, str) or not token.strip():
+            raise ValueError("token 不能为空")
+        if language not in ("en", "fr", "de", "es"):
+            raise ValueError("language 必须为 en、fr、de 或 es")
+        if (
+            isinstance(timeout, bool)
+            or not isinstance(timeout, (int, float))
+            or not math.isfinite(timeout)
+            or timeout <= 0
+        ):
+            raise ValueError("timeout 必须为有限的正数")
+        self.language = language
+        self.timeout = timeout
+        self._closed = False
+        self._session = requests.Session()
+        # 显式认证，避免 .netrc 中的 BasicAuth 覆盖传入 Token。
+        self._session.auth = _TokenAuth(token)
+        self._session.headers["Content-Type"] = "application/json"
+
+    def _ensure_open(self) -> None:
+        if self._closed:
+            raise RuntimeError("欧路客户端已关闭")
+
+    @staticmethod
+    def _validate_book_id(book_id: str | int) -> None:
+        if (
+            isinstance(book_id, bool)
+            or not isinstance(book_id, (str, int))
+            or (isinstance(book_id, str) and not book_id.strip())
+            or (isinstance(book_id, int) and book_id < 0)
+        ):
+            raise ValueError("book_id 必须为非空字符串或非负整数，0 表示默认生词本")
+
+    def _request(
+        self, method: str, path: str, *, expected_status: int, **kwargs
+    ) -> dict[str, Any]:
+        self._ensure_open()
+        with self._session.request(
+            method,
+            f"{BASE_URL}/{path}",
+            timeout=self.timeout,
+            allow_redirects=False,
+            **kwargs,
+        ) as response:
+            response.raise_for_status()
+            if response.status_code != expected_status:
+                raise requests.HTTPError(
+                    f"欧路接口返回非预期状态码 {response.status_code}，预期 {expected_status}",
+                    response=response,
+                )
+            body = response.json()
+        if not isinstance(body, dict):
+            raise ValueError("欧路接口响应必须为 JSON 对象")
+        return body
+
+    def create_book(self, name: str) -> dict[str, Any]:
+        """创建指定名称的生词本，返回完整响应；ID 位于 data.id。"""
+        self._ensure_open()
+        if not isinstance(name, str) or not name.strip():
+            raise ValueError("name 不能为空")
+        body = self._request(
+            "POST",
+            "category",
+            expected_status=201,
+            json={"language": self.language, "name": name},
+        )
+        data = body.get("data")
+        if not isinstance(data, dict) or "id" not in data:
+            raise ValueError("创建生词本的响应缺少 data.id")
+        self._validate_book_id(data["id"])
+        return body
+
+    def list_books(self) -> dict[str, Any]:
+        """返回所有生词本的完整响应，生词本列表位于 data。"""
+        body = self._request(
+            "GET", "category", expected_status=200, params={"language": self.language}
+        )
+        if not isinstance(body.get("data"), list):
+            raise ValueError("生词本列表响应缺少 data 数组")
+        return body
+
+    def add_words(
+        self, book_id: str | int, words: Iterable[str], *, star: int | None = None
+    ) -> dict[str, Any]:
+        """向指定生词本写入 1～1000 个单词，返回完整响应。
+
+        默认保留已有单词星级，使用服务端兼容的 words 字段；传入 star=1～5
+        时使用推荐的 items 字段，显式设置本批单词星级。不自动创建生词本。
+        """
+        self._ensure_open()
+        self._validate_book_id(book_id)
+        if star is not None and (
+            isinstance(star, bool) or not isinstance(star, int) or not 1 <= star <= 5
+        ):
+            raise ValueError("star 必须为 1 到 5 的整数或 None")
+        if isinstance(words, (str, bytes, Mapping)):
+            raise TypeError("words 必须为单词字符串的可迭代对象")
+        clean_words = []
+        for word in words:
+            if not isinstance(word, str) or not word.strip():
+                raise ValueError("单词必须为非空字符串")
+            clean_words.append(word.strip())
+            if len(clean_words) > 1000:
+                raise ValueError("单次最多导入 1000 个单词")
+        if not clean_words:
+            raise ValueError("words 不能为空")
+        body = {"category_id": book_id, "language": self.language}
+        if star is None:
+            body["words"] = clean_words
+        else:
+            body["items"] = [{"word": word, "star": star} for word in clean_words]
+        return self._request("POST", "words", expected_status=201, json=body)
+
+    def get_words(
+        self,
+        book_id: str | int,
+        *,
+        page: int = 0,
+        page_size: int = 100,
+        sort: str = "asc",
+    ) -> dict[str, Any]:
+        """获取一页完整响应，页码从 0 开始；单词、释义等原始字段位于 data。"""
+        self._ensure_open()
+        self._validate_book_id(book_id)
+        if (
+            isinstance(page, bool)
+            or not isinstance(page, int)
+            or not 0 <= page <= MAX_PAGE
+        ):
+            raise ValueError(f"page 必须为 0 到 {MAX_PAGE} 的整数")
+        if (
+            isinstance(page_size, bool)
+            or not isinstance(page_size, int)
+            or not 1 <= page_size <= 100
+        ):
+            raise ValueError("page_size 必须为 1 到 100 的整数")
+        if sort not in ("asc", "desc"):
+            raise ValueError("sort 必须为 asc 或 desc")
+        body = self._request(
+            "GET",
+            "words",
+            expected_status=200,
+            params={
+                "category_id": book_id,
+                "language": self.language,
+                "page": page,
+                "page_size": page_size,
+                "sort": sort,
+            },
+        )
+        if not isinstance(body.get("data"), list):
+            raise ValueError("单词列表响应缺少 data 数组")
+        return body
+
+    def export_book(
+        self, book_id: str | int, *, page_size: int = 100, sort: str = "asc"
+    ) -> list[dict[str, Any]]:
+        """从第 0 页导出，返回每页完整响应，不写文件。
+
+        短页/空页表示导出结束。若第 50 页仍为满页，受接口上限限制无法确认
+        是否完整，此时抛出 RuntimeError；请求失败也不返回部分结果冒充完整导出。
+        """
+        pages = []
+        for page in range(MAX_PAGE + 1):
+            body = self.get_words(book_id, page=page, page_size=page_size, sort=sort)
+            pages.append(body)
+            if len(body["data"]) < page_size:
+                return pages
+        raise RuntimeError("已达到欧路接口分页上限，无法确认已导出全部单词")
+
+    def close(self) -> None:
+        if not self._closed:
+            try:
+                self._session.close()
+            finally:
+                self._closed = True
+
+    def __enter__(self):
+        self._ensure_open()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.close()

jwf_common_python/file_storage/__init__.py

@@ -0,0 +1,2 @@
+from .local import LocalStorage
+from .tencent_cos import TencentCos

jwf_common_python/file_storage/file_interface.py

@@ -0,0 +1,29 @@
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import ByteString
+
+import requests
+
+
+class Storage(ABC):
+    @abstractmethod
+    def __init__(self, **kwargs) -> None:
+        ...
+
+    @abstractmethod
+    def save(self, filename: str, content: ByteString) -> tuple[bool, str]:
+        ...
+
+    @abstractmethod
+    def load(self, filename: str) -> tuple[bool, bytes]:
+        ...
+
+    def download_audio_file(self, url: str) -> bytes:
+        try:
+            response = requests.get(url=url)
+        except Exception as e:
+            raise ValueError(f"download error: {e}")
+        if response.status_code != 200:
+            raise ValueError(f"download failed,status code: {response.status_code}")
+        logging.info(f"download success: {url}")
+        return response.content