jwf-common-python 1.0.0__py3-none-any.whl

jwf_common_python/file_storage/local.py

@@ -0,0 +1,50 @@
+import os
+from collections.abc import ByteString
+
+from jwf_common_python.file_storage.file_interface import Storage
+
+
+class LocalStorage(Storage):
+    def __init__(self, **kwargs) -> None:
+        """
+            1. try to create directory if not exist
+        Args:
+            storage_home_dir(str): 本地存储的家目录
+
+        Raises:
+            ValueError: _description_
+            ValueError: _description_
+        """
+        if "storage_home_dir" not in kwargs:
+            raise ValueError("storage_home_dir must be provided")
+        storage_home_dir = kwargs["storage_home_dir"]
+        # 判断目录是否存在，不存在则创建
+        if not os.path.exists(storage_home_dir):
+            try:
+                os.mkdir(storage_home_dir)
+            except Exception as e:
+                raise ValueError(
+                    "storage_home_dir does not exist and cannot be created"
+                )
+        self.storage_home_dir = storage_home_dir
+
+    def save(self, filename: str, content: ByteString) -> tuple[bool, str]:
+        """保存文件内容到指定文件中.filename是文件名，不是文件全部路径"""
+        # 完整路径
+        file_path: str = os.path.join(self.storage_home_dir, filename)
+        try:
+            with open(file_path, "wb") as f:
+                f.write(content)
+        except Exception as e:
+            return False, e
+        else:
+            return True, file_path
+
+    def load(self, filename: str) -> tuple[bool, bytes]:
+        try:
+            with open(os.path.join(self.storage_home_dir, filename), "rb") as f:
+                content: bytes = f.read()
+        except Exception as e:
+            return False, e
+        else:
+            return True, content

jwf_common_python/file_storage/tencent_cos.py

@@ -0,0 +1,65 @@
+import logging
+from collections.abc import ByteString
+
+import requests
+
+from jwf_common_python.file_storage.file_interface import Storage
+from jwf_common_python.utils.tencent_utils import generate_tencent_key
+
+# TODO sdk方式封装
+# TODO 判断文件是否存在
+
+
+class TencentCos(Storage):
+    def __init__(self, **kwargs) -> None:
+        if not {"region", "bucket", "secret_key", "secret_id"}.issubset(kwargs):
+            raise ValueError("region,key,secret_key,secret_id must be provided")
+        self.bucket = kwargs["bucket"]
+        self.region = kwargs["region"]
+        self.secret_key = kwargs["secret_key"]
+        self.secret_id = kwargs["secret_id"]
+
+    # api 注明
+    def save(self, filename: str, content: ByteString) -> tuple[bool, str]:
+        """只是文件名，而不是文件全部路径"""
+        url = f"https://{self.bucket}.cos.{self.region}.myqcloud.com/{filename}"
+        authorization: str = generate_tencent_key(
+            secret_id=self.secret_id,
+            secret_key=self.secret_key,
+            key=filename,
+            method="put",
+        )
+        try:
+            s = requests.put(
+                url=url,
+                headers={"Authorization": authorization, "Content-Type": "image/jpeg"},
+                data=content,
+            )
+        except Exception as e:
+            print(e)
+            return False, e
+        logging.info(
+            f"tencent cos save file {filename} success. {s.status_code=} {s.content=}"
+        )
+        return True, filename
+
+    def load(self, filename: str) -> tuple[bool, bytes]:
+        self.key = filename
+        url = f"https://{self.bucket}.cos.{self.region}.myqcloud.com/{self.key}"
+        authorization: str = generate_tencent_key(
+            secret_id=self.secret_id, secret_key=self.secret_key, key=self.key
+        )
+        try:
+            s = requests.get(
+                url=url,
+                headers={
+                    "Authorization": authorization,
+                },
+            )
+        except Exception as e:
+            logging.error(f"tencent cos read failed {e}")
+            return False, b""
+
+        logging.info(f"tencent cos save file {self.key} success. {s.status_code}")
+        # print(s.content)
+        return True, s.content

jwf_common_python/geography/__init__.py

@@ -0,0 +1,8 @@
+# 1. district_供应商; 使用 指定供应商 获取 地理行政区域 信息
+# eg: 1. 获取所有国家
+# eg: 2. 获取国家下的所有省份
+# eg: 3. 获取省份下的所有城市
+
+
+# 2. search_供应商; 使用【指定供应商】搜索 【指定地理位置】 的 【关键字】
+# eg: 使用【高德】搜索【上海】所有的【星巴克】

jwf_common_python/geography/district_amap.py

@@ -0,0 +1,181 @@
+import requests
+
+
+class ChinaDistrictAmap:
+    """获取中国的行政区域
+    使用 高德地图 获取中国的行政区域信息
+
+    city_code: 城市编码。只有市级行政区有citycode。
+        - 县/区、城镇/街道 没有自己的citycode,使用所在市的citycode
+        - 省级行政区没有citycode
+    adcode: 行政区域编码。
+        - 省、市、区/县 的adcode是唯一的
+        - 城镇/街道 使用所在区/县的adcode
+    """
+
+    def __init__(self, key: str):
+        self.key = key
+        self.base_url = "https://restapi.amap.com/v3/config/district"
+        self.country = "中华人民共和国"
+        self._ = {
+            "810000": "香港特别行政区",
+            "820000": "澳门特别行政区",
+        }
+
+    def _send_request(
+        self,
+        url: str,
+        name: str | None = None,
+        adcode: str | None = None,
+        citycode: str | None = None,
+    ) -> dict:
+        """
+        name,adcode,citycode 用于校验和解析有效的districts
+         - 默认为None, 校验条件是districts长度是1
+         - 如果不为None, districts长度大于1,则根据 name,adcode,citycode 过滤，找到唯一值
+        """
+        response = requests.get(url)
+        if response.status_code != 200:
+            raise Exception("请求高德地图API失败")
+        res = response.json()
+
+        # 1. 校验响应状态
+        status = res["status"]
+        assert status == "1", f"请求高德地图API失败，status: {status}"
+        info = res.get("info")
+        assert info == "OK", f"请求高德地图API失败，info: {info}"
+        infocode = res.get("infocode")
+        assert infocode == "10000", f"请求高德地图API失败，infocode: {infocode}"
+        # count = res["count"]
+
+        # 2. 校验响应结构
+        districts = res["districts"]
+        # 符合条件的有效结果
+        response_data_info = []
+        if (name is not None) and (adcode is not None) and (citycode is not None):
+            # 3个同时不是None,进行过滤,此时districts长度 >= 1
+            assert len(districts) >= 1, "districts长度必须大于等于1"
+            for district in districts:
+                if (
+                    district["name"] == name
+                    and district["adcode"] == adcode
+                    and district["citycode"] == citycode
+                ):
+                    response_data_info.append(district)
+        else:
+            assert len(districts) == 1
+            response_data_info = districts
+
+        # 有效结果只能有1个
+        if len(response_data_info) != 1:
+            raise Exception(f"未找到符合条件的district, districts: {districts}")
+
+        # 3. 将实际有效的内容进行提取返回
+        return response_data_info[0]
+
+    def get_privince(self) -> list[dict]:
+        """获取省份
+
+        "citycode": [],
+        "adcode": "410000",
+        "name": "河南省",
+        "center": "113.753094,34.767052",
+        "level": "province",
+        "districts": []
+        """
+        url = f"{self.base_url}?key={self.key}&keywords={self.country}"
+
+        china_info = self._send_request(url)
+
+        # 获取国家信息，判断是否是中国
+        china_adcode = china_info["adcode"]
+        china_name = china_info["name"]
+        level = china_info["level"]
+        assert china_adcode == "100000"
+        assert china_name == self.country
+        assert level == "country"
+
+        privinces: list[dict] = china_info["districts"]
+
+        return privinces
+
+    def get_city(self, privince_name: str, privince_adcode: str) -> list[dict]:
+        """获取指定省份的所有城市
+        privince_name: 省份名称
+        privince_adcode: 省份行政区域编码
+
+        但是会跳过:香港特别行政区,澳门特别行政区,这2个属于省单位，但是没有市直接是区
+        因为这2个省级单位下面单位直接是 九龙城区 这种区，没有市所以直接跳过
+
+
+        返回:
+        1. 情况1
+            "citycode": "0379", # 城市编码
+            "adcode": "410300", # 行政区域编码
+            "name": "洛阳市",
+            "center": "112.453895,34.619702",
+            "level": "city",
+            "districts": []
+        2. 情况2
+            _message : "该省单位下面没有市"
+        """
+
+        # 跳过对 香港、澳门 下面的区,进行获取区下面(的区)
+        if privince_adcode in self._.keys():
+            print(f"跳过对 {self._.get(privince_adcode)} 获取下级(市)行政区")
+            return [{"_message": "该省单位下面没有市"}]
+
+        url = f"{self.base_url}?key={self.key}&keywords={privince_name}"
+        privince_info = self._send_request(url)
+
+        # chekc
+        privince_adcode_res = privince_info["adcode"]
+        privince_name_res = privince_info["name"]
+        level = privince_info["level"]
+        assert level == "province"
+        assert privince_name_res == privince_name
+        assert privince_adcode_res == privince_adcode
+
+        city_info: list[dict] = privince_info["districts"]
+        return city_info
+
+    def get_district(
+        self, city_name: str, city_code: str, city_adcode: str
+    ) -> list[dict]:
+        """获取指定城市的所有区/县
+        city_name: 城市名称
+        city_code: 城市编码
+        city_adcode: 城市行政区域编码
+
+        返回结果:
+        "citycode": "0379",
+        "adcode": "410323",
+        "name": "新安县",
+        "center": "112.13246,34.728909",
+        "level": "district",
+        "districts": []
+        """
+        url = f"{self.base_url}?key={self.key}&keywords={city_name}"
+        city_info = self._send_request(
+            url, name=city_name, adcode=city_adcode, citycode=city_code
+        )
+
+        # check
+        city_name_res = city_info["name"]
+        city_code_res = city_info["citycode"]
+        city_adcode_res = city_info["adcode"]
+        level = city_info["level"]
+
+        # 如果是香港、澳门,则level为province
+        # --- 暂时不对比citycode了
+        if city_name in self._.values() and city_adcode in self._.keys():
+            assert level == "province"
+        else:
+            assert level == "city"
+
+        assert city_name_res == city_name
+        assert city_code_res == city_code
+        assert city_adcode_res == city_adcode
+
+        district_info: list[dict] = city_info["districts"]
+        return district_info

jwf_common_python/geography/search_amap.py

@@ -0,0 +1,84 @@
+import time
+import requests
+
+
+class SearchLocationAmap:
+    """高德地图API搜索地理位置
+    - 使用2.0版本的API
+    """
+
+    def __init__(self, key: str):
+        self.key = key
+        self.page_size = 10
+        self.types = "汽车服务|汽车销售|汽车维修|摩托车服务|餐饮服务|购物服务|生活服务\
+|体育休闲服务|医疗保健服务|住宿服务|风景名胜|商务住宅|政府机构及社会团体|科教文化服务\
+|交通设施服务|金融保险服务|公司企业|道路附属设施|地名地址信息|公共设施|事件活动|室内设施|虚拟数据|通行设施"
+        self.base_url = f"https://restapi.amap.com/v5/place/text?key={self.key}\
+&types={self.types}&city_limit=true&page_size={self.page_size}"
+
+    def _request(self, url: str):
+        response = requests.get(url)
+        if response.status_code != 200:
+            raise Exception(f"请求失败，状态码：{response.status_code}")
+
+        res = response.json()
+        if res["info"] != "OK":
+            raise Exception(f"请求失败，错误信息：{res['info']}")
+        return res
+
+    def search(
+        self,
+        keyword: str,
+        city_name: str,
+    ) -> list[dict]:
+        """搜索指定城市,指定地理位置.
+        keywords: 地点关键字,需要被检索的地点文本信息。
+        city_name: 搜索区划, citycode，adcode，cityname；cityname 仅支持城市级别和中文，如“北京市”。
+
+        返回数据示例:
+        {
+            "parent": "",
+            "address": "通衢路19号",
+            "distance": "",
+            "pcode": "410000",
+            "adcode": "410311",
+            "pname": "河南省",
+            "cityname": "洛阳市",
+            "type": "交通设施服务;火车站;火车站",
+            "typecode": "150200",
+            "adname": "洛龙区",
+            "citycode": "0379",
+            "name": "洛阳龙门站",
+            "location": "112.456291,34.593842",
+            "id": "B017B02U7H"
+        },
+        """
+        # check
+        # 文本总长度不可超过80字符，这里限制为50
+        if len(keyword) > 50:
+            raise ValueError("keywords长度不能超过50")
+
+        res = []
+        num = 1
+
+        while True:
+            # 1. 获取数据
+            url = (
+                f"{self.base_url}&keywords={keyword}&region={city_name}&page_num={num}"
+            )
+            response = self._request(url)
+
+            # 2. 更新数据
+            pois = response["pois"]
+            res.extend(pois)
+
+            # 3. 判断是否进行下一页的数据获取
+            if int(response["count"]) != self.page_size:
+                # 本次请求返回的数据个数 不等于 page_size,说明本次请求是最后一页. 数据获取完毕
+                break
+            else:
+                # 存在下一页, 更新num
+                num += 1
+                # 限制请求次数,避免出现1秒超过3次的限制
+                time.sleep(1)
+        return res

jwf_common_python/github/__init__.py

@@ -0,0 +1,5 @@
+"""GitHub Webhook 验签和可选 HTTP 接入工具。"""
+
+from .webhooks import create_webhook_app, verify_signature
+
+__all__ = ["verify_signature", "create_webhook_app"]

jwf_common_python/github/webhooks.py

@@ -0,0 +1,87 @@
+"""基于原始请求体的 GitHub SHA-256 验签，不记录密钥或签名。"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import re
+from collections.abc import Callable
+
+
+def _secret_bytes(secret: str | bytes) -> bytes:
+    if isinstance(secret, str):
+        secret = secret.encode("utf-8")
+    if not isinstance(secret, bytes):
+        raise TypeError("secret 必须为 str 或 bytes")
+    if not secret:
+        raise ValueError("secret 不能为空")
+    return secret
+
+
+def verify_signature(
+    payload: bytes, secret: str | bytes, signature_header: str | None
+) -> bool:
+    """校验 X-Hub-Signature-256，返回 bool；密钥必须由调用方传入。
+
+    payload 必须为原始请求体字节，不能是重新序列化的 JSON。
+    签名缺失或格式错误时返回 False，配置/参数类型错误时抛出异常。
+    """
+    key = _secret_bytes(secret)
+    if not isinstance(payload, bytes):
+        raise TypeError("payload 必须为原始请求体 bytes")
+    if (
+        not isinstance(signature_header, str)
+        or re.fullmatch(r"sha256=[0-9a-f]{64}", signature_header) is None
+    ):
+        return False
+    expected = "sha256=" + hmac.new(key, payload, hashlib.sha256).hexdigest()
+    return hmac.compare_digest(expected, signature_header)
+
+
+def create_webhook_app(
+    secret: str | bytes,
+    *,
+    path: str = "/hook",
+    handler: Callable[[bytes, str | None, str | None], None] | None = None,
+):
+    """返回 Flask 应用，不启动服务器；只有调用此函数才导入 Flask。
+
+    只接受 POST。验签失败返回 403，成功时依次调用
+    handler(payload, event, delivery_id)，然后返回 {"status": 200}。
+    请求体不强制为 JSON，回调自行解析；回调失败由 Flask 按异常处理，不返回成功。
+    """
+    key = _secret_bytes(secret)
+    if (
+        not isinstance(path, str)
+        or not path.startswith("/")
+        or "<" in path
+        or ">" in path
+    ):
+        raise ValueError("path 必须为以 / 开头的静态路由路径")
+    if handler is not None and not callable(handler):
+        raise TypeError("handler 必须可调用或为 None")
+    try:
+        from flask import Flask, jsonify, request
+    except ImportError as exc:
+        raise ImportError("HTTP 接入需要可选依赖：pip install flask") from exc
+
+    app = Flask(__name__)
+
+    def webhook():
+        payload = request.get_data(cache=True)
+        if not verify_signature(
+            payload, key, request.headers.get("X-Hub-Signature-256")
+        ):
+            return jsonify({"error": "invalid_signature"}), 403
+        if handler is not None:
+            handler(
+                payload,
+                request.headers.get("X-GitHub-Event"),
+                request.headers.get("X-GitHub-Delivery"),
+            )
+        return jsonify({"status": 200})
+
+    app.add_url_rule(
+        path, endpoint="github_webhook", view_func=webhook, methods=["POST"]
+    )
+    return app

jwf_common_python/language_utils/__init__.py

@@ -0,0 +1,34 @@
+"""英文文本取词、词表处理与网页取词工具。"""
+
+from .core import (
+    extract_chapter,
+    extract_words,
+    filter_words,
+    format_words,
+    get_chapter_words,
+    group_words_by_length,
+    is_title_case_word,
+    parse_urls,
+    sort_words_by_length,
+    word_frequencies,
+)
+from .files import generate_file_menu, read_urls, read_word_lists, read_words
+from .web import fetch_words_from_pages
+
+__all__ = [
+    "extract_words",
+    "filter_words",
+    "extract_chapter",
+    "get_chapter_words",
+    "word_frequencies",
+    "format_words",
+    "group_words_by_length",
+    "sort_words_by_length",
+    "is_title_case_word",
+    "parse_urls",
+    "read_words",
+    "read_word_lists",
+    "read_urls",
+    "generate_file_menu",
+    "fetch_words_from_pages",
+]