jwf-common-python 1.0.0__py3-none-any.whl

jwf_common_python/message_queue/kafka.py

@@ -0,0 +1,145 @@
+"""基于 kafka-python 的同步生产和消费。"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .interface import MessageCallback, MessageQueue
+
+
+class KafkaQueue(MessageQueue):
+    def __init__(
+        self,
+        bootstrap_servers: list[str],
+        topic: str,
+        *,
+        group_id: str | None = None,
+        timeout: float = 10,
+        producer_config: dict[str, Any] | None = None,
+        consumer_config: dict[str, Any] | None = None,
+    ) -> None:
+        """连接按需创建，消费时必须提供 group_id。
+
+        config 参数透传 SDK 配置，例如 SSL/SASL；连接地址、消费组和
+        手动确认配置由本类管理，不能通过 config 覆盖。
+        """
+        super().__init__()
+        if not isinstance(bootstrap_servers, list) or not bootstrap_servers:
+            raise ValueError("bootstrap_servers 必须为非空 list[str]")
+        if not all(
+            isinstance(server, str) and server.strip() for server in bootstrap_servers
+        ):
+            raise ValueError("bootstrap_servers 必须为非空 list[str]")
+        if not isinstance(topic, str) or not topic.strip():
+            raise ValueError("topic 不能为空")
+        if group_id is not None and (
+            not isinstance(group_id, str) or not group_id.strip()
+        ):
+            raise ValueError("group_id 不能为空")
+        if timeout <= 0:
+            raise ValueError("timeout 必须大于 0")
+        self._producer_config = dict(producer_config or {})
+        self._consumer_config = dict(consumer_config or {})
+        if "bootstrap_servers" in self._producer_config:
+            raise ValueError("请通过 bootstrap_servers 参数设置连接地址")
+        if {
+            "bootstrap_servers",
+            "group_id",
+            "enable_auto_commit",
+        } & self._consumer_config.keys():
+            raise ValueError("consumer_config 不能覆盖连接地址、消费组或手动确认配置")
+
+        try:
+            import kafka
+            from kafka.structs import OffsetAndMetadata, TopicPartition
+        except ImportError as exc:
+            raise ImportError("请先安装 Kafka 依赖：pip install kafka-python") from exc
+
+        self._kafka = kafka
+        self._offset_metadata = OffsetAndMetadata
+        self._topic_partition = TopicPartition
+        self.bootstrap_servers = list(bootstrap_servers)
+        self.topic = topic
+        self.group_id = group_id
+        self.timeout = timeout
+        self._producer = None
+
+    @property
+    def producer(self):
+        """首次发送或访问此属性时创建生产者。"""
+        self._ensure_open()
+        if self._producer is None:
+            config = {
+                "acks": "all",
+                "retries": 5,
+                "max_in_flight_requests_per_connection": 1,
+                **self._producer_config,
+            }
+            self._producer = self._kafka.KafkaProducer(
+                bootstrap_servers=self.bootstrap_servers, **config
+            )
+        return self._producer
+
+    def enqueue(self, item: bytes, *, key: bytes | None = None) -> dict[str, Any]:
+        """同步发送并返回 topic、partition、offset；支持二进制内容和空 key。"""
+        self._ensure_open()
+        self._validate_item(item)
+        if key is not None and not isinstance(key, bytes):
+            raise TypeError("key 必须为 bytes 或 None")
+        metadata = self.producer.send(self.topic, value=item, key=key).get(
+            timeout=self.timeout
+        )
+        return {
+            "topic": metadata.topic,
+            "partition": metadata.partition,
+            "offset": metadata.offset,
+        }
+
+    def enqueue_json(self, message: Any, *, key: bytes | None = None) -> dict[str, Any]:
+        """发送 UTF-8 JSON，等待确认并返回 topic、partition、offset。
+
+        不可序列化的内容或 NaN/Infinity 在连接前报错；不能同时配置
+        value_serializer，避免对已编码 JSON 二次序列化。None 发送为 JSON null。
+        """
+        self._ensure_open()
+        if self._producer_config.get("value_serializer") is not None:
+            raise ValueError("enqueue_json 负责序列化，不能再配置 value_serializer")
+        payload = json.dumps(
+            message, ensure_ascii=False, allow_nan=False, separators=(",", ":")
+        ).encode("utf-8")
+        return self.enqueue(payload, key=key)
+
+    def consumer(self, callback: MessageCallback) -> None:
+        self._ensure_open()
+        if not callable(callback):
+            raise TypeError("callback 必须可调用")
+        if self.group_id is None:
+            raise ValueError("消费时必须提供 group_id，用于提交消费进度")
+        client = self._kafka.KafkaConsumer(
+            self.topic,
+            bootstrap_servers=self.bootstrap_servers,
+            group_id=self.group_id,
+            enable_auto_commit=False,
+            **self._consumer_config,
+        )
+        try:
+            for message in client:
+                callback(message.value)
+                # 只提交已处理消息所在分区的下一条 offset，避免确认预取的其他消息。
+                partition = self._topic_partition(message.topic, message.partition)
+                offset = {"offset": message.offset + 1, "metadata": ""}
+                # kafka-python 2.1+ 的 OffsetAndMetadata 新增了必填 leader_epoch。
+                if "leader_epoch" in self._offset_metadata._fields:
+                    offset["leader_epoch"] = getattr(message, "leader_epoch", -1)
+                client.commit(offsets={partition: self._offset_metadata(**offset)})
+        finally:
+            client.close(autocommit=False)
+
+    def close(self) -> None:
+        if not self._closed:
+            try:
+                if self._producer is not None:
+                    self._producer.close(timeout=self.timeout)
+            finally:
+                self._closed = True

jwf_common_python/message_queue/rabbitmq.py

@@ -0,0 +1,113 @@
+"""基于 pika BlockingConnection 的同步生产和消费。"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .interface import MessageCallback, MessageQueue
+
+
+class RabbitmqQueue(MessageQueue):
+    def __init__(
+        self,
+        queue_name: str,
+        host: str,
+        port: int | str = 5672,
+        *,
+        username: str,
+        password: str,
+        durable: bool = True,
+        prefetch_count: int = 1,
+        requeue_on_error: bool = True,
+        connection_config: dict[str, Any] | None = None,
+    ) -> None:
+        """默认持久化队列和消息，并启用发布确认。
+
+        若已有同名非持久化队列，应传 durable=False，与服务端保持一致。
+        connection_config 可传 virtual_host、SSL 等 pika 连接配置。
+        """
+        super().__init__()
+        if not isinstance(queue_name, str) or not queue_name.strip():
+            raise ValueError("queue_name 不能为空")
+        if not isinstance(host, str) or not host.strip():
+            raise ValueError("host 不能为空")
+        port = int(port)
+        if not 1 <= port <= 65535:
+            raise ValueError("port 必须在 1 到 65535 之间")
+        if not isinstance(prefetch_count, int) or prefetch_count <= 0:
+            raise ValueError("prefetch_count 必须为正整数")
+        config = dict(connection_config or {})
+        if {"host", "port", "credentials"} & config.keys():
+            raise ValueError("请通过独立参数设置 host、port、username 和 password")
+        try:
+            import pika
+        except ImportError as exc:
+            raise ImportError("请先安装 RabbitMQ 依赖：pip install pika") from exc
+
+        self.pika = pika
+        self.queue_name = queue_name
+        self.durable = durable
+        self.prefetch_count = prefetch_count
+        self.requeue_on_error = requeue_on_error
+        self.connect = pika.BlockingConnection(
+            pika.ConnectionParameters(
+                host=host,
+                port=port,
+                credentials=pika.PlainCredentials(username, password),
+                **{"blocked_connection_timeout": 30, **config},
+            )
+        )
+        try:
+            self.channel = self.connect.channel()
+            self.channel.queue_declare(queue=queue_name, durable=durable)
+            self.channel.confirm_delivery()
+        except BaseException:
+            self.close()
+            raise
+
+    def enqueue(self, item: bytes) -> None:
+        self._ensure_open()
+        self._validate_item(item)
+        self.channel.basic_publish(
+            exchange="",
+            routing_key=self.queue_name,
+            body=item,
+            properties=self.pika.BasicProperties(
+                delivery_mode=2 if self.durable else 1
+            ),
+            mandatory=True,
+        )
+
+    def consumer(self, callback: MessageCallback) -> None:
+        self._ensure_open()
+        if not callable(callback):
+            raise TypeError("callback 必须可调用")
+
+        def on_message(channel, method, properties, body):
+            try:
+                callback(body)
+            except Exception:
+                channel.basic_nack(
+                    delivery_tag=method.delivery_tag, requeue=self.requeue_on_error
+                )
+                raise
+            else:
+                channel.basic_ack(delivery_tag=method.delivery_tag)
+
+        try:
+            self.channel.basic_qos(prefetch_count=self.prefetch_count)
+            self.channel.basic_consume(
+                queue=self.queue_name, on_message_callback=on_message, auto_ack=False
+            )
+            self.channel.start_consuming()
+        finally:
+            # 关闭连接也会使尚未确认的消息重新可消费，避免遗留活动消费者。
+            self.close()
+
+    def close(self) -> None:
+        if not self._closed:
+            try:
+                if self.connect.is_open:
+                    self.connect.close()
+            finally:
+                self._closed = True

jwf_common_python/oauth2/google.py

@@ -0,0 +1,158 @@
+import requests
+
+
+class Oauth2Google:
+    def __init__(
+        self, client_id: str, client_secret: str, redirect_uri: str, scope: str
+    ):
+        self.client_id = client_id
+        self.client_secret = client_secret
+        self.response_type = "code"
+        self.access_type = "online"
+        self.grant_type = "authorization_code"
+        # 认证完成之后的回调地址,必须与google console中配置的一致
+        self.redirect_uri = redirect_uri
+        # 请求的权限
+        self.scope = scope
+
+    def get_web_auth_url(self) -> str:
+        """获取web页面上认证地址"""
+        url = f"https://accounts.google.com/o/oauth2/v2/auth?\
+client_id={self.client_id}&response_type={self.response_type}&access_type={self.access_type}\
+&redirect_uri={self.redirect_uri}&scope={self.scope}"
+        return url
+
+    def get_access_token(self, code: str) -> str:
+        """根据code获取access_token"""
+        url = "https://oauth2.googleapis.com/token"
+        data = {
+            "client_id": self.client_id,
+            "client_secret": self.client_secret,
+            "code": code,
+            "grant_type": self.grant_type,
+            "redirect_uri": self.redirect_uri,
+        }
+        response = requests.post(
+            url,
+            data=data,
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+        )
+        if response.status_code != 200:
+            raise Exception(f"获取access_token失败 {response.status_code} {response.text}")
+        res = response.json()
+        access_token = res["access_token"]
+        return access_token
+
+    def get_raw_user_info(self, access_token: str) -> dict:
+        """获取原生的用户信息
+
+        email: 用户邮箱
+        name: 用户名字
+        avatar_content: 用户头像字节流
+        """
+        url = "https://people.googleapis.com/v1/people/me?personFields=\
+names,emailAddresses,genders,nicknames,phoneNumbers,photos"
+        response = requests.get(
+            url, headers={"Authorization": f"Bearer {access_token}"}
+        )
+        if response.status_code != 200:
+            raise Exception(f"获取用户信息失败 {response.status_code} {response.text}")
+
+        res = response.json()
+        return res
+
+    def format_user_info(self, raw_user_info: dict) -> dict:
+        """格式化用户信息"""
+        _ = {
+            "resourceName": "people/110311852157832716656",
+            "etag": "%EgwBAxxxPj8axxxxBQc=",
+            "names": [
+                {
+                    "metadata": {
+                        "primary": True,
+                        "source": {"type": "PROFILE", "id": "1185215783"},
+                        "sourcePrimary": True,
+                    },
+                    "displayName": "姜xx",
+                    "familyName": "姜",
+                    "givenName": "xx",
+                    "displayNameLastFirst": "姜xx",
+                    "unstructuredName": "姜xx",
+                }
+            ],
+            "photos": [
+                {
+                    "metadata": {
+                        "primary": True,
+                        "source": {"type": "PROFILE", "id": "1215783xx6"},
+                    },
+                    "url": "https://lh3.googleusercontent.com/a/\
+ACg8ocLGxxxxkIpZx7vEfeD31o=s100",
+                }
+            ],
+            "emailAddresses": [
+                {
+                    "metadata": {
+                        "primary": True,
+                        "verified": True,
+                        "source": {"type": "ACCOUNT", "id": "11xxxxxx656"},
+                        "sourcePrimary": True,
+                    },
+                    "value": "xxxx@gmail.com",
+                },
+                {
+                    "metadata": {
+                        "verified": True,
+                        "source": {"type": "ACCOUNT", "id": "1103xxxxxx56"},
+                    },
+                    "value": "xxxx@xxx.com",
+                },
+            ],
+        }
+        # 1. 获取用户邮箱
+        email = None
+        emailAddresses = raw_user_info["emailAddresses"]
+        for email_info in emailAddresses:
+            # 获取主邮箱
+            metadata = email_info["metadata"]
+            if metadata.get("primary"):
+                # 当primary存在，且为True时，表示为主邮箱
+                email = email_info["value"]
+                break
+
+        # 2. 获取用户名字
+        name = None
+        names = raw_user_info["names"]
+        for name_info in names:
+            if name_info.get("displayName"):
+                # 当displayName存在时，表示为用户名
+                name = name_info["displayName"]
+                break
+
+        # 3. 获取用户头像
+        photo_url = None
+        photos = raw_user_info["photos"]
+        for photo_info in photos:
+            if photo_info.get("url"):
+                # 当url存在时，表示为用户头像
+                photo_url = photo_info["url"]
+                break
+        # 3.1 如果有头像,获取图片字节流
+        avatar_content = None
+        if photo_url:
+            response = requests.get(photo_url)
+            if response.status_code == 200:
+                avatar_content = response.content
+
+        return {
+            "email": email,
+            "name": name,
+            "avatar_content": avatar_content,
+        }
+
+    def get_user_info(self, code: str) -> dict:
+        """获取用户信息,格式化后返回"""
+        access_token = self.get_access_token(code)
+        raw_user_info = self.get_raw_user_info(access_token)
+        user_info = self.format_user_info(raw_user_info)
+        return user_info

jwf_common_python/parse_subtitle/__init__.py

@@ -0,0 +1,6 @@
+# [参考](https://github.com/cdown/srt)
+# [参考](https://srt.readthedocs.io/en/latest/quickstart.html#compose-an-srt-from-python-objects)
+
+# 1. 清理非标准字符
+# 2. 使用srt解析
+# 3. 清理每个字幕中的非标准字符