jwf-common-python 1.0.0__py3-none-any.whl

jwf_common_python/parse_subtitle/parse_srt_file.py

@@ -0,0 +1,60 @@
+class ParseSrtFile:
+    def __init__(self, file_name: str):
+        self.file_name = file_name
+        with open(self.file_name) as f:
+            content = f.readlines()
+            self.content: list = self.clean_non_text_lines(content)
+
+    def clean_non_text_lines(self, file_content: list) -> list:
+        """清理控制字符和空行"""
+        # 清理\n控制字符
+        file_content = list(map(lambda line: line.strip("\n"), file_content))
+        # 清理空行
+        file_content = list(filter(lambda line: line != "", file_content))
+        return file_content
+
+    def get_all_timer_shaft(self) -> list[tuple[str, str]]:
+        """获取所有时间轴
+        [
+            (start_time,end_time)
+        ]
+        """
+        # `:` 和 `-->` 同时存在的行就是时间轴所在的行
+        time_shaft_content: list = list(
+            filter(lambda line: ":" in line and "-->" in line, self.content)
+        )
+
+        # 将时间轴数据拆分为
+        def _clean(time_str: str):
+            return tuple(map(lambda x: x.strip(" "), time_str.split("-->")))
+
+        time_shaft_data: list = list(
+            map(
+                lambda time_str_line: _clean(time_str_line),
+                time_shaft_content,
+            )
+        )
+        import json
+
+        print(json.dumps(time_shaft_data, indent=2, ensure_ascii=False))
+        return time_shaft_data
+
+
+# p = ParseSrtFile("test.srt")
+# p.get_all_timer_shaft()
+import logging
+import os
+
+
+def split(time_range, video_name):
+    """单线程阻塞切割"""
+    for index, time in enumerate(time_range):
+        # cmd = f"ffmpeg -i 输入视频.mp4 -ss 00:02:10 -to 00:03:20 -c:v copy -c:a copy 输出视频.mp4"
+        cmd = f"ffmpeg -i {video_name} -ss {time[0]} -to {time[1]} -c:v copy -c:a copy {index}.mp4"
+        try:
+            os.system(cmd)
+        except Exception as e:
+            logging.error(f"出错: {cmd} {e}")
+        else:
+            logging.debug(f"cmd: {cmd}")
+            logging.info(f"{time}处理成功")

jwf_common_python/pdf/aaa.py

@@ -0,0 +1,38 @@
+[
+    BlockStruClean(
+        lines=[
+            LineStruClean(
+                spans=[SpanStruClean(text="Setting Up React and TypeScript")]
+            ),
+            LineStruClean(spans=[SpanStruClean(text="82")]),
+        ]
+    ),
+    BlockStruClean(
+        lines=[
+            LineStruClean(spans=[SpanStruClean(text="4.\t")]),
+            LineStruClean(
+                spans=[
+                    SpanStruClean(text="Similarly, install a Babel plugin called "),
+                    SpanStruClean(text=" that enables TypeScript "),
+                ]
+            ),
+            LineStruClean(
+                spans=[SpanStruClean(text="code to be transformed into JavaScript:")]
+            ),
+        ]
+    ),
+]
+
+[
+    LineStruClean(spans=[SpanStruClean(text="4.\t")]),
+    LineStruClean(
+        spans=[
+            SpanStruClean(text="Similarly, install a Babel plugin called "),
+            SpanStruClean(text="@babel/preset-typescript"),
+            SpanStruClean(text=" that enables TypeScript "),
+        ]
+    ),
+    LineStruClean(
+        spans=[SpanStruClean(text="code to be transformed into JavaScript:")]
+    ),
+]

jwf_common_python/pdf/local_parse.py

@@ -0,0 +1,259 @@
+from itertools import chain
+
+import pymupdf
+from pydantic import BaseModel
+from pymupdf import Document, Page, Pixmap
+
+
+# span 结构:
+class SpanStruClean(BaseModel):
+    text: str
+    """
+    _span_stru = {
+        "size": 10.0,
+        "flags": 4,
+        "font": "MinionPro-Regular",
+        "color": 0,
+        "ascender": 0.9890000224113464,
+        "descender": -0.36000001430511475,
+        "text": "Next, we will glue everything together with webpack.",
+        "origin": (81.0, 573.5889892578125),
+        "bbox": (81.0, 563.698974609375, 291.9960632324219, 577.18896484375),
+    }
+    """
+
+
+# 行结构
+class LineStruClean(BaseModel):
+    spans: list[SpanStruClean]
+    """
+    _line_stru = {
+        "spans": [
+            _span_stru,
+        ],
+        "wmode": 0,
+        "dir": (1.0, 0.0),
+        "bbox": (81.0, 563.698974609375, 291.9960632324219, 577.18896484375),
+    }
+    """
+
+
+# 块结构
+class BlockStruClean(BaseModel):
+    lines: list[LineStruClean]
+    """raw:
+    _block_stru = {
+        "number": 10,
+        "type": 0,
+        "bbox": (81.0, 563.698974609375, 291.9960632324219, 577.18896484375),
+        "lines": [_line_stru],
+    }
+    """
+
+    # 页结构
+    # class PageStruClean(BaseModel):
+    #     blocks: list[BlockStruClean]
+
+    # """
+    # raw页结构
+    # _page_stru = {
+    #     "width": 549.0,
+    #     "height": 684.0,
+    #     "blocks": [
+    #         _block_stru,
+    #     ],
+    # }
+    # """
+
+
+# 单个目录信息
+class RawCatalogueStru(BaseModel):
+    # level: 目录级别,也是目录缩进的空格数
+    level: int
+    # title: 目录标题
+    title: str
+    # page_num: 目录页码
+    page_num: int
+
+
+class CatalogueStru(BaseModel):
+    """子目录结构(仅有2级目录)"""
+
+    title: str
+    page_num: int
+
+
+class FatherCatalogueStru(BaseModel):
+    """根目录结构"""
+
+    title: str
+    page_num: int
+    children: list[CatalogueStru]
+
+
+class ParsePDF:
+    """
+    sample file:
+    """
+
+    def __init__(self, file_name: str):
+        self.doc: Document = pymupdf.open(file_name)
+        self.page_count = self.doc.page_count
+
+    def get_cover_image(self) -> bytes:
+        """获取封面图片,返回图片的字节数据"""
+        page: Page = self.doc[0]
+
+        pix: Pixmap = page.get_pixmap()
+
+        res: bytes = pix.tobytes()
+        return res
+
+    def _get_raw_catalogue(self) -> list[RawCatalogueStru]:
+        """获取pdf的目录信息"""
+        catalogue_info: list[list] = self.doc.get_toc(simple=False)
+
+        res = [
+            RawCatalogueStru(level=item[0], title=item[1], page_num=item[2])
+            for item in catalogue_info
+        ]
+
+        return res
+
+    def get_catalogue(self) -> list[FatherCatalogueStru]:
+        """获取经过处理的目录信息
+
+        返回结果数据sample:
+        [
+            FatherCatalogueStru(
+                title='Preface',
+                page_num=14,
+                children=[]
+            ),
+            FatherCatalogueStru(
+                title='Chapter 1: Introducing React',
+                page_num=22,
+                children=[
+                    CatalogueStru(title='Technical requirements', page_num=23),
+                    CatalogueStru(title='Understanding the benefits of React', page_num=23),
+                    CatalogueStru(title='Understanding JSX', page_num=24),
+                ]
+            )
+        ]
+        """
+        raw_catalogue = self._get_raw_catalogue()
+
+        # 1. 过滤目录层级超过2的目录,只能保留1和2级目录
+        raw_catalogue_only = list(filter(lambda x: x.level <= 2, raw_catalogue))
+
+        # 2. 生成目录树
+        # 2.1 将level是1的目录作为根目录,level是2的目录作为子目录
+        res = []
+        for catelogue in raw_catalogue_only:
+            if catelogue.level == 1:
+                f = FatherCatalogueStru(
+                    title=catelogue.title, page_num=catelogue.page_num, children=[]
+                )
+                res.append(f)
+
+            if catelogue.level == 2:
+                # 获取最后一个目录块，就是当前的根目录
+                current_fa = res.pop()
+                # 获取当前的根目录的子目录
+                current_children = current_fa.children
+
+                # 获取新的子目录
+                cate = CatalogueStru(title=catelogue.title, page_num=catelogue.page_num)
+
+                # 追加新的子目录
+                current_children.append(cate)
+
+                # 重新添加到字典中
+                current_fa.children = current_children
+
+                res.append(current_fa)
+        return res
+
+    def _get_raw_page_info(self, page_num: int) -> list[BlockStruClean]:
+        """获取指定页中的原生页面信息"""
+        page: Page = self.doc[page_num]
+        # 1. 已文本形式获取页面信息
+        page_info: dict = page.get_text("dict")
+
+        # 2. raw blocks
+        raw_blocks = page_info["blocks"]
+        # print("----", len(raw_blocks), raw_blocks[0], "----")
+
+        # 3. format block
+        format_blocks = [BlockStruClean(lines=block["lines"]) for block in raw_blocks]
+        return format_blocks
+
+    def _get_block_text(self, block_info: BlockStruClean) -> list[str]:
+        """获取指定块中每行文本信息
+        返回值:
+        [
+            "该块的第一行文字",
+            "该块的第二行文字",
+        ]
+        """
+        # 1. 获取块中的全部行信息
+        lines: list[LineStruClean] = block_info.lines
+
+        # 2. 获取每行的文字信息
+        lines_text = []
+        for line in lines:
+            # 1. 获取每行的span
+            spans: list[SpanStruClean] = line.spans
+            # 1. 获取每个span的文字
+            spans_text = [span.text for span in spans]
+            # 2. 合并所有span的文字. 获取指定行的所有文字信息
+            line_text = "".join(spans_text)
+            lines_text.append(line_text)
+        return lines_text
+
+    def get_page_text(self, page_num: int):
+        """从页面中获取原生的文本信息
+        func 是判断是否合并的函数,默认如果不传入,则不合并，展开二维数组
+        """
+
+        # 1. 获取页面中的所有块信息
+        raw_page_info = self._get_raw_page_info(page_num)
+
+        # 2. 获取页面的原生文本信息
+        raw_page_text: list[list[str]] = [
+            self._get_block_text(block) for block in raw_page_info
+        ]
+
+        # # 3 使用ai判断,是否将这些块进行合并
+        # # 这些行合并到一起是不是普通文本，如果是，则合并，否则不合并
+        # for block_text in raw_page_text:
+        #     # 1. 合并块中的所有行
+        #     block_text_single = " ".join(block_text)
+
+        #     # 2. 使用func判断合并完成后的文本是否是普通文本
+        #     msg = f"请判断下面这些文本是代码还是普通文本:\n {block_text_single}"
+        #     status = func(msg)
+
+        #     # 3. 确定最终的文本信息
+        #     if status:
+        #         # 是普通文本
+        #     else:
+        #         ...
+        # 3. 将二维数组展开
+        page_text = list(chain(*raw_page_text))
+        return page_text
+
+
+# if __name__ == "__main__":
+#     pdf = ParsePDF("/Users/jason/temp/test-pdf.pdf")
+#     # blocks = pdf._get_raw_page_info(100)
+#     # block = blocks[1]
+#     # pdf._get_block_text(block)
+#     print(pdf.get_page_text(100))
+# import json
+
+# print(json.dumps(pdf.get_top_catalogue(), indent=2))
+# print(pdf.get_cover_image())
+
+# with open("/Users/evans/Desktop/cover2.png", "wb") as f:
+#     f.write(pdf.get_cover_image())

jwf_common_python/sms/__init__.py

@@ -0,0 +1 @@
+from .tencent_sms import TencentSendSms

jwf_common_python/sms/interface.py

@@ -0,0 +1,7 @@
+from abc import ABCMeta, abstractmethod
+
+
+class SendSms(metaclass=ABCMeta):
+    @abstractmethod
+    def send(self, send_phone_number: str, veri_code: str):
+        ...

jwf_common_python/sms/tencent_sms.py

@@ -0,0 +1,117 @@
+import logging
+
+from tencentcloud.common import credential
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import (
+    TencentCloudSDKException,
+)
+from tencentcloud.common.profile.client_profile import ClientProfile
+from tencentcloud.common.profile.http_profile import HttpProfile
+from tencentcloud.sms.v20210111 import models, sms_client
+from tencentcloud.sms.v20210111.models import SendSmsResponse, SendStatus
+
+from jwf_common_python.sms.interface import SendSms
+
+
+class TencentSendSms(SendSms):
+    def __init__(self, **kwargs) -> None:
+        """
+        发送成功:
+        {
+            "SendStatusSet": [
+                {
+                "SerialNo": "2640:222395940116893272585707425",
+                "PhoneNumber": "+8618285574257",
+                "Fee": 1,
+                "SessionContext": "user session",
+                "Code": "Ok",
+                "Message": "send success",
+                "IsoCode": "CN"
+                }
+            ],
+            "RequestId": "bbc9e3e8-a67e-4ed0-9e85-713cda1ae041"
+        }
+
+        发送失败:
+         {
+            "SendStatusSet": [
+                {
+                "SerialNo": "",
+                "PhoneNumber": "+8618285574257",
+                "Fee": 0,
+                "SessionContext": "user session",
+                "Code": "FailedOperation.InsufficientBalanceInSmsPackage",
+                "Message": "insufficient balance in SMS package",
+                "IsoCode": ""
+                }
+            ],
+            "RequestId": "a28c5e46-5209-4425-8bfe-97784d205a52"
+        }
+        """
+        if not {
+            "secret_id",
+            "secret_key",
+            "app_id",
+            "sign_name",
+            "template_id",
+            "endpoint",
+            "veri_code_timeout",
+        }.issubset(kwargs.keys()):
+            raise ValueError(
+                "secret_id,secret_key,app_id,sign_name,template_id,endpoint,veri_code_timeout must be provided"
+            )
+        self.secret_id = kwargs["secret_id"]
+        self.secret_key = kwargs["secret_key"]
+        self.app_id = kwargs["app_id"]
+        self.sign_name = kwargs["sign_name"]
+        self.template_id = kwargs["template_id"]
+        self.endpoint = kwargs["endpoint"]
+        self.veri_code_timeout = kwargs["veri_code_timeout"]
+
+    def send(self, send_phone_number: str, veri_code: str, session_context: str = ""):
+        """
+        +8618285574257
+        2222
+
+        session_context:
+         用户的 session 内容（无需要可忽略）: 可以携带用户侧 ID 等上下文信息，server 会原样返回
+        """
+        try:
+            cred = credential.Credential(self.secret_id, self.secret_key)
+            httpProfile = HttpProfile()
+            httpProfile.reqMethod = "POST"
+            httpProfile.reqTimeout = 30
+            httpProfile.endpoint = self.endpoint
+
+            # 实例化一个客户端配置对象，可以指定超时时间等配置
+            clientProfile = ClientProfile()
+            clientProfile.signMethod = "TC3-HMAC-SHA256"  # 指定签名算法
+            clientProfile.language = "en-US"
+            clientProfile.httpProfile = httpProfile
+
+            client = sms_client.SmsClient(cred, "ap-guangzhou", clientProfile)
+
+            req = models.SendSmsRequest()
+
+            req.SmsSdkAppId = self.app_id
+            req.SignName = self.sign_name
+            # req.TemplateId = "1941616"
+            req.TemplateId = self.template_id
+            # 模板参数: 模板参数的个数需要与 TemplateId 对应模板的变量个数保持一致，，若无模板参数，则设置为空
+            req.TemplateParamSet = [veri_code, self.veri_code_timeout]
+            # req.PhoneNumberSet = ["+8618285574257"]
+            req.PhoneNumberSet = [send_phone_number]
+
+            req.SessionContext = session_context
+
+            response: SendSmsResponse = client.SendSms(req)
+            response_status: SendStatus = response.SendStatusSet[0]
+            assert response_status.PhoneNumber == send_phone_number, "发送号码和响应号码不一致"
+            if response_status.Code != "Ok":
+                logging.error(f"发送短信失败: {response_status}")
+                return False
+            logging.debug(f"发送短信成功: {response_status}")
+            return True
+        except TencentCloudSDKException as err:
+            print(err)
+            logging.error(f"发送短信失败: {err}")
+            return False

jwf_common_python/translate/__init__.py

@@ -0,0 +1,152 @@
+import hashlib
+import os
+from abc import ABCMeta, abstractmethod
+from typing import Literal, TypedDict
+
+# 后端支持的语言代码
+# PREF 优化为下面这种形式
+LanguageCodeOfBackend = Literal["zh-CN", "zh-TW", "en", "fr", "ja", "de", "es", "ko"]
+
+#  ！！！单词没有词性，只有翻译才有词性！！！
+# 翻译词性对照表,用来将词性标准化统一后端实现。
+#  HACK 每个提供商都需要实现这个对照表
+pos_tag_table = {
+    "提供商-词性1": ("Adjectives", "形容词"),
+    "提供商-词性2": ("Adverbs", "副词"),
+    "提供商-词性3": ("Conjunctions", "连词"),
+    "提供商-词性4": ("Determiners", "限定词"),
+    "提供商-词性5": ("Verbs", "动词"),
+    "提供商-词性6": ("Nouns", "名词"),
+    "提供商-词性7": ("Prepositions", "介词"),
+    "提供商-词性8": ("Pronouns", "代词"),
+    "提供商-词性9": ("Verbs", "动词"),
+    "提供商-词性10": ("Other", "其他"),
+}
+
+
+class WordTranslation(TypedDict):
+    """单词翻译结果结构"""
+
+    # 2.1 标准化之后的翻译
+    normalized: str
+    # 用于常见显示的翻译
+    display: str
+
+    # 2.2 翻译词性,是 part-of-speech tag 的简写
+    # 参考词性对照表.
+    #  ("Adjectives", "形容词")
+    pos_tag: tuple[str, str]
+
+    # 2.3 翻译频率
+    # 数据格式: 10.3 保留一位小数, 页面显示为10.3%
+    frequency: float
+
+    # 2.4 翻译的例句
+    """
+    该单词的翻译例句的结构:
+    - 一个单词出现在多个例句中
+    - 一个例句有多个翻译
+    [
+        {
+        "sourcePrefix": "前半句",
+        "sourceTerm": "关键词",
+        "sourceSuffix": "后半句",
+        "target": [
+            {
+                "targetPrefix": "翻译前半句",
+                "targetTerm": "翻译关键词",
+                "targetSuffix": "翻译后半句",
+            }
+        ],
+    ]
+    """
+    examples: list[dict]
+
+    # 2.4 翻译前缀
+    # prefixWord: str
+
+
+class WordInfo(TypedDict):
+    """单词查询信息结构"""
+
+    # 1. 传入word标准化之后的形式, word表中唯一标识
+    normalized_word: str
+    # 1.1 用于常见显示的的word, 通常处理大小写
+    display_word: str
+    # word: str
+
+    # 2. word翻译
+    translations: list[WordTranslation]
+
+    # 3. 单词其他变体形式, 例如复数, 过去式等
+    shapes: list
+
+    # 4. word的发音数据
+    audio_data: bytes
+    # speak_url: dict
+
+    # 5. 扩展信息
+    extended_info: dict
+
+    # 单词音标
+    # phonetic: dict
+    # 单词语法信息
+    # grammar_info: list
+
+
+class SentenceInfo(TypedDict):
+    """句子查询信息结构"""
+
+    # 1. 原始sentence文本
+    sentence: str
+
+    # 2. 翻译之后的文本。部分翻译提供商提供多个翻译
+    translations: list[str]
+
+    # 3. 原始sentence的发音数据
+    audio_data: bytes
+    # source_speak_url: str
+    # translation_speak_url: str
+
+
+class Translate(metaclass=ABCMeta):
+    @abstractmethod
+    def __init__(self, **kwargs) -> None:
+        """\n
+        1. 初始化全局配置
+        2. 初始化相关翻译/合成对象
+        """
+        ...
+
+    @abstractmethod
+    def translate_word(self, word: str) -> dict:
+        """获取简单处理之后的原始word翻译响应"""
+        ...
+
+    @abstractmethod
+    def translate_sentence(self, sentence: str) -> dict:
+        """获取简单处理之后的原始sentence翻译响应"""
+        ...
+
+    @classmethod
+    def format_word_response(cls, response: dict) -> WordInfo:
+        """将原始word翻译响应格式化为标准化的word查询信息"""
+        ...
+
+    @classmethod
+    def format_sentence_response(cls, response: dict) -> SentenceInfo:
+        """将原始sentence翻译响应格式化为标准化的sentence查询信息"""
+        ...
+
+    # DEPRE 移除youdao翻译时，删除这个
+    @classmethod
+    def download_audio_file(cls, url: str) -> bytes:
+        ...
+
+    # DEPRECATED youdao翻译时，移动到后端实现
+    @classmethod
+    def generate_word_audio_file_name(self, word: str, audio_type: str) -> str:
+        file_id: str = hashlib.md5(word.encode("utf-8")).hexdigest()
+        file_name: str = f"word_youdao_{audio_type}_{file_id}.mp3"
+        absolute_audio_file_path = os.path.join("/opt/", file_name)
+        return absolute_audio_file_path