sd-spider-utils 1.0.0__tar.gz

sd_spider_utils-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 星梦
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sd_spider_utils-1.0.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: sd_spider_utils
+Version: 1.0.0
+Summary: 爬虫工具包，写爬虫更快！！！
+Author-email: 星梦 <cpython666@gmail.com>
+License: MIT
+Project-URL: Homepage, https://space.bilibili.com/1909782963
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Natural Language :: Chinese (Simplified)
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.5
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+Dynamic: license-file

sd_spider_utils-1.0.0/pyproject.toml

@@ -0,0 +1,29 @@
+[project]
+name = "sd_spider_utils"
+version = "1.0.0"
+description = "爬虫工具包，写爬虫更快！！！"
+readme = "README.rst"
+license = {text = "MIT"}
+authors = [
+    { name = "星梦", email = "cpython666@gmail.com" },
+]
+urls = { Homepage = "https://space.bilibili.com/1909782963" }
+requires-python = ">=3.5"
+dependencies = ["requests", "beautifulsoup4"]
+
+classifiers = [
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Natural Language :: Chinese (Simplified)",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Topic :: Software Development :: Libraries",
+]
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"

sd_spider_utils-1.0.0/sd_spider_utils/common_spider.py

@@ -0,0 +1,4 @@
+import scrapy
+
+# TODO 通用爬虫
+print(scrapy.Request)

sd_spider_utils-1.0.0/sd_spider_utils/common_utils.py

@@ -0,0 +1,22 @@
+def strtobool(value):
+    """
+    常见字符串转布尔值
+    """
+    _MAP = {
+        "y": True,
+        "yes": True,
+        "t": True,
+        "true": True,
+        "on": True,
+        "1": True,
+        "n": False,
+        "no": False,
+        "f": False,
+        "false": False,
+        "off": False,
+        "0": False,
+    }
+    try:
+        return _MAP[str(value).lower()]
+    except KeyError:
+        raise ValueError('"{}" is not a valid bool value'.format(value))

sd_spider_utils-1.0.0/sd_spider_utils/data_utils.py

@@ -0,0 +1,30 @@
+def json2excel(filepath):
+    """
+    input: filepath: string
+    [
+    {"name":"小明","age",18},
+    ...
+    ]
+    """
+    import os
+    import json
+    import pandas as pd
+
+    # 获取输入文件所在的文件夹路径
+    folder_path = os.path.dirname(filepath)
+    # 获取输入文件的文件名（不带扩展名）
+    file_name = os.path.splitext(os.path.basename(filepath))[0]
+    # 构建输出的 Excel 文件路径
+    output_filepath = os.path.join(folder_path, f"{file_name}.xlsx")
+    # 读取 JSON 文件
+    with open(filepath, "r", encoding="utf-8") as file:
+        obj_list = json.load(file)
+    # 将数据转换为 DataFrame
+    df = pd.DataFrame(obj_list)
+    # 将 DataFrame 保存为 Excel 文件
+    df.to_excel(output_filepath, index=False)
+    print(f"Excel 文件已保存：{output_filepath}")
+
+
+if __name__ == "__main__":
+    json2excel(r"demo.json")

sd_spider_utils-1.0.0/sd_spider_utils/datetime_utils.py

@@ -0,0 +1,72 @@
+from typing import List
+from datetime import datetime
+import re
+
+
+def clean_text(text: str) -> str:
+    # 替换非断空白符为普通空格
+    text = text.replace("\xa0", " ")
+    # 移除字符串两端的空格
+    text = text.strip()
+    # 替换多个空格为单个空格
+    text = " ".join(text.split())
+    # 移除多余的标点符号，例如连续的逗号或逗号后面紧跟空格
+    text = text.replace(" ,", ",").replace(", ,", ",")
+    return text
+
+
+def contains_chinese(text: str) -> bool:
+    """
+    使用正则表达式检查是否包含汉字
+    """
+    return bool(re.search(r"[\u4e00-\u9fa5]", text))
+
+
+def contains_date(text: str) -> bool:
+    """
+    使用正则表达式检查是否包含类似 '2022年03月30日' 的日期
+    """
+    return bool(re.search(r"\d{4}[-/年]\d{1,2}[-/月]\d{1,2}日?", text))
+
+
+def extract_dates(text: str) -> List[datetime]:
+    """
+    提取文本中所有的日期并返回 datetime 对象列表
+    支持格式如: 2022年03月30日, 2022-03-30, 2022/03/30
+    """
+    # 正则表达式匹配日期
+    pattern: str = r"\d{4}[-/年]\d{1,2}[-/月]\d{1,2}日?"
+    matches = re.findall(pattern, text)
+
+    dates: List[datetime] = []
+    for match in matches:
+        # 替换中文字符为标准分隔符
+        normalized = match.replace("年", "-").replace("月", "-").replace("日", "")
+        try:
+            # 将字符串转换为 datetime 对象
+            date_obj = datetime.strptime(normalized, "%Y-%m-%d")
+            dates.append(date_obj)
+        except ValueError:
+            # 如果转换失败，跳过这个匹配
+            continue
+
+    return dates
+
+
+if __name__ == "__main__":
+    # 测试日期提取
+    test_text: str = "今天是2022年03月30日，昨天是2022-03-29，明天是2022/03/31。"
+    dates: List[datetime] = extract_dates(test_text)
+    for date in dates:
+        print(f"Found date: {date.strftime('%Y-%m-%d')}")  # 格式化输出日期
+
+    # 原有的测试代码
+    text1: str = "This is a te{||||  nmakldnsjdmksxm  15651654 st.把那家伙半小时·"
+    text2: str = "这是一个测试。"
+    print(contains_chinese(text1))  # False
+    print(contains_chinese(text2))  # True
+
+    text3: str = "今天是2022年03月30日，天气晴。"
+    text4: str = "这是一个没有日期的文本。"
+    print(contains_date(text3))  # True
+    print(contains_date(text4))  # False

sd_spider_utils-1.0.0/sd_spider_utils/parse_utils.py

@@ -0,0 +1,36 @@
+from typing import List
+
+
+def get_text_bs4(html: str, remove_blank_lines: bool = False) -> str:
+    from bs4 import BeautifulSoup
+
+    soup = BeautifulSoup(html, "html.parser")
+    if remove_blank_lines:
+        return "\n".join(
+            line.strip() for line in soup.text.splitlines() if line.strip()
+        )
+    else:
+        return soup.text.strip()
+
+
+def get_text_xpath(text: str) -> List[str]:
+    from lxml import etree
+
+    tree = etree.fromstring(text)
+    return tree.xpath("//text()")
+
+
+def get_text_scrapy(text: str) -> str:
+    from scrapy import Selector
+
+    response = Selector(text=text)
+    return "".join(response.xpath("//text()").getall())
+
+
+if __name__ == "__main__":
+    text: str = (
+        "<div><p>这是段落1</p><p>这是段落2<em>2nsjkxabs</em><p>这是段落3</p></p></div>"
+    )
+    print(get_text_scrapy(text))
+    print(get_text_bs4(text))
+    # ['这是段落1', '这是段落2', '2nsjkxabs', '这是段落3']

sd_spider_utils-1.0.0/sd_spider_utils/spider_demos.py

@@ -0,0 +1,12 @@
+def xpath_demo():
+    print(
+        """
+    类名为 cpython666 的节点 response.xpath("//*[@class='cpython666']")
+    类名以 cpython666 开头的节点 response.xpath("//*[starts-with(@class, 'cpython666')]")
+    文本包含 cpython666 的h2标签的后续兄弟p标签 response.xpath("//h2[contains(string(.), 'cpython666')]/following-sibling::p")
+    """
+    )
+
+
+if __name__ == "__main__":
+    xpath_demo()

sd_spider_utils-1.0.0/sd_spider_utils/text_utils.py

@@ -0,0 +1,96 @@
+import re
+
+
+def normal_text(text):
+    """
+    normalize_unicode_text
+    文本标准化：转换为标准形式
+    """
+    import unicodedata
+
+    normalized_text = unicodedata.normalize("NFKC", text)
+    return normalized_text
+
+
+def clean_text(text: str):
+    # 替换非断空白符为普通空格
+    text = text.replace("\xa0", " ")
+    # 移除字符串两端的空格
+    text = text.strip()
+    # 替换多个空格为单个空格
+    text = " ".join(text.split())
+    # 移除多余的标点符号，例如连续的逗号或逗号后面紧跟空格
+    text = text.replace(" ,", ",").replace(", ,", ",")
+    return text
+
+
+def remove_extra_spaces(text: str) -> str:
+    """
+    移除字符串中的多个空格为单个空格。
+    :param text: 输入字符串
+    :return: 处理后的字符串
+    # 示例
+    text = "This   is   a   text     with  multiple     spaces."
+    cleaned_text = remove_extra_spaces(text)
+    print(cleaned_text)  # 输出: "This is a text with multiple spaces."
+    """
+    return " ".join(text.split())
+
+
+def remove_extra_blank_spaces(text: str) -> str:
+    """
+    移除字符串中的多余空白字符（空格、制表符、换行符等），
+    并将其替换为单个空格。
+
+    :param text: 输入字符串
+    :return: 处理后的字符串
+
+    # 示例
+    text = "This   is   a   text     with  multiple     spaces."
+    cleaned_text = remove_extra_spaces(text)
+    print(cleaned_text)  # 输出: "This is a text with multiple spaces."
+    """
+    import re
+
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def contains_chinese(text: str) -> bool:
+    """
+    使用正则表达式检查是否包含汉字
+    """
+    import re
+
+    return bool(re.search(r"[\u4e00-\u9fa5]", text))
+
+
+def contains_date(text: str) -> bool:
+    """
+    使用正则表达式检查是否包含类似 '2022年03月30日' 的日期
+    """
+    import re
+
+    return bool(re.search(r"\d{4}[-/年]\d{1,2}[-/月]\d{1,2}日?", text))
+
+
+if __name__ == "__main__":
+    text = "This   is   a   text     with  multiple     spaces."
+    # 使用正则表达式替换一个或多个空白字符（包括空格、制表符、换行符等）为一个空格
+    cleaned_text = re.sub(r"\s+", " ", text).strip()
+
+    print(cleaned_text)
+
+    text1 = "This is a te{||||  nmakldnsjdmksxm  15651654 st.把那家伙半小时·"
+    text2 = "这是一个测试。"
+
+    print(contains_chinese(text1))  # False
+    print(contains_chinese(text2))  # True
+
+    # 测试
+    text1 = "今天是2022年03月30日，天气晴。"
+    text2 = "这是一个没有日期的文本。"
+
+    print(contains_date(text1))  # True
+    print(contains_date(text2))  # False
+    text = "Ｃａｆé['S.\u2009M. Koksbang\xa0', 'S.\u2009M. Koksbang']"  # 包含全角字符和组合字符
+    normalize_unicode_text(text)

sd_spider_utils-1.0.0/sd_spider_utils.egg-info/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: sd_spider_utils
+Version: 1.0.0
+Summary: 爬虫工具包，写爬虫更快！！！
+Author-email: 星梦 <cpython666@gmail.com>
+License: MIT
+Project-URL: Homepage, https://space.bilibili.com/1909782963
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Natural Language :: Chinese (Simplified)
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.5
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: beautifulsoup4
+Dynamic: license-file

sd_spider_utils-1.0.0/sd_spider_utils.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+LICENSE
+README.rst
+pyproject.toml
+sd_spider_utils/__init__.py
+sd_spider_utils/common_spider.py
+sd_spider_utils/common_utils.py
+sd_spider_utils/data_utils.py
+sd_spider_utils/datetime_utils.py
+sd_spider_utils/parse_utils.py
+sd_spider_utils/spider_demos.py
+sd_spider_utils/text_utils.py
+sd_spider_utils/url_utils.py
+sd_spider_utils.egg-info/PKG-INFO
+sd_spider_utils.egg-info/SOURCES.txt
+sd_spider_utils.egg-info/dependency_links.txt
+sd_spider_utils.egg-info/requires.txt
+sd_spider_utils.egg-info/top_level.txt

sd_spider_utils-1.0.0/sd_spider_utils.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

sd_spider_utils-1.0.0/sd_spider_utils.egg-info/requires.txt

@@ -0,0 +1,2 @@
+requests
+beautifulsoup4

sd_spider_utils-1.0.0/sd_spider_utils.egg-info/top_level.txt

@@ -0,0 +1 @@
+sd_spider_utils

sd_spider_utils-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+