crawlo 1.2.7__py3-none-any.whl → 1.2.9__py3-none-any.whl

crawlo/tools/data_validator.py

@@ -1,181 +1,181 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 数据验证工具
-"""
-
-import re
-from typing import Any, Union, Dict, List
-from datetime import datetime
-from urllib.parse import urlparse
-
-
-class DataValidator:
-    """数据验证工具类"""
-
-    @staticmethod
-    def validate_email(email: str) -> bool:
-        """
-        验证邮箱地址格式
-        
-        Args:
-            email (str): 邮箱地址
-            
-        Returns:
-            bool: 验证结果
-        """
-        pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
-        return bool(re.match(pattern, email))
-
-    @staticmethod
-    def validate_phone(phone: str, country_code: str = "CN") -> bool:
-        """
-        验证电话号码格式
-        
-        Args:
-            phone (str): 电话号码
-            country_code (str): 国家代码，默认为"CN"
-            
-        Returns:
-            bool: 验证结果
-        """
-        if country_code == "CN":
-            # 中国手机号码格式
-            pattern = r'^1[3-9]\d{9}$'
-            return bool(re.match(pattern, phone))
-        else:
-            # 通用格式，只检查是否全为数字且长度在7-15之间
-            pattern = r'^\d{7,15}$'
-            return bool(re.match(pattern, phone))
-
-    @staticmethod
-    def validate_url(url: str) -> bool:
-        """
-        验证URL格式
-        
-        Args:
-            url (str): URL地址
-            
-        Returns:
-            bool: 验证结果
-        """
-        try:
-            result = urlparse(url)
-            return all([result.scheme, result.netloc])
-        except Exception:
-            return False
-
-    @staticmethod
-    def validate_chinese_id_card(id_card: str) -> bool:
-        """
-        验证中国身份证号码格式
-        
-        Args:
-            id_card (str): 身份证号码
-            
-        Returns:
-            bool: 验证结果
-        """
-        # 18位身份证号码格式
-        pattern = r'^[1-9]\d{5}(18|19|20)\d{2}((0[1-9])|(1[0-2]))(([0-2][1-9])|10|20|30|31)\d{3}[0-9Xx]$'
-        return bool(re.match(pattern, id_card))
-
-    @staticmethod
-    def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
-        """
-        验证日期格式
-        
-        Args:
-            date_str (str): 日期字符串
-            date_format (str): 日期格式，默认为"%Y-%m-%d"
-            
-        Returns:
-            bool: 验证结果
-        """
-        try:
-            datetime.strptime(date_str, date_format)
-            return True
-        except ValueError:
-            return False
-
-    @staticmethod
-    def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
-                             max_val: Union[int, float]) -> bool:
-        """
-        验证数值是否在指定范围内
-        
-        Args:
-            value (Union[int, float]): 要验证的数值
-            min_val (Union[int, float]): 最小值
-            max_val (Union[int, float]): 最大值
-            
-        Returns:
-            bool: 验证结果
-        """
-        return min_val <= value <= max_val
-
-    @staticmethod
-    def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
-        """
-        检查数据完整性，确保关键字段不为空
-        
-        Args:
-            data (Dict[str, Any]): 要检查的数据
-            required_fields (List[str]): 必需字段列表
-            
-        Returns:
-            Dict[str, Any]: 检查结果，包含缺失字段和空值字段
-        """
-        missing_fields = []
-        empty_fields = []
-        
-        for field in required_fields:
-            if field not in data:
-                missing_fields.append(field)
-            elif data[field] is None or data[field] == "":
-                empty_fields.append(field)
-                
-        return {
-            "is_valid": len(missing_fields) == 0 and len(empty_fields) == 0,
-            "missing_fields": missing_fields,
-            "empty_fields": empty_fields
-        }
-
-
-# 便捷函数
-def validate_email(email: str) -> bool:
-    """验证邮箱地址格式"""
-    return DataValidator.validate_email(email)
-
-
-def validate_phone(phone: str, country_code: str = "CN") -> bool:
-    """验证电话号码格式"""
-    return DataValidator.validate_phone(phone, country_code)
-
-
-def validate_url(url: str) -> bool:
-    """验证URL格式"""
-    return DataValidator.validate_url(url)
-
-
-def validate_chinese_id_card(id_card: str) -> bool:
-    """验证中国身份证号码格式"""
-    return DataValidator.validate_chinese_id_card(id_card)
-
-
-def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
-    """验证日期格式"""
-    return DataValidator.validate_date(date_str, date_format)
-
-
-def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
-                         max_val: Union[int, float]) -> bool:
-    """验证数值是否在指定范围内"""
-    return DataValidator.validate_number_range(value, min_val, max_val)
-
-
-def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
-    """检查数据完整性"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-10 22:00
+# @Author  : crawl-coder
+# @Desc    : 数据验证工具
+"""
+
+import re
+from typing import Any, Union, Dict, List
+from datetime import datetime
+from urllib.parse import urlparse
+
+
+class DataValidator:
+    """数据验证工具类"""
+
+    @staticmethod
+    def validate_email(email: str) -> bool:
+        """
+        验证邮箱地址格式
+        
+        Args:
+            email (str): 邮箱地址
+            
+        Returns:
+            bool: 验证结果
+        """
+        pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+        return bool(re.match(pattern, email))
+
+    @staticmethod
+    def validate_phone(phone: str, country_code: str = "CN") -> bool:
+        """
+        验证电话号码格式
+        
+        Args:
+            phone (str): 电话号码
+            country_code (str): 国家代码，默认为"CN"
+            
+        Returns:
+            bool: 验证结果
+        """
+        if country_code == "CN":
+            # 中国手机号码格式
+            pattern = r'^1[3-9]\d{9}$'
+            return bool(re.match(pattern, phone))
+        else:
+            # 通用格式，只检查是否全为数字且长度在7-15之间
+            pattern = r'^\d{7,15}$'
+            return bool(re.match(pattern, phone))
+
+    @staticmethod
+    def validate_url(url: str) -> bool:
+        """
+        验证URL格式
+        
+        Args:
+            url (str): URL地址
+            
+        Returns:
+            bool: 验证结果
+        """
+        try:
+            result = urlparse(url)
+            return all([result.scheme, result.netloc])
+        except Exception:
+            return False
+
+    @staticmethod
+    def validate_chinese_id_card(id_card: str) -> bool:
+        """
+        验证中国身份证号码格式
+        
+        Args:
+            id_card (str): 身份证号码
+            
+        Returns:
+            bool: 验证结果
+        """
+        # 18位身份证号码格式
+        pattern = r'^[1-9]\d{5}(18|19|20)\d{2}((0[1-9])|(1[0-2]))(([0-2][1-9])|10|20|30|31)\d{3}[0-9Xx]$'
+        return bool(re.match(pattern, id_card))
+
+    @staticmethod
+    def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
+        """
+        验证日期格式
+        
+        Args:
+            date_str (str): 日期字符串
+            date_format (str): 日期格式，默认为"%Y-%m-%d"
+            
+        Returns:
+            bool: 验证结果
+        """
+        try:
+            datetime.strptime(date_str, date_format)
+            return True
+        except ValueError:
+            return False
+
+    @staticmethod
+    def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
+                             max_val: Union[int, float]) -> bool:
+        """
+        验证数值是否在指定范围内
+        
+        Args:
+            value (Union[int, float]): 要验证的数值
+            min_val (Union[int, float]): 最小值
+            max_val (Union[int, float]): 最大值
+            
+        Returns:
+            bool: 验证结果
+        """
+        return min_val <= value <= max_val
+
+    @staticmethod
+    def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
+        """
+        检查数据完整性，确保关键字段不为空
+        
+        Args:
+            data (Dict[str, Any]): 要检查的数据
+            required_fields (List[str]): 必需字段列表
+            
+        Returns:
+            Dict[str, Any]: 检查结果，包含缺失字段和空值字段
+        """
+        missing_fields = []
+        empty_fields = []
+        
+        for field in required_fields:
+            if field not in data:
+                missing_fields.append(field)
+            elif data[field] is None or data[field] == "":
+                empty_fields.append(field)
+                
+        return {
+            "is_valid": len(missing_fields) == 0 and len(empty_fields) == 0,
+            "missing_fields": missing_fields,
+            "empty_fields": empty_fields
+        }
+
+
+# 便捷函数
+def validate_email(email: str) -> bool:
+    """验证邮箱地址格式"""
+    return DataValidator.validate_email(email)
+
+
+def validate_phone(phone: str, country_code: str = "CN") -> bool:
+    """验证电话号码格式"""
+    return DataValidator.validate_phone(phone, country_code)
+
+
+def validate_url(url: str) -> bool:
+    """验证URL格式"""
+    return DataValidator.validate_url(url)
+
+
+def validate_chinese_id_card(id_card: str) -> bool:
+    """验证中国身份证号码格式"""
+    return DataValidator.validate_chinese_id_card(id_card)
+
+
+def validate_date(date_str: str, date_format: str = "%Y-%m-%d") -> bool:
+    """验证日期格式"""
+    return DataValidator.validate_date(date_str, date_format)
+
+
+def validate_number_range(value: Union[int, float], min_val: Union[int, float], 
+                         max_val: Union[int, float]) -> bool:
+    """验证数值是否在指定范围内"""
+    return DataValidator.validate_number_range(value, min_val, max_val)
+
+
+def check_data_integrity(data: Dict[str, Any], required_fields: List[str]) -> Dict[str, Any]:
+    """检查数据完整性"""
     return DataValidator.check_data_integrity(data, required_fields)

crawlo/tools/date_tools.py

@@ -1,36 +1,290 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-10 22:00
-# @Author  : crawl-coder
-# @Desc    : 日期工具封装
-"""
-
-# 从 utils 模块导入日期工具
-from ..utils.date_tools import (
-    TimeUtils,
-    parse_time,
-    format_time,
-    time_diff,
-    to_timestamp,
-    to_datetime,
-    now,
-    to_timezone,
-    to_utc,
-    to_local,
-    from_timestamp_with_tz
-)
-
-__all__ = [
-    "TimeUtils",
-    "parse_time",
-    "format_time",
-    "time_diff",
-    "to_timestamp",
-    "to_datetime",
-    "now",
-    "to_timezone",
-    "to_utc",
-    "to_local",
-    "from_timestamp_with_tz"
-]
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-05-17 10:20
+# @Author  : crawl-coder
+# @Desc    : 智能时间工具库（专为爬虫场景设计）
+"""
+import dateparser
+from typing import Optional, Union, Literal
+from datetime import datetime, timedelta, timezone
+from dateutil.relativedelta import relativedelta
+import pytz
+from pytz import timezone as pytz_timezone
+
+# 支持的单位类型
+TimeUnit = Literal["seconds", "minutes", "hours", "days"]
+# 时间输入类型
+TimeType = Union[str, datetime]
+# 时区类型
+TimezoneType = Union[str, timezone, pytz_timezone]
+
+# 常见时间格式列表（作为 dateparser 的后备方案）
+COMMON_FORMATS = [
+    "%Y-%m-%d %H:%M:%S",
+    "%Y/%m/%d %H:%M:%S",
+    "%d-%m-%Y %H:%M:%S",
+    "%d/%m/%Y %H:%M:%S",
+    "%Y-%m-%d",
+    "%Y/%m/%d",
+    "%d-%m-%Y",
+    "%d/%m/%Y",
+    "%b %d, %Y",
+    "%B %d, %Y",
+    "%Y年%m月%d日",
+    "%Y年%m月%d日 %H时%M分%S秒",
+    "%a %b %d %H:%M:%S %Y",
+    "%a, %d %b %Y %H:%M:%S",
+    "%Y-%m-%dT%H:%M:%S.%f",
+    "%Y-%m-%dT%H:%M:%S",
+]
+
+
+class TimeUtils:
+    """
+    时间处理工具类，提供日期解析、格式化、计算等一站式服务。
+    特别适用于爬虫中处理多语言、多格式、相对时间的场景。
+    """
+
+    @staticmethod
+    def _try_strptime(time_str: str) -> Optional[datetime]:
+        """尝试使用预定义格式解析，作为 dateparser 的后备"""
+        for fmt in COMMON_FORMATS:
+            try:
+                return datetime.strptime(time_str.strip(), fmt)
+            except ValueError:
+                continue
+        return None
+
+    @classmethod
+    def parse(cls, time_input: TimeType, *, default: Optional[datetime] = None) -> Optional[datetime]:
+        """
+        智能解析时间输入（字符串或 datetime）。
+
+        :param time_input: 时间字符串（支持各种语言、格式、相对时间）或 datetime 对象
+        :param default: 解析失败时返回的默认值
+        :return: 解析成功返回 datetime，失败返回 default
+        """
+        if isinstance(time_input, datetime):
+            return time_input
+
+        if not isinstance(time_input, str) or not time_input.strip():
+            return default
+
+        # 1. 优先使用 dateparser（支持多语言和相对时间）
+        try:
+            parsed = dateparser.parse(time_input.strip())
+            if parsed:
+                return parsed
+        except Exception:
+            pass  # 忽略异常，尝试后备方案
+
+        # 2. 尝试使用常见格式解析
+        try:
+            parsed = cls._try_strptime(time_input)
+            if parsed:
+                return parsed
+        except Exception:
+            pass
+
+        return default
+
+    @classmethod
+    def format(cls, dt: TimeType, fmt: str = "%Y-%m-%d %H:%M:%S") -> Optional[str]:
+        """
+        格式化时间。
+
+        :param dt: datetime 对象或可解析的字符串
+        :param fmt: 输出格式
+        :return: 格式化后的字符串，失败返回 None
+        """
+        if isinstance(dt, str):
+            dt = cls.parse(dt)
+            if dt is None:
+                return None
+        try:
+            return dt.strftime(fmt)
+        except Exception:
+            return None
+
+    @classmethod
+    def to_timestamp(cls, dt: TimeType) -> Optional[float]:
+        """转换为时间戳（秒级）"""
+        if isinstance(dt, str):
+            dt = cls.parse(dt)
+            if dt is None:
+                return None
+        try:
+            return dt.timestamp()
+        except Exception:
+            return None
+
+    @classmethod
+    def from_timestamp(cls, ts: float) -> Optional[datetime]:
+        """从时间戳创建 datetime"""
+        try:
+            return datetime.fromtimestamp(ts)
+        except Exception:
+            return None
+
+    @classmethod
+    def diff(cls, start: TimeType, end: TimeType, unit: TimeUnit = "seconds") -> Optional[int]:
+        """
+        计算两个时间的差值（自动解析字符串）。
+
+        :param start: 起始时间
+        :param end: 结束时间
+        :param unit: 单位 ('seconds', 'minutes', 'hours', 'days')
+        :return: 差值（绝对值），失败返回 None
+        """
+        start_dt = cls.parse(start)
+        end_dt = cls.parse(end)
+        if not start_dt or not end_dt:
+            return None
+
+        delta = abs((end_dt - start_dt).total_seconds())
+
+        unit_map = {
+            "seconds": 1,
+            "minutes": 60,
+            "hours": 3600,
+            "days": 86400,
+        }
+        return int(delta // unit_map.get(unit, 1))
+
+    @classmethod
+    def add_days(cls, dt: TimeType, days: int) -> Optional[datetime]:
+        """日期加减（天）"""
+        dt = cls.parse(dt)
+        if dt is None:
+            return None
+        return dt + timedelta(days=days)
+
+    @classmethod
+    def add_months(cls, dt: TimeType, months: int) -> Optional[datetime]:
+        """日期加减（月）"""
+        dt = cls.parse(dt)
+        if dt is None:
+            return None
+        return dt + relativedelta(months=months)
+
+    @classmethod
+    def days_between(cls, dt1: TimeType, dt2: TimeType) -> Optional[int]:
+        """计算两个日期之间的天数差"""
+        return cls.diff(dt1, dt2, "days")
+
+    @classmethod
+    def is_leap_year(cls, year: int) -> bool:
+        """判断是否是闰年"""
+        return (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)
+
+    @classmethod
+    def now(cls, fmt: Optional[str] = None) -> Union[datetime, str]:
+        """
+        获取当前时间。
+
+        :param fmt: 如果提供，则返回格式化字符串；否则返回 datetime 对象。
+        :return: datetime 或 str
+        """
+        dt = datetime.now()
+        if fmt is not None:
+            return dt.strftime(fmt)
+        return dt
+
+    @classmethod
+    def iso_format(cls, dt: TimeType) -> Optional[str]:
+        """返回 ISO 8601 格式字符串"""
+        dt = cls.parse(dt)
+        if dt is None:
+            return None
+        return dt.isoformat()
+
+    @classmethod
+    def to_timezone(cls, dt: TimeType, tz: TimezoneType) -> Optional[datetime]:
+        """将时间转换为指定时区"""
+        dt = cls.parse(dt)
+        if dt is None:
+            return None
+        
+        try:
+            if isinstance(tz, str):
+                tz = pytz_timezone(tz)
+            return dt.astimezone(tz)
+        except Exception:
+            return None
+
+    @classmethod
+    def to_utc(cls, dt: TimeType) -> Optional[datetime]:
+        """将时间转换为 UTC 时区"""
+        return cls.to_timezone(dt, pytz.UTC)
+
+    @classmethod
+    def to_local(cls, dt: TimeType) -> Optional[datetime]:
+        """将时间转换为本地时区"""
+        return cls.to_timezone(dt, pytz.timezone("Asia/Shanghai"))
+
+    @classmethod
+    def from_timestamp_with_tz(cls, ts: float, tz: TimezoneType = None) -> Optional[datetime]:
+        """从时间戳创建 datetime，并可选择指定时区"""
+        try:
+            dt = datetime.fromtimestamp(ts)
+            if tz:
+                if isinstance(tz, str):
+                    tz = pytz_timezone(tz)
+                dt = dt.replace(tzinfo=tz)
+            return dt
+        except Exception:
+            return None
+
+
+# =======================对外接口=======================
+
+def parse_time(time_input: TimeType, default: Optional[datetime] = None) -> Optional[datetime]:
+    """解析时间字符串或对象"""
+    return TimeUtils.parse(time_input, default=default)
+
+
+def format_time(dt: TimeType, fmt: str = "%Y-%m-%d %H:%M:%S") -> Optional[str]:
+    """格式化时间"""
+    return TimeUtils.format(dt, fmt)
+
+
+def time_diff(start: TimeType, end: TimeType, unit: TimeUnit = "seconds") -> Optional[int]:
+    """计算时间差"""
+    return TimeUtils.diff(start, end, unit)
+
+
+def to_timestamp(dt: TimeType) -> Optional[float]:
+    """转时间戳"""
+    return TimeUtils.to_timestamp(dt)
+
+
+def to_datetime(ts: float) -> Optional[datetime]:
+    """从时间戳转 datetime"""
+    return TimeUtils.from_timestamp(ts)
+
+
+def now(fmt: Optional[str] = None) -> Union[datetime, str]:
+    """获取当前时间"""
+    return TimeUtils.now(fmt)
+
+def to_timezone(dt: TimeType, tz: TimezoneType) -> Optional[datetime]:
+    """将时间转换为指定时区"""
+    return TimeUtils.to_timezone(dt, tz)
+
+def to_utc(dt: TimeType) -> Optional[datetime]:
+    """将时间转换为 UTC 时区"""
+    return TimeUtils.to_utc(dt)
+
+def to_local(dt: TimeType) -> Optional[datetime]:
+    """将时间转换为本地时区"""
+    return TimeUtils.to_local(dt)
+
+def from_timestamp_with_tz(ts: float, tz: TimezoneType = None) -> Optional[datetime]:
+    """从时间戳创建 datetime，并可选择指定时区"""
+    return TimeUtils.from_timestamp_with_tz(ts, tz)
+
+
+if __name__ == '__main__':
+    get_current_time = now(fmt="%Y-%m-%d %H:%M:%S")
+    print(get_current_time)