xtn-tools-pro 1.0.0.0.0__py3-none-any.whl

xtnkk_tools_pro/db/MongoDB.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    MongoDBPro
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/17    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+from xtn_tools_pro.tools_time import *
+from urllib import parse
+from pymongo import MongoClient as _MongoClient
+from pymongo.database import Database as _Database
+from typing import List, Dict, Optional
+from pymongo.collection import Collection as _Collection
+from pymongo.errors import DuplicateKeyError, BulkWriteError
+
+
+class MongoDBPro:
+    def __init__(self, ip=None, port=None, db=None, user_name=None, user_pass=None, url=None, **kwargs):
+        if url:
+            self.client = _MongoClient(url, **kwargs)
+        else:
+            self.client = _MongoClient(host=ip,
+                                       port=port,
+                                       username=user_name,
+                                       password=user_pass,
+                                       authSource=db)
+
+        self.db = self.get_database(db)
+
+    @classmethod
+    def from_url(cls, url, **kwargs):
+        url_parsed = parse.urlparse(url)
+        # 获取 URL的协议
+        db_type = url_parsed.scheme.strip()
+        if db_type != "mongodb":
+            raise Exception(
+                "url error, expect mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]], but get {}".format(
+                    url
+                ))
+        return cls(url=url, **kwargs)
+
+    def get_database(self, database, **kwargs) -> _Database:
+        """
+            根据db名获取数据库对象
+        """
+        return self.client.get_database(database, **kwargs)
+
+    def get_collection(self, coll_name, **kwargs) -> _Collection:
+        """
+            根据集合名获取集合对象
+        """
+        return self.db.get_collection(coll_name, **kwargs)
+
+    def run_command(self, command: Dict):
+        """
+            参考文档 https://www.geek-book.com/src/docs/mongodb/mongodb/docs.mongodb.com/manual/reference/command/index.html
+        """
+        return self.db.command(command)
+
+    def find(self, coll_name: str, condition: Optional[Dict] = None,
+             limit: int = 0, **kwargs) -> List[Dict]:
+        """
+            find
+            coll_name:集合名称
+            condition:查询条件 例如：{"name": "John"}、{"_id": "xxxxx"}
+        """
+        condition = {} if condition is None else condition
+        command = {"find": coll_name, "filter": condition, "limit": limit}
+        command.update(kwargs)
+        result = self.run_command(command)
+        cursor = result["cursor"]
+        cursor_id = cursor["id"]
+        while True:
+            for document in cursor.get("nextBatch", cursor.get("firstBatch", [])):
+                # 处理数据
+                yield document
+            if cursor_id == 0:
+                # 游标已经完全遍历，没有剩余的结果可供获取
+                # 游标的生命周期已经结束，例如在查询会话结束后。
+                # 游标被显式地关闭，例如使用 db.killCursor() 命令关闭游标。
+                break
+            result = self.run_command(
+                {
+                    "getMore": cursor_id,  # 类似于mongo命令行中的it命令，通过索引id用于获取下一批结果
+                    "collection": coll_name,
+                    "batchSize": kwargs.get("batchSize", 100),
+                }
+            )
+            # 覆盖原来的参数
+            cursor = result["cursor"]
+            cursor_id = cursor["id"]
+            # print("下一批获取")
+
+    def add_data_one(self, coll_name: str, data: Dict, insert_ignore=False,
+                     is_add_create_time=False,
+                     is_add_create_time_field_name="create_dt"):
+        """
+            添加单条数据
+            coll_name: 集合名
+            data: 单条数据
+            insert_ignore: 索引冲突是否忽略 默认False
+            is_add_create_time: 是否在数据中添加一个创建数据10时间戳字段 默认False不创建
+            is_add_create_time_field_name: 自定义创建数据时间戳字段名：默认：create_dt
+        Returns: 插入成功的行数
+        """
+        if is_add_create_time:
+            data[is_add_create_time_field_name] = get_time_now_timestamp(is_time_10=True)
+        collection = self.get_collection(coll_name)
+        try:
+            collection.insert_one(data)
+        except DuplicateKeyError as e:
+            if not insert_ignore:
+                raise e
+            return 0
+        return 1
+
+    def find_id_is_exist(self, coll_name, _id):
+        """
+            根据id查询id是否存在
+        :param _id:id
+        :return: 存在返回True 否则False
+        """
+        condition = {"_id": _id}
+        status = list(self.find(coll_name, condition))
+        if status:
+            return True
+        return False
+
+
+if __name__ == '__main__':
+    pass
+    # mongo_db = MongoDBPro("127.0.0.1", 27017, "spider_pro")
+    # # mongo_db.add_data_one("test", {"_id": "1", "data": "aaa"})
+    # print(mongo_db.find_id_is_exist("test", "1"))
+    # print(mongo_db.find_id_is_exist("test", "11"))

xtnkk_tools_pro/db/RedisDB.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    程序说明xxxxxxxxxxxxxxxxxxx
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/18    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import redis, time
+
+
+class RedisDBPro:
+    def __init__(self, ip=None, port=None, db=None,
+                 user_pass=None, url=None, decode_responses=True, max_connections=1000, **kwargs):
+
+        self._ip = ip
+        self._port = port
+        self._db = db
+        self._user_pass = user_pass
+        self._url = url
+        self._decode_responses = decode_responses  # 自动解码返回的字节串
+        self._max_connections = max_connections  # 同一个redis对象使用的并发数（连接池的最大连接数），超过这个数量会抛出redis.ConnectionError
+        self._kwargs = kwargs
+        # 连接
+        self.__redis = None
+        self.get_connect()
+
+    @classmethod
+    def from_url(cls, url):
+        """
+
+        Args:
+            url: redis://[[username]:[password]]@[host]:[port]/[db]
+
+        Returns:
+
+        """
+        return cls(url=url)
+
+    @property
+    def _redis(self):
+        try:
+            if not self.__redis.ping():
+                raise ConnectionError("unable to connect to redis")
+        except:
+            self._reconnect()
+
+        return self.__redis
+
+    @_redis.setter
+    def _redis(self, val):
+        self.__redis = val
+
+    def _reconnect(self):
+        # 检测连接状态, 当数据库重启或设置 timeout 导致断开连接时自动重连
+        retry_count = 0
+        while True:
+            try:
+                retry_count += 1
+                print("redis 连接断开, 重新连接 {retry_count}".format(retry_count=retry_count))
+                if self.get_connect():
+                    print("redis 连接成功")
+                    return True
+            except (ConnectionError, TimeoutError) as e:
+                print("连接失败 e: {e}".format(e=e))
+            time.sleep(2)
+
+    def get_connect(self):
+        # 数据库连接
+        try:
+            if not self._url:
+                self._redis = redis.StrictRedis(
+                    host=self._ip,
+                    port=self._port,
+                    db=self._db,
+                    password=self._user_pass,
+                    decode_responses=self._decode_responses,
+                    max_connections=self._max_connections,
+                    **self._kwargs,
+                )
+            else:
+                # url连接
+                self._redis = redis.StrictRedis.from_url(
+                    self._url, decode_responses=self._decode_responses, **self._kwargs
+                )
+
+        except Exception as e:
+            raise e
+
+        return self.__redis.ping()
+
+    def sadd(self, table, values):
+        """
+        使用无序set集合存储数据， 去重
+        table:集合名
+        values: 值； 支持list 或 单个值
+        ---------
+        @result: 若库中存在 返回0，否则入库，返回1。 批量添加返回None
+        """
+        if isinstance(values, list):
+            pipe = self._redis.pipeline()  # 创建一个pipeline对象，可以添加多个操作，最后execute提交
+            pipe.multi()
+            for value in values:
+                pipe.sadd(table, value)
+            result = pipe.execute()
+            # print(result) # [1, 1, 0, 0, 0, 0, 0]
+            return result.count(1)
+        else:
+            return self._redis.sadd(table, values)
+
+
+if __name__ == '__main__':
+    pass
+    # r = RedisDBPro(ip="127.0.0.1", port=6379, db=0, user_pass="xtn-kk")
+    # status = r.sadd("test_redis_pro", [1, 2, 3, 4, 5, "6", "7"])
+    # print(status)

xtnkk_tools_pro/db/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    程序说明xxxxxxxxxxxxxxxxxxx
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/18    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------

xtnkk_tools_pro/tools.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    tools
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/17    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import hashlib, json, math
+from urllib.parse import urlencode
+
+
+def get_md5_32(s, is_upper=False):
+    """
+        获取文本的md5值 32位
+    :param s: 文本
+    :param is_upper: 是否转大写 默认False
+    :return:
+    """
+    # s.encode()#变成bytes类型才能加密
+    m = hashlib.md5(s.encode())  # 长度是32
+    if is_upper:
+        return m.hexdigest().upper()
+    return m.hexdigest()
+
+
+def get_md5_16(s, is_upper=False):
+    """
+        获取文本的md5值 16位
+    :param s: 文本
+    :param is_upper: 是否转大写 默认False
+    :return:
+    """
+    result = get_md5_32(s, is_upper)
+    return result[8:24]
+
+
+def get_file_md5_32(file_path):
+    """
+        获取文件md5值
+    :param file_path: 文件路径
+    :return: 
+    """
+    with open(file_path, 'rb') as file:
+        data = file.read()
+        md5_hash = hashlib.md5(data).hexdigest()
+    return md5_hash
+
+
+def get_file_md5_16(file_path):
+    """
+        获取文件md5值
+    :param file_path: 文件路径
+    :return: 
+    """
+    result = get_file_md5_32(file_path)
+    return result[8:24]
+
+
+
+def get_str_to_json(str_json):
+    """
+        字符串类型的json格式 转 json
+    :param str_json: 字符串json
+    :return:
+    """
+    try:
+        new_str_json = str_json.replace("'", '"'). \
+            replace("None", "null").replace("True", "true"). \
+            replace("False", "false")
+        return json.loads(new_str_json)
+    except Exception as e:
+        return {}
+
+
+def get_build_url_with_params(url, params):
+    """
+        传入url和params拼接完整的url ->效果 https://wwww.xxxx.com/?xxx1=1&xxx2=2
+    :param url:
+    :param params:
+    :return:
+    """
+    encoded_params = urlencode(params)
+    full_url = url + "?" + encoded_params
+    return full_url
+
+
+def get_calculate_total_page(total, limit):
+    """
+        根据total和limit计算出一共有多少页
+    :param total:
+    :param limit:
+    :return:
+    """
+    if limit <= 0:
+        return 0
+    # 根据总条数和limit计算总页数
+    total_pages = math.ceil(total / limit)
+    return total_pages
+
+
+if __name__ == '__main__':
+    pass
+

xtnkk_tools_pro/tools_time.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    程序说明xxxxxxxxxxxxxxxxxxx
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/18    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import time, datetime
+
+
+def get_time_now_timestamp(is_time_10=False, is_time_13=False):
+    """
+        获取当前时间戳
+    :param is_time_10: 是否需要处理为10位的时间戳，默认不处理
+    :param is_time_13: 是否需要处理为13位的时间戳，默认不处理
+    :return:
+    """
+
+    if is_time_10:
+        val = int(time.time())
+    elif is_time_13:
+        val = int(time.time() * 1000)
+    else:
+        val = time.time()
+    return val
+
+
+def get_time_now_day0_timestamp(is_time_13=False):
+    """
+        获取当天0点时间戳
+    :param is_time_13: 是否需要处理为13位的时间戳，默认不处理并且返回10位时间戳
+    :return:
+    """
+    val = time.mktime(datetime.date.today().timetuple())
+    if is_time_13:
+        return int(val * 1000)
+    else:
+        return int(val)
+
+
+def get_time_now_day59_timestamp(is_time_13=False):
+    """
+        获取当天23:59:59点时间戳
+    :param is_time_13: 是否需要处理为13位的时间戳，默认不处理并且返回10位时间戳
+    :return:
+    """
+    # 获取当前日期时间
+    now = datetime.datetime.now()
+    # 设置小时、分钟、秒为 23:59:59
+    last_second = now.replace(hour=23, minute=59, second=59)
+    # 转换为时间戳
+    timestamp = time.mktime(last_second.timetuple())
+    # 转换为整数类型
+    if is_time_13:
+        return get_time_10_to_13_timestamp(timestamp)
+    else:
+        return int(timestamp)
+
+
+def get_time_x_day_timestamp(x, is_time_13=False):
+    """
+        获取x天的0点的时间戳
+    :param x: 0:当天; 1:1天后; -1:一天前
+    :param is_time_13: 是否需要处理为13位的时间戳，默认不处理并且返回10位时间戳
+    :return:
+    """
+    if x == 0:
+        date_string = datetime.datetime.now().strftime("%Y-%m-%d")  # 当天日期
+    elif x > 0:
+        future_date = datetime.datetime.now() + datetime.timedelta(days=x)
+        date_string = future_date.strftime("%Y-%m-%d")  # x天后的日期
+    else:
+        past_date = datetime.datetime.now() - datetime.timedelta(days=abs(x))
+        date_string = past_date.strftime("%Y-%m-%d")  # x天前的日期
+
+    timestamp = get_time_datestr_to_timestamp(date_string=date_string, is_time_13=is_time_13)
+    return timestamp
+
+
+def get_time_datestr_to_timestamp(date_string, date_format="%Y-%m-%d", is_time_13=False):
+    """
+        根据日期格式转换为时间戳，date_string和date_format需要配合，自行传参修改，这里是以%Y-%m-%d为格式也就是2024-04-18
+    :param date_string: 字符串类型的日期格式 例如：2024-04-18
+    :param date_format: 时间格式
+    :param is_time_13: 是否需要处理为13位的时间戳，默认不处理并且返回10位时间戳
+    :return:
+    """
+    date_obj = datetime.datetime.strptime(date_string, date_format)
+    timestamp = date_obj.timestamp()
+    if is_time_13:
+        return get_time_10_to_13_timestamp(timestamp)
+    else:
+        return int(timestamp)
+
+
+def get_time_10_to_13_timestamp(timestamp):
+    """
+        10位时间戳转13位时间戳
+    :param timestamp:
+    :return:
+    """
+    val = int(timestamp)
+    if len(str(val)) == 10:
+        return int(val * 1000)
+    return val
+
+
+def get_time_13_to_10_timestamp(timestamp):
+    """
+        13位时间戳转10位时间戳
+    :param timestamp:
+    :return:
+    """
+    val = int(timestamp)
+    if len(str(val)) == 13:
+        return int(val // 1000)
+    return val
+
+
+def get_time_timestamp_to_datestr(format='%Y-%m-%d %H:%M:%S', now_time=0):
+    """
+        根据时间戳转换为日期格式，兼容10位时间戳和13位时间戳
+    :param format: 日期格式，常用：%Y-%m-%d %H:%M:%S、%Y-%m-%d、%Y/%m/%d、%H:%M:%S ...
+    :param now_time: 时间戳，默认0表示当前时间戳
+    :return:
+    """
+    # 根据格式获取当前转换好的时间
+    if not now_time:
+        now_time = get_time_now_timestamp()
+    now_time = get_time_13_to_10_timestamp(now_time)
+    val = time.strftime(format, time.localtime(now_time))
+    return val
+
+
+if __name__ == '__main__':
+    pass
+    print(get_time_timestamp_to_datestr())
+    print(get_time_timestamp_to_datestr(format="%H:%M:%S", now_time=get_time_now_timestamp(is_time_10=True)))
+    print(get_time_timestamp_to_datestr(now_time=get_time_now_timestamp(is_time_13=True)))