xtn-tools-pro 1.0.0.0.1__py3-none-any.whl → 1.0.0.0.3__py3-none-any.whl

xtn_tools_pro/db/RedisDB.py

@@ -27,6 +27,9 @@ class RedisDBPro:
         self.__redis = None
         self.get_connect()
 
+    def __del__(self):
+        self.__redis.close()
+
     @classmethod
     def from_url(cls, url):
         """
@@ -110,9 +113,53 @@ class RedisDBPro:
         else:
             return self._redis.sadd(table, values)
 
+    def incr(self, key):
+        """
+            对一个键的值进行自增操作
+        :param key: 需要自增的key
+        :return:
+        """
+        return self._redis.incr(key)
+
+    def get_all_key(self, path):
+        """
+            获取所有的key
+            常用的path：前缀为test的key test*，中间为test的key *test*
+        :param path:
+        :return:
+        """
+        return list(self._redis.scan_iter(path))
+
+    def get(self, table):
+        return self._redis.get(table)
+
+    def set(self, table, value, **kwargs):
+        """
+            字符串 set
+        :param table: 表
+        :param value: 值
+        :param kwargs: 参数解释为chatgpt提供
+        :param kwargs: ex（可选）：设置键的过期时间，以秒为单位。例如，ex=10表示键将在10秒后过期
+        :param kwargs: px（可选）：设置键的过期时间，以毫秒为单位。例如，px=10000表示键将在10秒后过期
+        :param kwargs: nx（可选）：如果设置为True，则只有在键不存在时才设置键的值
+        :param kwargs: xx（可选）：如果设置为True，则只有在键已存在时才设置键的值
+        :param kwargs: kepp_ttl（可选）：如果设置为True，则保留键的过期时间。仅当键已存在且设置了过期时间时才有效
+        :param kwargs: exat（可选）：设置键的过期时间，以UNIX时间戳表示。
+        :param kwargs: pxat（可选）：设置键的过期时间，以毫秒级的UNIX时间戳表示。
+        :param kwargs: replace（可选）：如果设置为True，则无论键是否存在，都会设置键的值。
+        :return:
+        """
+        return self._redis.set(table, value, **kwargs)
+
+    def delete(self, table):
+        return self._redis.delete(table)
+
 
 if __name__ == '__main__':
     pass
     # r = RedisDBPro(ip="127.0.0.1", port=6379, db=0, user_pass="xtn-kk")
+    r = RedisDBPro.from_url('redis://:xtn-kk@127.0.0.1:6379/0')
     # status = r.sadd("test_redis_pro", [1, 2, 3, 4, 5, "6", "7"])
     # print(status)
+    # print(r.get_all_key("*http*"))
+    print(r.delete("test_redis_pro"))

xtn_tools_pro/proxy/XiaoXiangProxy.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    小象代理专用
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/27    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import requests, time, random
+from xtn_tools_pro.db.RedisDB import RedisDBPro
+from xtn_tools_pro.tools_time import get_time_now_timestamp, get_time_now_day59_timestamp
+
+import warnings
+from urllib3.exceptions import InsecureRequestWarning
+
+warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+
+
+class ProxyPool:
+    def __init__(self, ip, port, db=0, user_pass="", redis_proxy_name="XiaoXiangProxy",
+                 XiaoXiangProxyAppKey=None, XiaoXiangProxyAppSecret=None, usage_cnt=100, usage_time=100):
+        """
+            小象代理专用
+        :param ip: redis 数据库 ip
+        :param port: redis 数据库 端口
+        :param db: redis 数据库 db
+        :param user_pass: redis 数据库 密码
+        :param redis_proxy_name: redis 数据库 用于存储代理的key
+        :param XiaoXiangProxyAppKey: 小象代理 应用id appKey
+        :param XiaoXiangProxyAppSecret: 小象代理 应用密码 appSecret
+        :param usage_cnt: 每个代理最长使用次数 单位秒 维护代理时用
+        :param usage_time: 每个代理最长使用时间 单位秒 维护代理时用
+        :param is_log: 是否记录日志
+        """
+        if not XiaoXiangProxyAppSecret or not XiaoXiangProxyAppKey:
+            raise Exception("应用密码或应用id 不能为空")
+
+        r = RedisDBPro(ip=ip, port=port, db=db, user_pass=user_pass)
+        self.__redis_pool = r
+        self.__redisProxyName = redis_proxy_name
+        self.__XiaoXiangProxyAPI = "https://api.xiaoxiangdaili.com/ip/get?appKey={appKey}&appSecret={appSecret}&cnt=&wt=json".format(
+            appKey=XiaoXiangProxyAppKey, appSecret=XiaoXiangProxyAppSecret)
+        self.__XiaoXiangAutoBinding = "https://api.xiaoxiangdaili.com/app/bindIp?appKey={appKey}&appSecret={appSecret}&i=1".format(
+            appKey=XiaoXiangProxyAppKey, appSecret=XiaoXiangProxyAppSecret)
+
+        # 获取当天0点时间戳
+        self.__now_day59_timestamp = get_time_now_day59_timestamp()
+        self.__usage_cnt = usage_cnt
+        self.__usage_time = usage_time
+
+        # if is_log:
+        #     # 日志
+        #     nowDate = str(datetime.datetime.now().strftime('%Y_%m_%d'))
+        #     logger.add(loggerPath.format(t=nowDate))
+
+    def __log(self, text):
+        """
+            记录日志
+        :param text:
+        :return:
+        """
+        print(text)
+
+    def __check_proxy(self):
+        """
+            维护检查代理，删除无用代理
+            删除标准：1.代理使用超过xx次；2.使用时间超过xx秒；3.被爬虫标记使用次数为 999999 会被删除
+        :return:
+        """
+        proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+        for proxy_val in proxy_val_list:
+            # 获取时间
+            time_out = proxy_val.split(":")[-1]
+            # 获取使用次数
+            proxy_val_count = self.__redis_pool.get(proxy_val)
+            if int(time_out) + self.__usage_time < get_time_now_timestamp(is_time_10=True):
+                del_state = self.__redis_pool.delete(proxy_val)
+                self.__log(
+                    "当前代理状态:{proxy_val},{time_out}_{py_time}当前代理已超过使用时间,删除状态为：{del_state}".
+                        format(proxy_val=proxy_val, del_state=del_state,
+                               time_out=time_out, py_time=get_time_now_timestamp(is_time_10=True)))
+            elif int(proxy_val_count) >= self.__usage_cnt:
+                del_state = self.__redis_pool.delete(proxy_val)
+                self.__log(
+                    "当前代理状态:{proxy_val},{text},删除状态为：{del_state}".format(proxy_val=proxy_val,
+                                                                         text="当前代理被爬虫标记为不可用" if proxy_val_count >= 999999 else "当前代理已超过使用时间",
+                                                                         del_state=del_state))
+
+    def __get_proxy_length(self):
+        """
+            获取代理数
+        :return:
+        """
+        proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+        return len(proxy_val_list)
+
+    def __incr_proxy(self, proxy_val):
+        """
+            自增代理使用次数
+        :param proxy_val: 代理
+        :return:
+        """
+        proxy_val_con = self.__redis_pool.incr(proxy_val)
+        return proxy_val_con
+
+    def __get_api_proxy(self):
+        """
+            通过接口获取小象代理，并存储至数据库
+            响应：
+                {"code":1010,"success":false,"data":null,"msg":"请求过于频繁"}
+                {"code":200,"success":true,"data":[{"ip":"125.123.244.60","port":37635,"realIp":null,"startTime":"2024-04-27 14:09:42","during":2}],"msg":"操作成功"}
+        :return:
+        """
+        while True:
+            self.__check_proxy()
+            try:
+                response = requests.get(url=self.__XiaoXiangProxyAPI, verify=False, timeout=3)
+                if response.status_code == 200:
+                    if response.json().get("msg") == "请求过于频繁":
+                        self.__log("获取小象代理过于频繁，等待2s，{content}".format(content=response.text))
+                        time.sleep(2)
+                        continue
+                    # 获取data
+                    proxy_data_list = response.json().get("data", [])
+                    if not proxy_data_list:
+                        self.__log("获取小象代理失败 data 为空，等待2s，{content}".format(content=response.text))
+                        time.sleep(2)
+                        continue
+                    else:
+                        for data in proxy_data_list:
+                            ip = "http://{ip}".format(ip=data.get("ip"))
+                            port = data.get("port")
+                            time_out = get_time_now_timestamp(is_time_10=True)
+                            proxy_key = "{redis_proxy_name}:{ip}:{port}:{timeOut}".format(
+                                redis_proxy_name=self.__redisProxyName,
+                                ip=ip,
+                                port=port,
+                                timeOut=time_out,
+                            )
+                            proxy_key_con = self.__incr_proxy(proxy_key)
+                            self.__log("获取代理:{proxy_key}，插入数据库状态为{proxy_key_con}".format(proxy_key=proxy_key,
+                                                                                         proxy_key_con=proxy_key_con))
+                        return True  # 获取成功
+                else:
+                    self.__log("获取小象代理返回响应码不为200，等待2s，{content}".format(content=response.text))
+                    time.sleep(2)
+                    continue
+            except Exception as e:
+                self.__log("获取小象代理报错：{e}".format(e=e))
+
+    def run(self):
+        try:
+            # 手动绑定终端IP
+            response = requests.get(url=self.__XiaoXiangAutoBinding, verify=False, timeout=3)
+            self.__log(response.text)
+            while True:
+                # 检查代理
+                self.__check_proxy()
+                # 获取小象代理
+                self.__get_api_proxy()
+                time.sleep(1)
+                # 判断时间是否超过当前23:59分时间戳
+                if get_time_now_timestamp(is_time_10=True) >= self.__now_day59_timestamp:
+                    self.__log("时间23:59，结束循环，{t}".format(t=int(time.time())))
+                    break
+        except Exception as eee:
+            self.__log("程序异常报错:{eee}".format(eee=eee))
+            # self.__del__()
+
+    def get_proxy(self):
+        """
+            从代理池中获取代理
+        :return:
+        """
+        try:
+            while True:
+                proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+                if proxy_val_list:
+                    proxy_val = random.choice(proxy_val_list)
+                    proxy_v = ":".join(str(proxy_val).split(":")[1:-1])
+                    self.__log("获取到的代理为：{proxy_v}".format(proxy_v=proxy_v))
+                    return proxy_v
+                else:
+                    self.__log("暂无代理，等待中")
+                    time.sleep(2)
+        except Exception as e:
+            self.__log("从代理池中获取代理：{e}".format(e=e))
+
+    def set_proxy_error(self, proxy_v):
+        """
+            爬虫手动传入代理，设置为 999999 不可用
+        :param proxyV:
+        :return:
+        """
+        try:
+            self.__redis_pool.set(proxy_v, "999999")
+            self.__log("设置不可用的代理 {proxy_v} 为 999999".format(proxy_v=proxy_v))
+        except Exception as e:
+            self.__log("爬虫手动传入代理：{e}".format(e=e))
+
+
+if __name__ == '__main__':
+    p = ProxyPool(ip="127.0.0.1", port=6379, db=0, user_pass="xtn-kk", XiaoXiangProxyAppKey="1101384562594172928",
+                  XiaoXiangProxyAppSecret="PJ0QBWML")
+    print(p.get_proxy())

{xtnkk_tools → xtn_tools_pro/proxy}/__init__.py

@@ -2,9 +2,9 @@
 # -*- coding: utf-8 -*-
 
 # 说明：
-#    __init__
+#    程序说明xxxxxxxxxxxxxxxxxxx
 # History:
 # Date          Author    Version       Modification
 # --------------------------------------------------------------------------------------------------
-# 2024/4/17    xiatn     V00.01.000    新建
+# 2024/4/27    xiatn     V00.01.000    新建
 # --------------------------------------------------------------------------------------------------

xtn_tools_pro/tools_flie.py

@@ -21,7 +21,7 @@ def get_file_extension(file_name):
     return file_extension
 
 
-def get_check_filename(file_name):
+def get_file_check_filename(file_name):
     """
         传入文件名返回一个合法的文件名 会替换掉一些特殊符号 常用于爬虫写文件时文件名中带有特殊符号的情况...
     :param filename: 文件名

{xtn_tools_pro-1.0.0.0.1.dist-info → xtn_tools_pro-1.0.0.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.0.1
+Version: 1.0.0.0.3
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@163.com
@@ -10,5 +10,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pymongo
 Requires-Dist: redis
+Requires-Dist: requests
 
 xtnkk-tools

xtn_tools_pro-1.0.0.0.3.dist-info/RECORD

@@ -0,0 +1,14 @@
+xtn_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
+xtn_tools_pro/tools.py,sha256=e9KSPqaFBIptBGvexShCcn0nZmUQ5omlVwXgEfWZf5Y,2630
+xtn_tools_pro/tools_flie.py,sha256=B_P3J_R-nRLt_IFutnOVrBRGf6_SZ_cXoIoeaT9B7tk,1512
+xtn_tools_pro/tools_time.py,sha256=DMjsw9h4E_mrPsanPA8CEhpUE1AA6Z2FU4OJqJKZc1k,4867
+xtn_tools_pro/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
+xtn_tools_pro/db/RedisDB.py,sha256=ep32Yj8AAkUHRshSBhKsdl06UwO7Z-gQJLaezspVRKw,6053
+xtn_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
+xtn_tools_pro/proxy/XiaoXiangProxy.py,sha256=xrEBJZ6Cjuh6IBZVB17oXHwByaeKcWVUSSWSg17tujE,9842
+xtn_tools_pro/proxy/__init__.py,sha256=WRwh6s2lruMu5buh0ejo9EK54kWT_VQhCsFGNFAmcyo,418
+xtn_tools_pro-1.0.0.0.3.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.0.3.dist-info/METADATA,sha256=VG4Bs3muLvi85SaWcP3Tw3q4TFfafKDtZhbQ-x9CpRw,358
+xtn_tools_pro-1.0.0.0.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.0.3.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.0.3.dist-info/RECORD,,

xtn_tools_pro-1.0.0.0.1.dist-info/RECORD

@@ -1,25 +0,0 @@
-xtn_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
-xtn_tools_pro/tools.py,sha256=e9KSPqaFBIptBGvexShCcn0nZmUQ5omlVwXgEfWZf5Y,2630
-xtn_tools_pro/tools_flie.py,sha256=KuA1sowK31LuOfQMPRsQYPgxr8y7491caepRFYOwqG0,1507
-xtn_tools_pro/tools_time.py,sha256=DMjsw9h4E_mrPsanPA8CEhpUE1AA6Z2FU4OJqJKZc1k,4867
-xtn_tools_pro/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
-xtn_tools_pro/db/RedisDB.py,sha256=qMffCNIHa3o7KD_yVQlsj3OupsNXMsDPRi03migwSu0,4003
-xtn_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
-xtnkk_tools/MongoDB.py,sha256=2mwln6JPfu5N1N8Hbh6KvN6sED-KPTrOteCBHVFjvwM,5497
-xtnkk_tools/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
-xtnkk_tools/tools.py,sha256=KYoTds_c7XZBL9yLeoKksHz39QPh02DNQupRKJWx_II,2626
-xtnkk_tools/tools_time.py,sha256=n4-T2tNSHnsh-X89IbjahCmoiDcmjZTKJlWyqGOmJQY,4877
-xtnkk_tools/update.py,sha256=VygnKO9dXo02JyUEkpbJoBE6BceYARZEn-O1i6AO6E0,911
-xtnkk_tools/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
-xtnkk_tools/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
-xtnkk_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
-xtnkk_tools_pro/tools.py,sha256=KYoTds_c7XZBL9yLeoKksHz39QPh02DNQupRKJWx_II,2626
-xtnkk_tools_pro/tools_time.py,sha256=n4-T2tNSHnsh-X89IbjahCmoiDcmjZTKJlWyqGOmJQY,4877
-xtnkk_tools_pro/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
-xtnkk_tools_pro/db/RedisDB.py,sha256=qMffCNIHa3o7KD_yVQlsj3OupsNXMsDPRi03migwSu0,4003
-xtnkk_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
-xtn_tools_pro-1.0.0.0.1.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.0.1.dist-info/METADATA,sha256=cGqGoLCT0OxwP7KI82OKR1JUF9d9XnNk_Xg4DCgOSvU,333
-xtn_tools_pro-1.0.0.0.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.0.1.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.0.1.dist-info/RECORD,,

xtnkk_tools/MongoDB.py

@@ -1,137 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# 说明：
-#    MongoDBPro
-# History:
-# Date          Author    Version       Modification
-# --------------------------------------------------------------------------------------------------
-# 2024/4/17    xiatn     V00.01.000    新建
-# --------------------------------------------------------------------------------------------------
-from xtn_tools_pro.tools_time import *
-from urllib import parse
-from typing import List, Dict, Optional
-from pymongo import MongoClient
-from pymongo.database import Database
-from pymongo.collection import Collection
-from pymongo.errors import DuplicateKeyError, BulkWriteError
-
-
-class MongoDBPro:
-    def __init__(self, ip=None, port=None, db=None, user_name=None, user_pass=None, url=None, **kwargs):
-        if url:
-            self.client = MongoClient(url, **kwargs)
-        else:
-            self.client = MongoClient(host=ip,
-                                      port=port,
-                                      username=user_name,
-                                      password=user_pass,
-                                      authSource=db)
-
-        self.db = self.get_database(db)
-
-    @classmethod
-    def from_url(cls, url, **kwargs):
-        url_parsed = parse.urlparse(url)
-        # 获取 URL的协议
-        db_type = url_parsed.scheme.strip()
-        if db_type != "mongodb":
-            raise Exception(
-                "url error, expect mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]], but get {}".format(
-                    url
-                ))
-        return cls(url=url, **kwargs)
-
-    def get_database(self, database, **kwargs) -> Database:
-        """
-            根据db名获取数据库对象
-        """
-        return self.client.get_database(database, **kwargs)
-
-    def get_collection(self, coll_name, **kwargs) -> Collection:
-        """
-            根据集合名获取集合对象
-        """
-        return self.db.get_collection(coll_name, **kwargs)
-
-    def run_command(self, command: Dict):
-        """
-            参考文档 https://www.geek-book.com/src/docs/mongodb/mongodb/docs.mongodb.com/manual/reference/command/index.html
-        """
-        return self.db.command(command)
-
-    def find(self, coll_name: str, condition: Optional[Dict] = None,
-             limit: int = 0, **kwargs) -> List[Dict]:
-        """
-            find
-            coll_name:集合名称
-            condition:查询条件 例如：{"name": "John"}、{"_id": "xxxxx"}
-        """
-        condition = {} if condition is None else condition
-        command = {"find": coll_name, "filter": condition, "limit": limit}
-        command.update(kwargs)
-        result = self.run_command(command)
-        cursor = result["cursor"]
-        cursor_id = cursor["id"]
-        while True:
-            for document in cursor.get("nextBatch", cursor.get("firstBatch", [])):
-                # 处理数据
-                yield document
-            if cursor_id == 0:
-                # 游标已经完全遍历，没有剩余的结果可供获取
-                # 游标的生命周期已经结束，例如在查询会话结束后。
-                # 游标被显式地关闭，例如使用 db.killCursor() 命令关闭游标。
-                break
-            result = self.run_command(
-                {
-                    "getMore": cursor_id,  # 类似于mongo命令行中的it命令，通过索引id用于获取下一批结果
-                    "collection": coll_name,
-                    "batchSize": kwargs.get("batchSize", 100),
-                }
-            )
-            # 覆盖原来的参数
-            cursor = result["cursor"]
-            cursor_id = cursor["id"]
-            # print("下一批获取")
-
-    def add_data_one(self, coll_name: str, data: Dict, insert_ignore=False,
-                     is_add_create_time=False,
-                     is_add_create_time_field_name="create_dt"):
-        """
-            添加单条数据
-            coll_name: 集合名
-            data: 单条数据
-            insert_ignore: 索引冲突是否忽略 默认False
-            is_add_create_time: 是否在数据中添加一个创建数据10时间戳字段 默认False不创建
-            is_add_create_time_field_name: 自定义创建数据时间戳字段名：默认：create_dt
-        Returns: 插入成功的行数
-        """
-        if is_add_create_time:
-            data[is_add_create_time_field_name] = get_time_now_timestamp(is_time_10=True)
-        collection = self.get_collection(coll_name)
-        try:
-            collection.insert_one(data)
-        except DuplicateKeyError as e:
-            if not insert_ignore:
-                raise e
-            return 0
-        return 1
-
-    def find_id_is_exist(self, coll_name, _id):
-        """
-            根据id查询id是否存在
-        :param _id:id
-        :return: 存在返回True 否则False
-        """
-        condition = {"_id": _id}
-        status = list(self.find(coll_name, condition))
-        if status:
-            return True
-        return False
-
-
-if __name__ == '__main__':
-    mongo_db = MongoDBPro("127.0.0.1", 27017, "spider_pro")
-    # mongo_db.add_data_one("test", {"_id": "1", "data": "aaa"})
-    print(mongo_db.find_id_is_exist("test", "1"))
-    print(mongo_db.find_id_is_exist("test", "11"))

xtnkk_tools/db/MongoDB.py

@@ -1,138 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# 说明：
-#    MongoDBPro
-# History:
-# Date          Author    Version       Modification
-# --------------------------------------------------------------------------------------------------
-# 2024/4/17    xiatn     V00.01.000    新建
-# --------------------------------------------------------------------------------------------------
-from xtn_tools_pro.tools_time import *
-from urllib import parse
-from pymongo import MongoClient as _MongoClient
-from pymongo.database import Database as _Database
-from typing import List, Dict, Optional
-from pymongo.collection import Collection as _Collection
-from pymongo.errors import DuplicateKeyError, BulkWriteError
-
-
-class MongoDBPro:
-    def __init__(self, ip=None, port=None, db=None, user_name=None, user_pass=None, url=None, **kwargs):
-        if url:
-            self.client = _MongoClient(url, **kwargs)
-        else:
-            self.client = _MongoClient(host=ip,
-                                       port=port,
-                                       username=user_name,
-                                       password=user_pass,
-                                       authSource=db)
-
-        self.db = self.get_database(db)
-
-    @classmethod
-    def from_url(cls, url, **kwargs):
-        url_parsed = parse.urlparse(url)
-        # 获取 URL的协议
-        db_type = url_parsed.scheme.strip()
-        if db_type != "mongodb":
-            raise Exception(
-                "url error, expect mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]], but get {}".format(
-                    url
-                ))
-        return cls(url=url, **kwargs)
-
-    def get_database(self, database, **kwargs) -> _Database:
-        """
-            根据db名获取数据库对象
-        """
-        return self.client.get_database(database, **kwargs)
-
-    def get_collection(self, coll_name, **kwargs) -> _Collection:
-        """
-            根据集合名获取集合对象
-        """
-        return self.db.get_collection(coll_name, **kwargs)
-
-    def run_command(self, command: Dict):
-        """
-            参考文档 https://www.geek-book.com/src/docs/mongodb/mongodb/docs.mongodb.com/manual/reference/command/index.html
-        """
-        return self.db.command(command)
-
-    def find(self, coll_name: str, condition: Optional[Dict] = None,
-             limit: int = 0, **kwargs) -> List[Dict]:
-        """
-            find
-            coll_name:集合名称
-            condition:查询条件 例如：{"name": "John"}、{"_id": "xxxxx"}
-        """
-        condition = {} if condition is None else condition
-        command = {"find": coll_name, "filter": condition, "limit": limit}
-        command.update(kwargs)
-        result = self.run_command(command)
-        cursor = result["cursor"]
-        cursor_id = cursor["id"]
-        while True:
-            for document in cursor.get("nextBatch", cursor.get("firstBatch", [])):
-                # 处理数据
-                yield document
-            if cursor_id == 0:
-                # 游标已经完全遍历，没有剩余的结果可供获取
-                # 游标的生命周期已经结束，例如在查询会话结束后。
-                # 游标被显式地关闭，例如使用 db.killCursor() 命令关闭游标。
-                break
-            result = self.run_command(
-                {
-                    "getMore": cursor_id,  # 类似于mongo命令行中的it命令，通过索引id用于获取下一批结果
-                    "collection": coll_name,
-                    "batchSize": kwargs.get("batchSize", 100),
-                }
-            )
-            # 覆盖原来的参数
-            cursor = result["cursor"]
-            cursor_id = cursor["id"]
-            # print("下一批获取")
-
-    def add_data_one(self, coll_name: str, data: Dict, insert_ignore=False,
-                     is_add_create_time=False,
-                     is_add_create_time_field_name="create_dt"):
-        """
-            添加单条数据
-            coll_name: 集合名
-            data: 单条数据
-            insert_ignore: 索引冲突是否忽略 默认False
-            is_add_create_time: 是否在数据中添加一个创建数据10时间戳字段 默认False不创建
-            is_add_create_time_field_name: 自定义创建数据时间戳字段名：默认：create_dt
-        Returns: 插入成功的行数
-        """
-        if is_add_create_time:
-            data[is_add_create_time_field_name] = get_time_now_timestamp(is_time_10=True)
-        collection = self.get_collection(coll_name)
-        try:
-            collection.insert_one(data)
-        except DuplicateKeyError as e:
-            if not insert_ignore:
-                raise e
-            return 0
-        return 1
-
-    def find_id_is_exist(self, coll_name, _id):
-        """
-            根据id查询id是否存在
-        :param _id:id
-        :return: 存在返回True 否则False
-        """
-        condition = {"_id": _id}
-        status = list(self.find(coll_name, condition))
-        if status:
-            return True
-        return False
-
-
-if __name__ == '__main__':
-    pass
-    # mongo_db = MongoDBPro("127.0.0.1", 27017, "spider_pro")
-    # # mongo_db.add_data_one("test", {"_id": "1", "data": "aaa"})
-    # print(mongo_db.find_id_is_exist("test", "1"))
-    # print(mongo_db.find_id_is_exist("test", "11"))

xtnkk_tools/db/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-# 说明：
-#    程序说明xxxxxxxxxxxxxxxxxxx
-# History:
-# Date          Author    Version       Modification
-# --------------------------------------------------------------------------------------------------
-# 2024/4/18    xiatn     V00.01.000    新建
-# --------------------------------------------------------------------------------------------------