mdbq 3.3.5__py3-none-any.whl → 3.3.7__py3-none-any.whl

mdbq/mongo/mongo.py

@@ -8,11 +8,23 @@ import pandas as pd
 import numpy as np
 import pymongo
 from functools import wraps
+import socket
+import platform
 from concurrent.futures import ThreadPoolExecutor
-from mdbq.config import get_myconf
+from mdbq.config import myconfig
 from mdbq.dataframe import converter
 
 warnings.filterwarnings('ignore')
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['remoto']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
 
 
 def rename_col(username, password, host, db_name, collection_name, old_name, new_name, port: int = 27017,):
@@ -679,30 +691,21 @@ class OptimizeDatas:
 
 
 def upload_one_dir():
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
+    if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+        conf = myconfig.main()
+        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
+        username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+            'port']
+    else:
+        conf = myconfig.main()
+        conf_data = conf['Windows']['company']['mysql']['remoto']
+        username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+            'port']
+
     p = UploadMongo(username=username, password=password, host=host, port=port, drop_duplicates=False)
     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
     print(f'{now}数据处理中...')
 
-    p.db_name = ''
-    p.collection_name = f''
-    path = os.path.join('C:\\同步空间', 'BaiduSyncdisk', '原始文件2', r'京东报表', 'JD流量来源')
-
-    for root, dirs, files in os.walk(path, topdown=False):
-        for name in files:
-            if '按天_' not in name:
-                continue
-            if name.endswith('.csv') and 'baidu' not in name:
-                df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                for col in df.columns.tolist():
-                    if '日期' in col:
-                        df[col] = df[col].apply(lambda x: pd.to_datetime(x) if x else pd.to_datetime('2099-01-01'))
-                p.df_to_mongo(df=df)
-    if p.client:
-        p.client.close()
-    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-    print(f'{now}数据完成!')
-
 
 def main():
     pass
@@ -710,7 +713,6 @@ def main():
 
 if __name__ == '__main__':
     # main()
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
     print(username, password, host, port)
 
     # for db_name in [

mdbq/mysql/recheck_mysql.py

@@ -156,7 +156,7 @@ class ReCheckMysql:
 
 
 def recheck_csv():
-    path = '/Users/xigua/数据中心/原始文件2/推广报表34324234'
+    path = ''
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~' in name or 'baidu' in name or 'Ds_' in name or 'xunlei' in name:

mdbq/spider/aikucun.py

@@ -17,8 +17,6 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.service import Service
 from mdbq.config import set_support
 from selenium.webdriver.common.keys import Keys
-from mdbq.aggregation import aggregation
-from mdbq.clean import data_clean
 from mdbq.other import ua_sj
 from mdbq.mysql import mysql
 from mdbq.config import myconfig
@@ -46,7 +44,7 @@ else:
     D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
     Share_Path = str(pathlib.Path('/Volumes/时尚事业部/01.运营部/天猫报表'))  # 共享文件根目录
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
-upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹
+upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
 
 m_engine = mysql.MysqlUpload(username='', password='', host='', port=0, charset='utf8mb4')
 company_engine = mysql.MysqlUpload(username='', password='', host='', port=0, charset='utf8mb4')
@@ -458,26 +456,6 @@ class AikuCun:
 def akucun(headless=True, date_num=10):
     akc = AikuCun()
     akc.get_data(shop_name='aikucun', date_num=date_num, headless=headless)  # 获取最近 N 天数据，0表示今天
-    # akc.clean_data()
-
-    # # 新版 数据分类
-    # dp = aggregation.DatabaseUpdate(path=upload_path)
-    # dp.new_unzip(is_move=True)
-    # dp.cleaning(is_move=False, is_except=['临时文件'])  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # # 将 self.datas 更新至数据库
-    # dp.upload_df(service_databases=[
-    #     # {'home_lx': 'mongodb'},
-    #     # {'home_lx': 'mysql'},
-    #     {'company': 'mysql'},
-    #     # {'nas': 'mysql'},
-    # ])
-    # # 数据分类
-    # c = data_clean.DataClean(path=upload_path, source_path=Source_Path)
-    # c.set_up_to_mogo = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
-    # c.set_up_to_mysql = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
-    # c.new_unzip(is_move=True, )  # 解压文件
-    # c.change_and_sort(is_except=['临时文件'])
-    # c.move_all(is_except=['临时文件'])  # 移到文件到原始文件夹
 
 
 class AikuCunNew:
@@ -529,7 +507,7 @@ class AikuCunNew:
 
 if __name__ == '__main__':
     get_cookie_aikucun()  # 登录并获取 cookies
-    akucun(date_num=10, headless=True)  # 下载数据
+    akucun(date_num=5, headless=True)  # 下载数据
 
     # a = AikuCunNew(shop_name='aikucun')
     # a.akc()

{mdbq-3.3.5.dist-info → mdbq-3.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.3.5
+Version: 3.3.7
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.3.5.dist-info → mdbq-3.3.7.dist-info}/RECORD

@@ -2,32 +2,23 @@ mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=-yzApnlqSN2L0E1YMu5ml-W827qpKQvWPCOI7jj2kzY,80264
-mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
-mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
 mdbq/aggregation/query_data.py,sha256=_5mnSFHV6xAFs_1YF_H2zMOdJeMavgga4lZQ_qpqxPQ,167637
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
-mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=yMAb6tV9XHhFJbRrCOeaPfszApJ9y5M4-hQGuBSXNqE,67799
-mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
-mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=eFu6focRqm2Njn_XN1KW2ZYJiTv6EYgsdBCLokobyxQ,21572
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/myconfig.py,sha256=EGymTlAimtHIDJ9egCtOehBEPOj6rea504kvsEZu64o,854
 mdbq/config/products.py,sha256=Ab6eaAUMUtjRL8z9NvYukyCjp3nAi4OYISY_IdPhAJ0,6279
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
-mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
-mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
+mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=ZK6E-idQWrURtoimc0uG8B1tnhtSFQXDJHfu8sWeJg4,85675
-mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
+mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
 mdbq/mysql/s_query.py,sha256=MbIprZ4yJDAZ9AahZPzl7hqS695Vs0P-AJNwAtA_EEc,9287
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -40,11 +31,9 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
-mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=BKVa0xbTkyhIH5kQgOdyPDtwFPScbMNAFYup_-fFF9Y,24809
-mdbq-3.3.5.dist-info/METADATA,sha256=ndDUyzoqoEH--Mhw_0Z7xXEPTwWuLyBkT8jsfUx8AxA,243
-mdbq-3.3.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.3.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.3.5.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=UFY-TwlvquEYK58rTdRuv5Wx3KA21m-bIrwvvfPRyOk,23749
+mdbq-3.3.7.dist-info/METADATA,sha256=aMT3CEx_q-0vG6CncxkAXv1PPk2RsieIZxrkYyk1jPE,243
+mdbq-3.3.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.3.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.3.7.dist-info/RECORD,,

mdbq/aggregation/df_types.py

@@ -1,188 +0,0 @@
-# -*- coding:utf-8 -*-
-import warnings
-import pandas as pd
-import numpy as np
-import chardet
-import zipfile
-
-from numpy import dtype
-from pandas.tseries.holiday import next_monday
-from pyzipper import PyZipFile
-import os
-import platform
-import json
-import pymysql
-from mdbq.mongo import mongo
-from mdbq.mysql import mysql
-from mdbq.mysql import s_query
-from mdbq.config import get_myconf
-from mdbq.config import set_support
-from mdbq.dataframe import converter
-import datetime
-import time
-import re
-import shutil
-import getpass
-
-from sqlalchemy.dialects.postgresql.pg_catalog import pg_get_serial_sequence
-
-warnings.filterwarnings('ignore')
-"""
-1. 记录 dataframe 或者数据库的列信息(dtypes)
-2. 更新 mysql 中所有数据库的 dtypes 信息到本地 json
-"""
-
-
-class DataTypes:
-    """
-     数据简介: 记录 dataframe 或者数据库的列信息(dtypes)，可以记录其信息或者加载相关信息用于入库使用，
-     第一字段为分类(如 dataframe/mysql)，第二字段为数据库名，第三字段为集合名，第四段列名及其数据类型
-    """
-    def __init__(self, path=None):
-        self.datas = {
-            "json统计":
-                {
-                    "字段量": 0,
-                    "数据库量": 0,
-                    "集合数量": 0
-                }
-        }
-        self.path = path
-        if not self.path:
-            self.path = set_support.SetSupport(dirname='support').dirname
-        self.json_file = os.path.join(self.path, 'df_types.json')
-        if not os.path.isdir(self.path):
-            os.makedirs(self.path)
-        if not os.path.isfile(self.json_file):
-            with open(self.json_file, 'w', encoding='utf-8_sig') as f:
-                json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
-        self.json_before()
-
-    def json_before(self):
-        """ 本地 json 文件的 dtypes 信息, 初始化更新给 self.datas """
-        with open(self.json_file, 'r', encoding='utf-8_sig') as f:
-            json_ = json.load(f)
-            self.datas.update(json_)
-
-    def get_df_types(self, db_name, collection_name, df=pd.DataFrame(), is_file_dtype=True):
-        """
-        读取 df 的 dtypes, 并更新本地 json 文件
-        期间会 清理不合规的列名, 并对数据类型进行转换(尝试将 object 类型转为 int 或 float)
-        返回: df 的 dtypes, 后续使用示例: df = df.astype(dtypes, errors='ignore')
-        is_file_dtype=True: 默认情况下以旧 json 优先, 即允许手动指定 json 文件里面的数据类型
-        """
-        if len(df) == 0:
-            return
-        cv = converter.DataFrameConverter()
-        df = cv.convert_df_cols(df=df)  # 清理 dataframe 非法值
-        dtypes = df.dtypes.apply(str).to_dict()
-        dtypes = {db_name: {collection_name: dtypes}}
-
-        if not self.datas:  # 如果不存在本地 json 文件, 直接返回即可
-            self.datas.update(dtypes)
-            return self.datas[db_name][collection_name]
-        else:  # 存在则读取，并更新 df 的 dtypes
-            if db_name in list(self.datas.keys()):  # ['京东数据2', '推广数据2', '生意参谋2', '生意经2']
-                if collection_name in list(self.datas[db_name].keys()):
-                    if is_file_dtype:  # 旧数据优先
-                        # # 用 dtypes 更新, 允许手动指定 json 文件里面的数据类型
-                        dtypes[db_name][collection_name].update(self.datas[db_name][collection_name])
-                        # 将 dtypes 更新进去，使 self.datas 包含新旧信息
-                        self.datas[db_name][collection_name].update(dtypes[db_name][collection_name])
-                    else:  # 新数据优先
-                        self.datas[db_name][collection_name].update(dtypes[db_name][collection_name])
-                else:
-                    if is_file_dtype:  # 旧数据优先
-                        dtypes[db_name].update(self.datas[db_name])
-                        self.datas[db_name].update(dtypes[db_name])
-                    else:
-                        self.datas[db_name].update(dtypes[db_name])
-            else:
-                # dtypes.update(self.datas)  # 可以注释掉, 因为旧数据 self.datas 是空的
-                self.datas.update(dtypes)
-            dbs = 0
-            collections = 0
-            cols = 0
-            # self.datas.pop('json统计')
-            for k, v in self.datas.items():
-                if k == 'json统计':
-                    continue
-                dbs += 1
-                for d, j in v.items():
-                    collections += 1
-                    for t, p in j.items():
-                        cols += 1
-            tips = {'json统计': {'数据库量': dbs, '集合数量': collections, '字段量': cols}}
-            self.datas.update(tips)
-            return self.datas[db_name][collection_name]  # 返回 df 的 dtypes
-
-    def as_json_file(self):
-        """ 保存为本地 json 文件 """
-        self.datas = {k: 'null' if v is None else v for k, v in self.datas.items()}  # 替换字典中，值存在空值的值
-        self.datas = {k if k != None else 'null': v for k, v in self.datas.items()}  # 替换字典中，键存在空值的键
-        if 'null' in str(self.datas):
-            print(f'self.datas 数据中存在空值，可能有未匹配的数据库名或数据表名，请检查 《标题对照表.csv》，已取消写入 df_types.json ')
-            print('self.datas: ', self.datas)
-            return
-        with open(self.json_file, 'w', encoding='utf-8_sig') as f:
-            json.dump(
-                self.datas,
-                f,
-                ensure_ascii=False,  # 默认True，非ASCII字符将被转义。如为False，则非ASCII字符会以\uXXXX输出
-                sort_keys=True,  # 默认为False。如果为True，则字典的输出将按键排序。
-                indent=4,
-            )
-        time.sleep(1)
-
-    def df_dtypes_to_json(self, db_name, collection_name, path, df=pd.DataFrame(), is_file_dtype=True):
-        if len(df) == 0:
-            return
-        cv = converter.DataFrameConverter()
-        df = cv.convert_df_cols(df=df)  # 清理 dataframe 列名的不合规字符
-        dtypes = df.dtypes.apply(str).to_dict()
-        dtypes = {'dataframe': {db_name: {collection_name: dtypes}}}
-        self.dtypes_to_json(dtypes=dtypes, cl='dataframe', db_name=db_name, collection_name=collection_name, path=path, is_file_dtype=is_file_dtype)
-
-    def load_dtypes(self, db_name, collection_name):
-        if db_name in list(self.datas.keys()):
-            if collection_name in list(self.datas[db_name].keys()):
-                return self.datas[db_name][collection_name]
-            else:
-                print(f'不存在的集合名信息: {collection_name}, 文件位置: {self.json_file}')
-                return {}
-        else:
-            print(f'不存在的数据库信息: {db_name}, 文件位置: {self.json_file}')
-            return {}
-
-
-def update_df_types_to_json(file, db_name, collection_name, is_file_dtype=True):
-    """ 更新一个文件的 dtype 信息到 json 文件 """
-    df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
-    df_to_json = DataTypes()
-    df_to_json.get_df_types(
-        df=df,
-        db_name=db_name,
-        collection_name=collection_name,
-        is_file_dtype=is_file_dtype,  # 日常需开启文件优先, 正常不要让新文件修改 json 已有的类型
-    )
-    df_to_json.as_json_file()
-    print(f'json文件已存储: {df_to_json.json_file}')
-
-
-def test_load_dtypes(db_name, collection_name):
-    d = DataTypes()
-    res = d.load_dtypes(db_name=db_name, collection_name=collection_name)
-    print(res)
-
-
-if __name__ == '__main__':
-    file = '/Users/xigua/数据中心/pandas数据源/店铺日报.csv'
-    update_df_types_to_json(
-        file=file,
-        db_name='pandas数据源',
-        collection_name='店铺日报',
-        is_file_dtype=True,
-    )
-    # test_load_dtypes(db_name='pandas数据源', collection_name='店铺日报')
-
-

mdbq/aggregation/mysql_types.py

@@ -1,240 +0,0 @@
-# -*- coding:utf-8 -*-
-import warnings
-import pandas as pd
-import os
-import platform
-import json
-import pymysql
-import socket
-from mdbq.mongo import mongo
-from mdbq.mysql import mysql
-from mdbq.mysql import s_query
-from mdbq.config import myconfig
-from mdbq.config import set_support
-from mdbq.dataframe import converter
-import datetime
-import time
-import re
-
-from sqlalchemy.dialects.postgresql.pg_catalog import pg_get_serial_sequence
-
-warnings.filterwarnings('ignore')
-"""
-1. 记录 dataframe 或者数据库的列信息(dtypes)
-2. 更新 mysql 中所有数据库的 dtypes 信息到本地 json
-"""
-
-
-class DataTypes:
-    """
-     数据简介: 记录 dataframe 或者数据库的列信息(dtypes)，可以记录其信息或者加载相关信息用于入库使用，
-     第一字段为分类(如 dataframe/mysql)，第二字段为数据库名，第三字段为集合名，第四段列名及其数据类型
-    """
-    def __init__(self, path=None, service_name=None):
-        self.datas = {
-            '_json统计':
-                {
-                    '分类': 0,
-                    '数据库量': 0,
-                    '集合数量': 0,
-                    '字段量': 0,
-                    '数据简介': '记录数据库各表的数据类型信息',
-                }
-        }
-        self.path = path
-        if not self.path:
-            self.path = set_support.SetSupport(dirname='support').dirname
-        self.service_name = service_name
-        if not self.service_name:
-            self.service_name = 'xigua_lx'
-        self.json_file = os.path.join(self.path, f'mysql_types_{self.service_name}.json')
-        if not os.path.isdir(self.path):
-            os.makedirs(self.path)
-        if not os.path.isfile(self.json_file):
-            with open(self.json_file, 'w', encoding='utf-8_sig') as f:
-                json.dump(self.datas, f, ensure_ascii=False, sort_keys=True, indent=4)
-        self.json_before()
-
-    def json_before(self):
-        """ 本地 json 文件的 dtypes 信息, 初始化更新给 self.datas """
-        with open(self.json_file, 'r', encoding='utf-8_sig') as f:
-            json_ = json.load(f)
-            self.datas.update(json_)
-
-    def get_mysql_types(self, cl, dtypes, db_name, table_name, is_file_dtype=True):
-        """ 更新 mysql 的 types 信息到 json 文件 """
-        if cl in self.datas.keys():
-            if db_name in list(self.datas[cl].keys()):  # ['京东数据2', '推广数据2', '生意参谋2', '生意经3']
-                if table_name in list(self.datas[cl][db_name].keys()):
-                    if is_file_dtype:  # 旧数据优先
-                        # # 用 dtypes 更新, 允许手动指定 json 文件里面的数据类型
-                        dtypes[cl][db_name][table_name].update(self.datas[cl][db_name][table_name])
-                        # 将 dtypes 更新进去，使 self.datas 包含新旧信息
-                        self.datas[cl][db_name][table_name].update(dtypes[cl][db_name][table_name])
-                    else:  # 新数据优先
-                        self.datas[cl][db_name][table_name].update(dtypes[cl][db_name][table_name])
-                else:
-                    if is_file_dtype:  # 旧数据优先
-                        dtypes[cl][db_name].update(self.datas[cl][db_name])
-                        self.datas[cl][db_name].update(dtypes[cl][db_name])
-                    else:
-                        self.datas[cl][db_name].update(dtypes[cl][db_name])
-            else:
-                # dtypes.update(self.datas)  # 可以注释掉, 因为旧数据 self.datas 是空的
-                self.datas[cl].update(dtypes[cl])
-        else:
-            self.datas.update(dtypes)
-
-        cif = 0  # 分类
-        dbs = 0  # 数据库
-        collections = 0  # 集合
-        cols = 0  # 字段
-        for k, v in self.datas.items():
-            if k == '_json统计':
-                continue  # 不统计头信息
-            cif += 1
-            for t, g in v.items():
-                dbs += 1
-                for d, j in g.items():
-                    collections += 1
-                    for t, p in j.items():
-                        cols += 1
-        tips = {'分类': cif, '数据库量': dbs, '集合数量': collections, '字段量': cols}
-        self.datas['_json统计'].update(tips)
-        # with open(json_file, 'w', encoding='utf-8_sig') as f:
-        #     json.dump(
-        #         self.datas,
-        #         f,
-        #         ensure_ascii=False,  # 默认True，非ASCII字符将被转义。如为False，则非ASCII字符会以\uXXXX输出
-        #         sort_keys=True,  # 默认为False。如果为True，则字典的输出将按键排序。
-        #         indent=4,
-        #     )
-
-    def as_json_file(self):
-        """ 保存为本地 json 文件 """
-        with open(self.json_file, 'w', encoding='utf-8_sig') as f:
-            json.dump(
-                self.datas,
-                f,
-                ensure_ascii=False,  # 默认True，非ASCII字符将被转义。如为False，则非ASCII字符会以\uXXXX输出
-                sort_keys=True,  # 默认为False。如果为True，则字典的输出将按键排序。
-                indent=4,
-            )
-        print(f'已更新 json 文件: {self.json_file}')
-        time.sleep(1)
-
-    def load_dtypes(self, db_name, table_name, cl='mysql', ):
-        """
-        mysql.py 程序从本地文件中读取 dtype 信息
-        如果缺失 dtypes 信息，则执行 mysql_all_dtypes 以便更新所有数据库 dtypes 信息到 json 文件
-        """
-        if cl in self.datas.keys():
-            if db_name in list(self.datas[cl].keys()):
-                if table_name in list(self.datas[cl][db_name].keys()):
-                    return self.datas[cl][db_name][table_name], None, None, None
-                else:
-                    print(f'不存在的集合名信息: {table_name}, 文件位置: {self.json_file}')
-                    # mysql_all_dtypes(db_name=db_name, table_name=table_name)   # 更新一个表的 dtypes
-                    return {}, cl, db_name, table_name
-            else:
-                print(f'不存在的数据库信息: {db_name}, 文件位置: {self.json_file}')
-                # mysql_all_dtypes(db_name=db_name)  # 更新一个数据库的 dtypes
-                return {}, cl, db_name, None
-        else:
-            print(f'不存在的数据分类: {cl}, 文件位置: {self.json_file}')
-            # mysql_all_dtypes()  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
-            return {}, cl, None, None  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
-
-
-def mysql_all_dtypes(db_name=None, table_name=None, path=None):
-    """
-    更新 mysql 中所有数据库的 dtypes 信息到本地 json
-    """
-    username, password, host, port, service_name = None, None, None, None, None
-    conf = myconfig.main()
-    if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
-        data = conf['Windows']['xigua_lx']['mysql']['local']
-        username, password, host, port = data['username'], data['password'], data['host'], data['port']
-        service_name = 'xigua_lx'  # 影响 mysql_types_xigua_lx.json 文件名
-    elif socket.gethostname() in ['company', 'Mac2.local']:
-        data = conf['Windows']['company']['mysql']['local']
-        username, password, host, port = data['username'], data['password'], data['host'], data['port']
-        service_name = 'company'  # 影响 mysql_types_company.json 文件名
-    if not username or not service_name:
-        return
-
-    config = {
-        'host': host,
-        'port': int(port),
-        'user': username,
-        'password': password,
-        'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
-        'cursorclass': pymysql.cursors.DictCursor,
-    }
-    connection = pymysql.connect(**config)  # 连接数据库
-    with connection.cursor() as cursor:
-        sql = "SHOW DATABASES;"
-        cursor.execute(sql)
-        db_name_lists = cursor.fetchall()
-        db_name_lists = [item['Database'] for item in db_name_lists]
-        connection.close()
-
-    sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
-    db_name_lists = [item for item in db_name_lists if item not in sys_lists]
-
-    results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经3': 'e3_零售明细统计'}]
-    for db_ in db_name_lists:
-        config.update({'database': db_})  # 添加更新 config 字段
-        connection = pymysql.connect(**config)  # 连接数据库
-        try:
-            with connection.cursor() as cursor:
-                sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_}';"
-                sql = "SHOW TABLES;"
-                cursor.execute(sql)
-                res_tables = cursor.fetchall()
-                for res_table in res_tables:
-                    for k, v in res_table.items():
-                        results.append({db_: v})
-        except:
-            pass
-        finally:
-            connection.close()
-        time.sleep(0.5)
-
-    d = DataTypes(path=path, service_name=service_name)
-    for result in results:
-        for db_n, table_n in result.items():
-            # print(db_n, table_n, db_name, table_name)
-            if db_name and table_name:  # 下载一个指定的数据表
-                if db_name != db_n or table_name != table_n:
-                    continue
-            elif db_name:  # 下载一个数据库的所有数据表
-                if db_name != db_n:
-                    continue
-            # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
-            print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
-            sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-            # 获取数据表的指定列, 返回列表
-            # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
-            name_type = sq.dtypes_to_list(db_name=db_n, table_name=table_n)
-            if name_type:
-                dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
-                dtypes = {'mysql': {db_n: {table_n: dtypes}}}
-                d.get_mysql_types(
-                    dtypes=dtypes,
-                    cl='mysql',
-                    db_name=db_n,
-                    table_name=table_n,
-                    is_file_dtype=True  # True表示旧文件有限
-                )
-            else:
-                print(f'数据库回传数据(name_type)为空')
-        # print(d.datas)
-    d.as_json_file()  # 2024.11.05 改
-
-
-if __name__ == '__main__':
-    # 更新 mysql 中所有数据库的 dtypes 信息到本地 json
-    mysql_all_dtypes(
-        path='/Users/xigua/Downloads',
-    )

mdbq/clean/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# 数据清洗, 此版本不再维护