mdbq 3.6.12__py3-none-any.whl → 3.6.13__py3-none-any.whl

mdbq/mysql/s_query.py

@@ -49,81 +49,78 @@ class QueryDatas:
             columns = cursor.fetchall()
             return columns
 
-    def data_to_df(self, db_name, table_name, start_date, end_date, projection: dict=[]):
+    def data_to_df(self, db_name, table_name, start_date, end_date, projection: dict = None):
         """
-        projection = {'日期': 1, '场景名字': 1,}
+        从数据库表获取数据到DataFrame，支持列筛选和日期范围过滤
+        Args:
+            db_name: 数据库名
+            table_name: 表名
+            start_date: 起始日期（包含）
+            end_date: 结束日期（包含）
+            projection: 列筛选字典，e.g. {'日期': 1, '场景名字': 1}
         """
-        if start_date:
-            start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
-        else:
-            start_date = '1970-01-01'
-        if end_date:
-            end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
-        else:
-            end_date = datetime.datetime.today().strftime('%Y-%m-%d')
-        df = pd.DataFrame()  # 初始化df
-
-        if self.check_infos(db_name, table_name) == False:
+        # 初始化默认参数
+        projection = projection or {}
+        df = pd.DataFrame()
+        # 日期处理
+        start_date = pd.to_datetime(start_date or '1970-01-01').strftime('%Y-%m-%d')
+        end_date = pd.to_datetime(end_date or datetime.datetime.today()).strftime('%Y-%m-%d')
+
+        # 前置检查
+        if not self.check_infos(db_name, table_name):
             return df
 
-        self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # 配置数据库连接
+        self.config['database'] = db_name
+        connection = None
 
-        with connection.cursor() as cursor:
-            # 3. 获取数据表的所有列信息
-            sql = 'SELECT `COLUMN_NAME` FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-            cursor.execute(sql, (db_name, {table_name}))
-            columns = cursor.fetchall()
-            cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+        try:
+            connection = pymysql.connect(**self.config)
+            with connection.cursor() as cursor:
+                # 获取表结构（排除id列）
+                cursor.execute(
+                    """SELECT COLUMN_NAME 
+                    FROM information_schema.columns 
+                    WHERE table_schema = %s AND table_name = %s""",
+                    (db_name, table_name)
+                )
+                cols_exist = {col['COLUMN_NAME'] for col in cursor.fetchall()} - {'id'}
+
+                # 处理列选择
+                selected_columns = []
+                if projection:
+                    selected_columns = [k for k, v in projection.items() if v and k in cols_exist]
+                    if not selected_columns:
+                        print("Warning: Projection 参数不匹配任何数据库字段")
+                        return df
+                else:
+                    selected_columns = list(cols_exist)
+                # 构建基础SQL
+                quoted_columns = [f'`{col}`' for col in selected_columns]
+                base_sql = f"SELECT {', '.join(quoted_columns)} FROM `{db_name}`.`{table_name}`"
+
+                # 添加日期条件
+                if '日期' in cols_exist:
+                    base_sql += f" WHERE 日期 BETWEEN '{start_date}' AND '{end_date}'"
+
+                # 执行查询
+                cursor.execute(base_sql)
+                result = cursor.fetchall()
+
+                # 处理结果集
+                if result:
+                    df = pd.DataFrame(result, columns=[desc[0] for desc in cursor.description])
+                    # 类型转换优化
+                    decimal_cols = [col for col in df.columns if df[col].apply(lambda x: isinstance(x, Decimal)).any()]
+                    df[decimal_cols] = df[decimal_cols].astype(float)
 
-            # 4. 构建 SQL 查询语句
-            if projection:  # 获取指定列
-                columns_in = []
-                for key, value in projection.items():
-                    if value == 1 and key in cols_exist:
-                        columns_in.append(key)  # 提取值为 1 的键并清理不在数据表的键
-                columns_in = [f"`{item}`" for item in columns_in]
-                if not columns_in:
-                    print(f'传递的参数 projection，在数据库中没有找到匹配的列，请检查 projection： {projection}')
-                    return df
-                columns_in = ', '.join(columns_in)
-                if '日期' in cols_exist:  # 不论是否指定, 只要数据表有日期，则执行
-                    sql = (f"SELECT {columns_in} FROM `{db_name}`.`{table_name}` "
-                           f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
-                else:  # 数据表没有日期列时，返回指定列的所有数据
-                    sql = f"SELECT {columns_in} FROM `{db_name}`.`{table_name}`"
-            else:  # 没有指定获取列时
-                if '日期' in cols_exist:  # 但数据表有日期，仍然执行
-                    cols_exist = [f"`{item}`" for item in cols_exist]
-                    columns_in = ', '.join(cols_exist)
-                    sql = (f"SELECT {columns_in} FROM `{db_name}`.`{table_name}` "
-                           f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
-                else:  # 没有指定获取列，且数据表也没有日期列，则返回全部列的全部数据
-                    all_col = ', '.join([f"`{item}`" for item in cols_exist if item != 'id'])
-                    sql = f"SELECT %s FROM `%s`.`%s`" % (all_col, db_name, table_name)
-            # print(sql)
-            cursor.execute(sql)
-            rows = cursor.fetchall()  # 获取查询结果
-            columns = [desc[0] for desc in cursor.description]
-            df = pd.DataFrame(rows, columns=columns)  # 转为 df
-            # 使用applymap将每个Decimal转换为float
-            df_float = df.applymap(lambda x: float(x) if isinstance(x, Decimal) else x)
-
-            if 'id' in df.columns.tolist():
-                df.pop('id')  # 默认不返回 id 列
-            if len(df) == 0:
-                print(f's_query.py -> data_to_df -> database: {db_name}, table: {table_name} 查询的数据为空1')
-            connection.close()
-            return df
+        except Exception as e:
+            print(f"Database operation failed: {str(e)}")
+        finally:
+            if connection:
+                connection.close()
 
-        # if len(df) == 0:
-        #     print(f'database: {db_name}, table: {table_name} 查询的数据为空2')
-        #     return pd.DataFrame()
-        # cv = converter.DataFrameConverter()
-        # df = cv.convert_df_cols(df)
-        # if 'id' in df.columns.tolist():
-        #     df.pop('id')  # 默认不返回 id 列
-        # return df
+        return df
 
     def columns_to_list(self, db_name, table_name,  columns_name) -> list:
         """

mdbq/redis/getredis.py

@@ -358,6 +358,7 @@ class RedisDataHash(object):
                 return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date, projection)
 
             filtered_df = self._filter_by_date_range(cache_data, start_dt, end_dt)
+
             if not filtered_df.empty:
                 if '日期' in filtered_df.columns.tolist():
                     exsit_min_date = filtered_df['日期'].min()

{mdbq-3.6.12.dist-info → mdbq-3.6.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.12
+Version: 3.6.13
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.12.dist-info → mdbq-3.6.13.dist-info}/RECORD

@@ -21,7 +21,7 @@ mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=_jFo2_OC1BNm5wEmoYiBG_TcuNNA2xUWKNhMBfgDiAM,99699
 mdbq/mysql/mysql_bak.py,sha256=_jFo2_OC1BNm5wEmoYiBG_TcuNNA2xUWKNhMBfgDiAM,99699
 mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
-mdbq/mysql/s_query.py,sha256=rUyemVsjFM7OYG_o-DYGy18aRCbMsDlpzNGNS8Un7dg,9722
+mdbq/mysql/s_query.py,sha256=gzXUZ8J4ibavAii2cTH7PsTSIkkIfow7Qa_4k8OU6yY,8698
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=GdphR7Q3psXXVuZoyJ4u_6OWn_rWlcbT0iJ-1zPT6O0,45368
@@ -34,11 +34,11 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=DKahNJeO3W3RZ-u6LsVbbGLi-CK-dZ8y3UV9dxk8YM8,26720
+mdbq/redis/getredis.py,sha256=TJjApXH1w6MA17n_bBEYtjteBZ_ZUp6OTil9uNmdgSk,26722
 mdbq/redis/getredis_优化hash.py,sha256=q7omKJCPw_6Zr_r6WwTv4RGSXzZzpLPkIaqJ22svJhE,29104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
-mdbq-3.6.12.dist-info/METADATA,sha256=La5gbnCv5fzKJ69TGAZ6HZPgeF_bc8-jJnM0D1r06Fs,244
-mdbq-3.6.12.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.6.12.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.6.12.dist-info/RECORD,,
+mdbq-3.6.13.dist-info/METADATA,sha256=YeJ-D2GfLIL744EmH07-AvofY8PBIUuWRE6Y2Yzb3So,244
+mdbq-3.6.13.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-3.6.13.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.6.13.dist-info/RECORD,,