mdbq 3.6.10__py3-none-any.whl → 3.6.11__py3-none-any.whl

mdbq/redis/getredis.py

@@ -343,7 +343,6 @@ class RedisDataHash(object):
             end_month = end_dt.to_period('M')
             months = pd.period_range(start_month, end_month, freq='M').strftime("%Y%m").tolist()
             cache_data = self._fetch_redis_data(cache_key, months)
-
             if cache_data.empty:
                 self._trigger_async_cache_update(
                     cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
@@ -386,10 +385,26 @@ class RedisDataHash(object):
             combined_data = self._merge_data(new_data, existing_data)
 
             if not combined_data.empty:
-                if '日期' not in combined_data.columns.tolist():
-                    serialized_data = self._serialize_data(combined_data)
-                    self.redis_engine.hset(cache_key, "all", serialized_data)
-                    self.redis_engine.expire(cache_key, self.cache_ttl)
+                if '日期' not in combined_data.columns:
+                    # 原子化删除旧分片
+                    # 优化分片存储性能
+                    chunk_size = 5000
+                    with self.redis_engine.pipeline(transaction=False) as pipe:
+                        # 批量删除旧分片
+                        for key in self.redis_engine.hscan_iter(cache_key, match="all_*"):
+                            pipe.hdel(cache_key, key[0])
+
+                        # 批量写入新分片
+                        for idx in range(0, len(combined_data), chunk_size):
+                            chunk = combined_data.iloc[idx:idx + chunk_size]
+                            chunk_key = f"all_{idx // chunk_size:04d}"
+                            pipe.hset(cache_key, chunk_key, self._serialize_data(chunk))
+
+                        pipe.expire(cache_key, self.cache_ttl)
+                        pipe.execute()
+                    # serialized_data = self._serialize_data(combined_data)
+                    # self.redis_engine.hset(cache_key, "all", serialized_data)
+                    # self.redis_engine.expire(cache_key, self.cache_ttl)
                 else:
                     # 按月分片存储
                     combined_data['month'] = combined_data['日期'].dt.to_period('M').dt.strftime("%Y%m")
@@ -422,6 +437,69 @@ class RedisDataHash(object):
             return pd.DataFrame()
 
     def _fetch_redis_data(self, cache_key: str, months: list = None) -> pd.DataFrame:
+        try:
+            dfs = []
+
+            if months is not None:
+                # 1. 获取指定月份数据
+                month_fields = months.copy()
+                month_data = self.redis_engine.hmget(cache_key, month_fields)
+
+                # 处理月份数据
+                for data, field in zip(month_data, month_fields):
+                    if data:
+                        try:
+                            df = pd.DataFrame(json.loads(data.decode("utf-8")))
+                            df = self._convert_date_columns(df)
+                            dfs.append(df)
+                        except Exception as e:
+                            logger.error(f"月份数据解析失败 {field}: {e}")
+
+                # 2. 获取所有分片数据
+                # 优化分片数据获取
+                pipeline = self.redis_engine.pipeline()
+                cursor, keys = self.redis_engine.hscan(cache_key, match="all_*")
+                while True:
+                    for key in keys:
+                        pipeline.hget(cache_key, key)
+                    if cursor == 0:
+                        break
+                    cursor, keys = self.redis_engine.hscan(cache_key, cursor=cursor, match="all_*")
+                shard_values = pipeline.execute()
+
+                # 处理分片数据
+                for value in shard_values:
+                    if value:
+                        try:
+                            df = pd.DataFrame(json.loads(value.decode("utf-8")))
+                            dfs.append(self._convert_date_columns(df))
+                        except Exception as e:
+                            logger.error(f"分片数据解析失败: {e}")
+
+            else:
+                # 原有全量获取逻辑保持不变
+                data_dict = self.redis_engine.hgetall(cache_key)
+                for field, data in data_dict.items():
+                    try:
+                        df = pd.DataFrame(json.loads(data.decode("utf-8")))
+                        df = self._convert_date_columns(df)
+                        dfs.append(df)
+                    except Exception as e:
+                        logger.error(f"Redis 数据解析失败 {field.decode()}: {e}")
+
+            # 统一合并和排序处理
+            if dfs:
+                final_df = pd.concat(dfs, ignore_index=True)
+                if '日期' in final_df.columns:
+                    final_df = final_df.sort_values('日期', ascending=False)
+                return final_df
+            return pd.DataFrame()
+
+        except Exception as e:
+            logger.error(f"Redis 数据获取失败 {cache_key}: {e}")
+            return pd.DataFrame()
+
+    def _fetch_redis_data_bak(self, cache_key: str, months: list = None) -> pd.DataFrame:
         try:
             if months is not None:
                 fields = months.copy()
@@ -435,15 +513,29 @@ class RedisDataHash(object):
                         dfs.append(df)
                 return pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
             else:
-                data_dict = self.redis_engine.hgetall(cache_key)
+                # 优化分片数据获取
+                cursor, data = self.redis_engine.hscan(cache_key, match="all_*")
                 dfs = []
-                for field, data in data_dict.items():
-                    try:
-                        df = pd.DataFrame(json.loads(data.decode("utf-8")))
-                        df = self._convert_date_columns(df)
-                        dfs.append(df)
-                    except Exception as e:
-                        logger.error(f"Redis 数据解析失败 {cache_key} 字段 {field}: {e}")
+                while True:
+                    for field, value in data.items():
+                        try:
+                            df = pd.DataFrame(json.loads(value))
+                            dfs.append(self._convert_date_columns(df))
+                        except Exception as e:
+                            logger.error(f"分片解析失败 {field}: {e}")
+                    if cursor == 0:
+                        break
+                    cursor, data = self.redis_engine.hscan(cache_key, cursor=cursor, match="all_*")
+                return pd.concat(dfs) if dfs else pd.DataFrame()
+                # data_dict = self.redis_engine.hgetall(cache_key)
+                # dfs = []
+                # for field, data in data_dict.items():
+                #     try:
+                #         df = pd.DataFrame(json.loads(data.decode("utf-8")))
+                #         df = self._convert_date_columns(df)
+                #         dfs.append(df)
+                #     except Exception as e:
+                #         logger.error(f"Redis 数据解析失败 {cache_key} 字段 {field}: {e}")
                 return pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
         except Exception as e:
             logger.error(f"Redis 数据获取失败 {cache_key}: {e}")

{mdbq-3.6.10.dist-info → mdbq-3.6.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.10
+Version: 3.6.11
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.10.dist-info → mdbq-3.6.11.dist-info}/RECORD

@@ -34,11 +34,11 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=pBgRyUrRmOlW-oXry3Hat9GahZgljvidNEDZJFn-geU,23932
+mdbq/redis/getredis.py,sha256=QAiqkxgrQf6AHgWQdIKah3FKkM5HE8TqwJdTXrlyR6c,28427
 mdbq/redis/getredis_优化hash.py,sha256=q7omKJCPw_6Zr_r6WwTv4RGSXzZzpLPkIaqJ22svJhE,29104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
-mdbq-3.6.10.dist-info/METADATA,sha256=D8ooXZMsVBNM_wbcXjE4xq2wHJU200gXHbEPkRpKioA,244
-mdbq-3.6.10.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.6.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.6.10.dist-info/RECORD,,
+mdbq-3.6.11.dist-info/METADATA,sha256=nf9h8l9QqT6ZrZ-J4cassVWcqRi2r3Oicu9eicCtCaA,244
+mdbq-3.6.11.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-3.6.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.6.11.dist-info/RECORD,,