rgwfuncs 0.0.93__py3-none-any.whl → 0.0.95__py3-none-any.whl

rgwfuncs/df_lib.py

@@ -2085,13 +2085,14 @@ def sync_dataframe_to_sqlite_database(
             conn.execute(f"ALTER TABLE {new_table_name} RENAME TO {tablename}")
 
 
-def load_fresh_data_or_pull_from_cache(fetch_func: Callable[[], pd.DataFrame], cache_dir: str, file_prefix: str, cache_cutoff_hours: int) -> pd.DataFrame:
+def load_fresh_data_or_pull_from_cache(fetch_func: Callable[[], pd.DataFrame], cache_dir: str, file_prefix: str, cache_cutoff_hours: int, dtype: dict = None) -> pd.DataFrame:
     """
-    Retrieve data from a cache if a recent cache file exists, or fetch fresh data, save it to the cache, and return it.
+    Retrieve data from a cache if a recent cache file exists, or fetch fresh data, save it to the cache, remove older cache files, and return it.
 
     This function checks a specified directory for the most recent cache file matching a specified prefix.
     If a recent cache file (within the cutoff time in hours) is found, the data is read from there.
-    Otherwise, it calls the data-fetching function, saves the newly fetched data to a new cache file, and returns it.
+    Otherwise, it calls the data-fetching function, saves the newly fetched data to a new cache file,
+    removes all earlier cache files with the same prefix, and returns the data.
 
     Parameters:
     - fetch_func (typing.Callable[[], pd.DataFrame]):
@@ -2103,16 +2104,18 @@ def load_fresh_data_or_pull_from_cache(fetch_func: Callable[[], pd.DataFrame], c
     - cache_cutoff_hours (int):
         The maximum age of a cache file (in hours) to be considered valid.
         If no file is fresh enough, fresh data will be fetched.
+    - dtype (dict, optional):
+        A dictionary specifying the data types for columns when reading the CSV cache file.
+        Passed to pd.read_csv() to handle mixed-type columns explicitly. Defaults to None.
 
     Returns:
     - pd.DataFrame:
         The pandas DataFrame containing either cached or freshly fetched data.
     """
-
     # Ensure the directory exists
     os.makedirs(cache_dir, exist_ok=True)
 
-    # Generate the current timestamp in the required format
+    # Generate the current timestamp
     now: datetime = datetime.now()
 
     # Initialize cache file details
@@ -2133,7 +2136,7 @@ def load_fresh_data_or_pull_from_cache(fetch_func: Callable[[], pd.DataFrame], c
 
     # If a valid cache exists and is within the cutoff time, read from it
     if latest_cache_time and now - latest_cache_time < timedelta(hours=cache_cutoff_hours):
-        df: pd.DataFrame = pd.read_csv(os.path.join(cache_dir, latest_cache_filename))
+        df: pd.DataFrame = pd.read_csv(os.path.join(cache_dir, latest_cache_filename), dtype=dtype)
     else:
         # Fetch new data via the provided function
         df = fetch_func()
@@ -2143,4 +2146,12 @@ def load_fresh_data_or_pull_from_cache(fetch_func: Callable[[], pd.DataFrame], c
         cache_filename: str = f"{file_prefix}_{current_time_str}.csv"
         df.to_csv(os.path.join(cache_dir, cache_filename), index=False)
 
+        # Remove all earlier cache files with the same prefix
+        for filename in os.listdir(cache_dir):
+            if filename.startswith(file_prefix) and filename.endswith(".csv") and filename != cache_filename:
+                try:
+                    os.remove(os.path.join(cache_dir, filename))
+                except OSError:
+                    continue
+
     return df

{rgwfuncs-0.0.93.dist-info → rgwfuncs-0.0.95.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rgwfuncs
-Version: 0.0.93
+Version: 0.0.95
 Summary: A functional programming paradigm for mathematical modelling and data science
 Home-page: https://github.com/ryangerardwilson/rgwfunc
 Author: Ryan Gerard Wilson
@@ -1734,32 +1734,39 @@ Processes and saves a DataFrame to an SQLite database, adding a timestamp column
 --------------------------------------------------------------------------------
 
 ### 46. `load_fresh_data_or_pull_from_cache`
-Retrieves data from a cache if a recent cache file exists, or fetches fresh data, saves it to the cache, and returns it. If the cache is too old or doesn't exist, it uses a fetching function to get new data, which it caches and returns.
 
-• Parameters:
-  - `fetch_func` (typing.Callable[[], pd.DataFrame]): A callable function that fetches fresh data and returns it as a pandas DataFrame.
-  - `cache_dir` (str): The directory where cache files are stored.
-  - `file_prefix` (str): The prefix used for cache filenames to identify relevant cache files.
-  - `cache_cutoff_hours` (int): The age in hours beyond which a cache file is considered obsolete.
+Retrieves data from a cache if a recent cache file exists, or fetches fresh data, saves it to the cache, removes older cache files, and returns it. If the cache is too old or doesn’t exist, it uses a fetching function to get new data, which it caches and returns. When fresh data is fetched and saved, all earlier cache files with the same prefix are deleted to keep the cache directory clean. An optional `dtype` parameter allows specifying column data types when reading from the cache, preventing issues with mixed-type columns.
 
-• Returns:
-  - `pd.DataFrame`: The DataFrame containing cached or freshly fetched data.
+#### Parameters:
+- **`fetch_func` (typing.Callable[[], pd.DataFrame])**: A callable function that fetches fresh data and returns it as a pandas DataFrame.
+- **`cache_dir` (str)**: The directory where cache files are stored.
+- **`file_prefix` (str)**: The prefix used for cache filenames to identify relevant cache files.
+- **`cache_cutoff_hours` (int)**: The age in hours beyond which a cache file is considered obsolete.
+- **`dtype` (dict, optional)**: A dictionary specifying the data types for columns when reading the CSV cache file. Passed to `pd.read_csv()` to handle mixed-type columns explicitly. Defaults to `None`, in which case pandas infers the types.
 
-• Example:
+#### Returns:
+- **`pd.DataFrame`**: The DataFrame containing cached or freshly unmarked data.
+
+#### Example:
 
     from rgwfuncs import load_fresh_data_or_pull_from_cache
     import pandas as pd
 
     def fetch_data():
         # This is your data-fetching logic. Replace with real fetching code.
-        return pd.DataFrame({'Column1': [1, 2, 3], 'Column2': [4, 5, 6]})
+        return pd.DataFrame({'Column1': [1, 2, 3], 'Column2': ['4', '5', '6']})
 
     cache_dir = 'cache_directory'
     file_prefix = 'cached_data'
     cache_cutoff_hours = 24
 
+    # Without dtype (pandas infers types)
     df = load_fresh_data_or_pull_from_cache(fetch_data, cache_dir, file_prefix, cache_cutoff_hours)
 
+    # With dtype to handle mixed types
+    df = load_fresh_data_or_pull_from_cache(fetch_data, cache_dir, file_prefix, cache_cutoff_hours, dtype={'Column2': str})
+    print(df)
+
 --------------------------------------------------------------------------------
 
 ## Additional Info

{rgwfuncs-0.0.93.dist-info → rgwfuncs-0.0.95.dist-info}/RECORD

@@ -1,12 +1,12 @@
 rgwfuncs/__init__.py,sha256=LSn54Tlyskcb6Wab_wUpPLB6UGMe5LdrB3GU88mDEbU,1712
 rgwfuncs/algebra_lib.py,sha256=rKFITfpWfgdBswnbMUuS41XgndEt-jUVz2ObO_ik7eM,42234
-rgwfuncs/df_lib.py,sha256=LHG6E-umLGVdDRWjziFrRb_YSlTronHv2QwEFBrTAt4,75528
+rgwfuncs/df_lib.py,sha256=BoBICtf_uy1AOuduGenJemtTTpF4alalSq0Nuyy88F4,76207
 rgwfuncs/docs_lib.py,sha256=i63NzX-V8cGhikYdtkRGAEe2VcuwpXxDUyTRa9xI7l8,1972
 rgwfuncs/interactive_shell_lib.py,sha256=YN0ZnM5twIsOeDKuOQ9ZGURCvvBX0RZjM4a1vO1C3E8,4281
 rgwfuncs/str_lib.py,sha256=hE0VfP6rhQpczsKyCZvH3G1aMRwngKnkW3NTYCEc0Po,3208
-rgwfuncs-0.0.93.dist-info/licenses/LICENSE,sha256=jLvt20gcUZYB8UOvyBvyKQ1qhYYhD__qP7ZDx2lPFkU,1062
-rgwfuncs-0.0.93.dist-info/METADATA,sha256=eEiNMD4k_feeK6mxhrVqlvDTRONwsPosvR-pj9hYlAI,61443
-rgwfuncs-0.0.93.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rgwfuncs-0.0.93.dist-info/entry_points.txt,sha256=j-c5IOPIQ0252EaOV6j6STio56sbXl2C4ym_fQ0lXx0,43
-rgwfuncs-0.0.93.dist-info/top_level.txt,sha256=aGuVIzWsKiV1f2gCb6mynx0zx5ma0B1EwPGFKVEMTi4,9
-rgwfuncs-0.0.93.dist-info/RECORD,,
+rgwfuncs-0.0.95.dist-info/licenses/LICENSE,sha256=jLvt20gcUZYB8UOvyBvyKQ1qhYYhD__qP7ZDx2lPFkU,1062
+rgwfuncs-0.0.95.dist-info/METADATA,sha256=TPX6a5GjVlF_KIUytC2J_bi5nILxBSp9ArUoMuwRC8s,62223
+rgwfuncs-0.0.95.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rgwfuncs-0.0.95.dist-info/entry_points.txt,sha256=j-c5IOPIQ0252EaOV6j6STio56sbXl2C4ym_fQ0lXx0,43
+rgwfuncs-0.0.95.dist-info/top_level.txt,sha256=aGuVIzWsKiV1f2gCb6mynx0zx5ma0B1EwPGFKVEMTi4,9
+rgwfuncs-0.0.95.dist-info/RECORD,,