ccfx 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

ccfx/ccfx.py

@@ -26,6 +26,7 @@ import geopandas, pandas
 from osgeo import gdal, ogr, osr
 import py7zr
 import subprocess
+import multiprocessing
 
 
 
@@ -56,8 +57,10 @@ def getExtension(filePath:str) -> str:
     '''
     Get the extension of a file
     filePath: file path
+
+    return: file extension without the dot
     '''
-    return os.path.splitext(filePath)[1]
+    return os.path.splitext(filePath)[1].lstrip('.')
 
 
 def deleteFile(filePath:str, v:bool = False) -> bool:
@@ -86,6 +89,47 @@ def deleteFile(filePath:str, v:bool = False) -> bool:
     
     return deleted
 
+def deletePath(path:str, v:bool = False) -> bool:
+    '''
+    Delete a directory
+
+    path: directory
+    v: verbose (default is True)
+
+    return: True if the directory is deleted, False otherwise
+    '''
+    deleted = False
+    if os.path.exists(path):
+        try:
+            shutil.rmtree(path)
+            deleted = True
+        except:
+            print(f'! Could not delete {path}')
+            deleted = False
+        if v:
+            print(f'> {path} deleted')
+    else:
+        if v:
+            print(f'! {path} does not exist')
+        deleted = False
+
+
+def mergeRasterTiles(tileList:list, outFile:str) -> str:
+    '''
+    Merge raster tiles into one raster file
+    tileList: list of raster files
+    outFile: output raster file
+    '''
+    gdal.Warp(outFile, tileList)
+    return outFile
+
+def mergeRasterFiles(tileList:list, outFile:str) -> str:
+    '''
+    this function is an alias for mergeRasterTiles
+    '''
+    return mergeRasterTiles(tileList, outFile)
+
+
 def systemPlatform() -> str:
     '''
     Get the system platform
@@ -112,6 +156,55 @@ def fileCount(path:str = "./", extension:str = ".*", v:bool = True) -> int:
         print(f'> there are {count} {extension if not extension ==".*" else ""} files in {path}')
     return count
 
+def resampleRaster(inFile:str, outFile:str, resolution:float, dstSRS = None, resamplingMethod = 'bilinear', replaceOutput:bool = True, v:bool = True) -> str:
+    '''
+    Resample a raster file
+    inFile: input raster file
+    outFile: output raster file
+    resolution: resolution in the same units as the input raster
+    v: verbose (default is True)
+    available resample types:
+        'nearest', 'bilinear', 'cubic', 'cubicspline', 'lanczos', 'average', 'mode', 'max', 'min', 'med', 'q1', 'q3'
+    
+    return: output raster file path
+    '''
+
+    resamleTypes = {
+        'nearest': gdal.GRA_NearestNeighbour,
+        'bilinear': gdal.GRA_Bilinear,
+        'cubic': gdal.GRA_Cubic,
+        'cubicspline': gdal.GRA_CubicSpline,
+        'lanczos': gdal.GRA_Lanczos,
+        'average': gdal.GRA_Average,
+        'mode': gdal.GRA_Mode,
+        'max': gdal.GRA_Max,
+        'min': gdal.GRA_Min,
+        'med': gdal.GRA_Med,
+        'q1': gdal.GRA_Q1,
+        'q3': gdal.GRA_Q3
+    }
+
+    if not os.path.exists(inFile):
+        print(f'! {inFile} does not exist')
+        return None
+    
+    if os.path.exists(outFile):
+        if replaceOutput:
+            os.remove(outFile)
+        else:
+            print(f'! {outFile} already exists')      
+            return None
+    
+    if v:
+        print(f'> resampling {inFile} to {outFile} at {resolution}')
+    
+    ds = gdal.Open(inFile)
+    if dstSRS is None: gdal.Warp(outFile, ds, xRes=resolution, yRes=resolution, resampleAlg=resamleTypes[resamplingMethod])
+    else: gdal.Warp(outFile, ds, xRes=resolution, yRes=resolution, resampleAlg=resamleTypes[resamplingMethod], dstSRS=dstSRS)
+
+    ds = None
+    return outFile
+
 def watchFileCount(path:str="./", extension:str = ".*", interval:float = 0.2, duration = 3, v:bool = True) -> None:
     '''
     Watch the number of files in a directory with a specific extension
@@ -639,7 +732,7 @@ def netcdfExportTif(ncFile: str, variable: str, outputFile: str = None, band: in
     return dataset
 
 
-def getFileBaseName(filePath:str, extension:bool = False) -> str:
+def getFileBaseName(filePath:str, extension:bool = True) -> str:
     '''
     Get the base name of a file
     filePath: file path
@@ -710,12 +803,64 @@ def copyFile(source:str, destination:str, v:bool = True) -> None:
     source: source file
     destination: destination file
     '''
+    if not exists(os.path.dirname(destination)): createPath(f"{os.path.dirname(destination)}/")
     with open(source, 'rb') as src:
         with open(destination, 'wb') as dest: dest.write(src.read())
     
     if v: print(f'> {source} copied to \t - {destination}')
 
 
+def copyDirectory(source:str, destination:str, recursive = True, v:bool = True, filter = []) -> None:
+    '''
+    Copy a directory from source to destination
+    source: source directory
+    destination: destination directory
+    recursive: copy subdirectories (default is True)
+    v: verbose (default is True)
+    filter: list of file extensions to filter out
+    '''
+    if not exists(destination): os.makedirs(destination)
+
+    itemCount = None
+    counter = 1
+
+    if recursive:
+        if len(filter) > 0:
+            itemCount = len([fn for fn in listAllFiles(source) if not getExtension(fn) in filter])
+        else:
+            itemCount = len(listAllFiles(source))
+    else:
+        if len(filter) > 0:
+            itemCount = len([fn for fn in listFiles(source) if not getExtension(fn) in filter])
+        else:
+            itemCount = len(listFiles(source))
+
+
+    for item in os.listdir(source):
+        s = os.path.join(source, item)
+        d = os.path.join(destination, item)
+        if os.path.isdir(s):
+            if recursive: copyDirectory(s, d, recursive, v, filter)
+        else:
+            if len(filter) > 0:
+                if not getExtension(s) in filter:
+                    copyFile(s, d, v = False)
+                    counter += 1
+                    if v: showProgress(counter, itemCount, f'copying {getFileBaseName(item)}\t\t', barLength=50)
+            else:
+                copyFile(s, d, v = False)
+                if v: showProgress(counter, itemCount, f'copying {getFileBaseName(item)}\t\t', barLength=50)
+                counter += 1
+    print()
+
+
+def copyFolder(source:str, destination:str, v:bool = True) -> None:
+    '''
+    this function is an alias for copyDirectory
+    '''
+    copyDirectory(source, destination, v)
+
+
 def convertCoordinates(lon, lat, srcEPSG, dstCRS) -> tuple:
     """
     this function converts coordinates from one CRS to another
@@ -806,7 +951,9 @@ def showProgress(count: int, end: int, message: str, barLength: int = 100) -> No
     filled = int(barLength * count / end)
     bar = '█' * filled + '░' * (barLength - filled)
     print(f'\r{bar}| {percentStr}% [{count}/{end}] | {message}       ', end='', flush=True)
-    if count == end: print()
+    if count == end:
+        print(f'\r{bar}| {percentStr}% [{count}/{end}]                          ', end='', flush=True)
+        print()
 
 
 def listAllFiles(folder, extension="*"):
@@ -854,4 +1001,317 @@ def createPointGeometry(coords: list, proj: str = "EPSG:4326") -> geopandas.GeoD
     gdf.reset_index(inplace=True)
     return gdf
 
+def calculateTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> dict:
+    '''
+    Calculate statistics for a timeseries
+
+    the assumed dataframe structure is:
+        date - DateTime
+        observed - float
+        simulated - float
+        
+    Parameters:
+        data: pandas.DataFrame
+            DataFrame containing the timeseries data
+        observed: str
+            name of the observed column
+        simulated: str
+            name of the simulated column
+        resample: str
+            if specified, the data will be resampled to the specified frequency
+            available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    Returns:
+        dict: Dictionary containing the following statistics:
+            NSE: Nash-Sutcliffe Efficiency
+            KGE: Kling-Gupta Efficiency
+            PBIAS: Percent Bias
+            LNSE: Log Nash-Sutcliffe Efficiency
+            R2: R-squared
+            RMSE: Root Mean Square Error
+            MAE: Mean Absolute Error
+            MSE: Mean Square Error
+            MAPE: Mean Absolute Percentage Error
+            alpha: Ratio of standard deviations
+            beta: Ratio of means
+    '''
+
+    options = {'H': '1H', 'D': '1D', 'M': '1M', 'Y': '1Y'}
+
+    if resample:
+        if resample not in options:
+            raise ValueError(f"Invalid resample option. Choose from {list(options.keys())}")
+        if not 'date' in data.columns:
+            for col in data.columns:
+                if data[col].dtype == 'datetime64[ns]':
+                    data = data.set_index(col).resample(options[resample]).mean()
+                    break
+            else:
+                raise ValueError("No datetime column found for resampling.")
+        else:
+            data = data.set_index('date').resample(options[resample]).mean()
+
+    # Auto-detect columns if not specified
+    if not observed and not simulated:
+        datetime_cols = [col for col in data.columns if data[col].dtype == 'datetime64[ns]']
+        if datetime_cols:
+            data = data.drop(datetime_cols, axis=1)
+        
+        if len(data.columns) == 2:
+            observed = data.columns[0]
+            simulated = data.columns[1]
+        else:
+            raise ValueError("Could not automatically detect observed and simulated columns")
+    elif not observed or not simulated:
+        raise ValueError("Both observed and simulated columns must be specified if one is specified")
+
+    # Extract data
+    obs = data[observed].values
+    sim = data[simulated].values
+
+    # Remove any rows where either observed or simulated is NaN
+    mask = ~(numpy.isnan(obs) | numpy.isnan(sim))
+    obs = obs[mask]
+    sim = sim[mask]
+
+    if len(obs) == 0:
+        raise ValueError("No valid data points after removing NaN values")
+
+    # Calculate statistics with error handling
+    try:
+        # Nash-Sutcliffe Efficiency (NSE)
+        denominator = numpy.sum((obs - numpy.mean(obs)) ** 2)
+        nse = 1 - numpy.sum((obs - sim) ** 2) / denominator if denominator != 0 else numpy.nan
+
+        # Kling-Gupta Efficiency (KGE) components
+        r = numpy.corrcoef(obs, sim)[0, 1]
+        obs_std = numpy.std(obs)
+        sim_std = numpy.std(sim)
+        obs_mean = numpy.mean(obs)
+        sim_mean = numpy.mean(sim)
+        
+        alpha = sim_std / obs_std if obs_std != 0 else numpy.nan
+        beta = sim_mean / obs_mean if obs_mean != 0 else numpy.nan
+        
+        # KGE calculation
+        if not any(numpy.isnan([r, alpha, beta])):
+            kge = 1 - numpy.sqrt((r - 1) ** 2 + (alpha - 1) ** 2 + (beta - 1) ** 2)
+        else:
+            kge = numpy.nan
+
+        # Percent Bias (PBIAS)
+        pbias = 100 * numpy.sum(sim - obs) / numpy.sum(obs) if numpy.sum(obs) != 0 else numpy.nan
+
+        # Log Nash-Sutcliffe Efficiency (LNSE)
+        eps = 0.0001
+        log_obs = numpy.log(obs + eps)
+        log_sim = numpy.log(sim + eps)
+        log_denominator = numpy.sum((log_obs - numpy.mean(log_obs)) ** 2)
+        lnse = 1 - numpy.sum((log_obs - log_sim) ** 2) / log_denominator if log_denominator != 0 else numpy.nan
+
+        # R-squared (R2)
+        r2 = r ** 2 if not numpy.isnan(r) else numpy.nan
+
+        # Root Mean Square Error (RMSE)
+        rmse = numpy.sqrt(numpy.mean((obs - sim) ** 2))
+
+        # Mean Absolute Error (MAE)
+        mae = numpy.mean(numpy.abs(obs - sim))
+
+        # Mean Square Error (MSE)
+        mse = numpy.mean((obs - sim) ** 2)
+
+        # Mean Absolute Percentage Error (MAPE)
+        with numpy.errstate(divide='ignore', invalid='ignore'):
+            mape = numpy.mean(numpy.abs((obs - sim) / obs) * 100)
+            mape = numpy.nan if numpy.isinf(mape) else mape
+
+    except Exception as e:
+        print(f"Warning: Error in statistical calculations: {str(e)}")
+        return {stat: numpy.nan for stat in ['NSE', 'KGE', 'PBIAS', 'LNSE', 'R2', 'RMSE', 'MAE', 'MSE', 'MAPE', 'alpha', 'beta']}
+
+    return {
+        'NSE': nse,
+        'KGE': kge,
+        'PBIAS': pbias,
+        'LNSE': lnse,
+        'R2': r2,
+        'RMSE': rmse,
+        'MAE': mae,
+        'MSE': mse,
+        'MAPE': mape,
+        'alpha': alpha,
+        'beta': beta
+    }
+
+
+def getNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the NSE
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        NSE value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['NSE']
+
+def getKGE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the KGE
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        KGE value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['KGE']
+
+def getPBIAS(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the PBIAS
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        PBIAS value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['PBIAS']
+
+
+def getLNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the LNSE
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        LNSE value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['LNSE']
+
+def getR2(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the R2
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        R2 value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['R2']
+
+def getRMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the RMSE
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        RMSE value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['RMSE']
+
+def getMAE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the MAE
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        MAE value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['MAE']
+
+def getMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
+    '''
+    this function is a wrapper for calculateTimeseriesStats specifically to return the MSE
+
+    data: pandas.DataFrame
+        DataFrame containing the timeseries data
+    observed: str
+        name of the observed column
+    simulated: str
+        name of the simulated column
+    resample: str
+        if specified, the data will be resampled to the specified frequency
+        available options: 'H' (hourly), 'D' (daily), 'M' (monthly), 'Y' (yearly)
+
+    return: float
+        MSE value
+    '''
+    stats = calculateTimeseriesStats(data, observed, simulated, resample)
+
+    return stats['MSE']
+
 ignoreWarnings()

ccfx/sqliteConnection.py

@@ -49,9 +49,8 @@ class sqliteConnection:
             self.cursor.execute(query, (new_value, val_1))
             
             if v:
-                self.report(f"\t -> updated value in {self.db_name.split('/')[-1].split('\\')[-1]} table: {table_name}")
+                self.report(f"\t -> updated value in {self.db_name} table: {table_name}")
             
-            # Commit the transaction if needed (add self.conn.commit() if not in a transaction)
             
         except Exception as e:
             raise Exception(f"Error updating value: {str(e)}")

{ccfx-0.2.0.dist-info → ccfx-0.4.0.dist-info}/METADATA

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: ccfx
-Version: 0.2.0
-Summary: Your package description here
+Version: 0.4.0
+Summary: This package implifies regular coommon actions for quick prototyping in a user friendly way
 Author-email: Celray James CHAWANDA <celray@chawanda.com>
 License: MIT
 Project-URL: Homepage, https://github.com/celray/ccfx
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: netCDF4

ccfx-0.4.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+ccfx/__init__.py,sha256=VmBeF3oj6JTJ_793d4i8PvhyF8_FxaxA1L_FmHWqitc,142
+ccfx/ccfx.py,sha256=TcGb46QY57295iRfRnMwdm_fjjQ1Z-VDDlzveahRoEw,44547
+ccfx/excel.py,sha256=cQ4TQW49XqbMB3sSS0IOhO3-WArIolEBIrvOvhFyPtI,4757
+ccfx/mssqlConnection.py,sha256=TwyZXhHHI7zy6BSfH1pszuHVJ5cmndRC5dVxvEtSTks,7904
+ccfx/sqliteConnection.py,sha256=jEJ94D5ySt84N7AeDpa27Rclt1NaKhkX6nYzidwApIg,11104
+ccfx/word.py,sha256=AGa64jX5Zl5qotZh5L0QmrsjTnktIBhmj_ByRKZ88vw,3061
+ccfx-0.4.0.dist-info/LICENSE,sha256=2-M3fBUS3FmrSIrqd3cZDmxXxojWVJtZY-SHSRE6RxM,1098
+ccfx-0.4.0.dist-info/METADATA,sha256=3llLy7xyGpG1tpvmB9LZc4csbq-R41UoGMlR95ajT-s,5482
+ccfx-0.4.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ccfx-0.4.0.dist-info/top_level.txt,sha256=_cSvSA1WX2K8TgoV3iBJUdUZZqMKJbOPLNnKLYSLHaw,5
+ccfx-0.4.0.dist-info/RECORD,,

{ccfx-0.2.0.dist-info → ccfx-0.4.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

ccfx-0.2.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-ccfx/__init__.py,sha256=VmBeF3oj6JTJ_793d4i8PvhyF8_FxaxA1L_FmHWqitc,142
-ccfx/ccfx.py,sha256=lB6hTCywvbMpHH6lMgw0UJuoiU-7vA22yvttvKFG5Gc,28450
-ccfx/excel.py,sha256=cQ4TQW49XqbMB3sSS0IOhO3-WArIolEBIrvOvhFyPtI,4757
-ccfx/mssqlConnection.py,sha256=TwyZXhHHI7zy6BSfH1pszuHVJ5cmndRC5dVxvEtSTks,7904
-ccfx/sqliteConnection.py,sha256=BsS3jzHSevXLDmtPIFPVuzKqB981su0VpYadTk4xFEQ,11231
-ccfx/word.py,sha256=AGa64jX5Zl5qotZh5L0QmrsjTnktIBhmj_ByRKZ88vw,3061
-ccfx-0.2.0.dist-info/LICENSE,sha256=2-M3fBUS3FmrSIrqd3cZDmxXxojWVJtZY-SHSRE6RxM,1098
-ccfx-0.2.0.dist-info/METADATA,sha256=2GD2kIkcVUo6FG4Th3IEAucmYmhZ4DvdbtGPTI328-c,5419
-ccfx-0.2.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-ccfx-0.2.0.dist-info/top_level.txt,sha256=_cSvSA1WX2K8TgoV3iBJUdUZZqMKJbOPLNnKLYSLHaw,5
-ccfx-0.2.0.dist-info/RECORD,,