ccfx 1.0.0__tar.gz → 1.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ccfx-1.0.0/ccfx.egg-info → ccfx-1.0.3}/PKG-INFO +1 -1
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx/ccfx.py +281 -58
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx/excel.py +2 -2
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx/mssqlConnection.py +36 -5
- {ccfx-1.0.0 → ccfx-1.0.3/ccfx.egg-info}/PKG-INFO +1 -1
- {ccfx-1.0.0 → ccfx-1.0.3}/pyproject.toml +1 -1
- {ccfx-1.0.0 → ccfx-1.0.3}/LICENSE +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/MANIFEST.in +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/README.md +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx/__init__.py +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx/sqliteConnection.py +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx/word.py +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx.egg-info/SOURCES.txt +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx.egg-info/dependency_links.txt +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx.egg-info/requires.txt +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/ccfx.egg-info/top_level.txt +0 -0
- {ccfx-1.0.0 → ccfx-1.0.3}/setup.cfg +0 -0
@@ -23,6 +23,7 @@ import pickle
|
|
23
23
|
import time
|
24
24
|
from shapely.geometry import box, Point
|
25
25
|
import geopandas, pandas
|
26
|
+
from collections import defaultdict
|
26
27
|
import py7zr
|
27
28
|
import subprocess
|
28
29
|
import multiprocessing
|
@@ -34,9 +35,10 @@ import requests
|
|
34
35
|
from tqdm import tqdm
|
35
36
|
import yt_dlp
|
36
37
|
from typing import Optional
|
38
|
+
from datetime import datetime, timedelta
|
37
39
|
|
38
40
|
# functions
|
39
|
-
def listFiles(path: str, ext: str = None) -> list:
|
41
|
+
def listFiles(path: str, ext: Optional[str] = None) -> list:
|
40
42
|
'''
|
41
43
|
List all files in a directory with a specific extension
|
42
44
|
path: directory
|
@@ -132,7 +134,7 @@ def guessMimeType(imagePath):
|
|
132
134
|
return 'image/png'
|
133
135
|
|
134
136
|
|
135
|
-
def downloadYoutubeVideo(url: str, dstDir: str, audioOnly: bool = False, cookiesFile: str = None, dstFileName: Optional[str] = None ) -> str:
|
137
|
+
def downloadYoutubeVideo(url: str, dstDir: str, audioOnly: bool = False, cookiesFile: Optional[str] = None, dstFileName: Optional[str] = None ) -> str:
|
136
138
|
"""
|
137
139
|
Download from YouTube via yt-dlp.
|
138
140
|
|
@@ -236,6 +238,105 @@ def parseYoutubeChannelVideos(channelUrl: str, maxItems: Optional[int] = None) -
|
|
236
238
|
return [f"https://www.youtube.com/watch?v={e['id']}" for e in entries if e.get("id")]
|
237
239
|
|
238
240
|
|
241
|
+
def runSWATPlus(txtinoutDir: str, finalDir: str, executablePath: str = "swatplus", v: bool = True):
|
242
|
+
os.chdir(txtinoutDir)
|
243
|
+
|
244
|
+
if not v:
|
245
|
+
# Run the SWAT+ but ignore output and errors
|
246
|
+
subprocess.run([executablePath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
247
|
+
else:
|
248
|
+
|
249
|
+
yrs_line = readFrom('time.sim')[2].strip().split()
|
250
|
+
|
251
|
+
yr_from = int(yrs_line[1])
|
252
|
+
yr_to = int(yrs_line[3])
|
253
|
+
|
254
|
+
delta = datetime(yr_to, 12, 31) - datetime(yr_from, 1, 1)
|
255
|
+
|
256
|
+
CREATE_NO_WINDOW = 0x08000000
|
257
|
+
|
258
|
+
if platform.system() == "Windows":
|
259
|
+
process = subprocess.Popen(executablePath, stdout=subprocess.PIPE, creationflags=CREATE_NO_WINDOW )
|
260
|
+
else:
|
261
|
+
process = subprocess.Popen(executablePath, stdout=subprocess.PIPE)
|
262
|
+
|
263
|
+
current = 0
|
264
|
+
number_of_days = delta.days + 1
|
265
|
+
|
266
|
+
day_cycle = []
|
267
|
+
previous_time = None
|
268
|
+
|
269
|
+
counter = 0
|
270
|
+
while True:
|
271
|
+
line = process.stdout.readline()
|
272
|
+
line_parts = str(line).strip().split()
|
273
|
+
if not "Simulation" in line_parts: pass
|
274
|
+
elif 'Simulation' in line_parts:
|
275
|
+
ref_index = str(line).strip().split().index("Simulation")
|
276
|
+
year = line_parts[ref_index + 3]
|
277
|
+
month = line_parts[ref_index + 1]
|
278
|
+
day = line_parts[ref_index + 2]
|
279
|
+
|
280
|
+
|
281
|
+
month = f"0{month}" if int(month) < 10 else month
|
282
|
+
day = f"0{day}" if int(day) < 10 else day
|
283
|
+
|
284
|
+
current += 1
|
285
|
+
|
286
|
+
if not previous_time is None:
|
287
|
+
day_cycle.append(datetime.now() - previous_time)
|
288
|
+
|
289
|
+
if len(day_cycle) > 40:
|
290
|
+
if len(day_cycle) > (7 * 365.25):
|
291
|
+
del day_cycle[0]
|
292
|
+
|
293
|
+
av_cycle_time = sum(day_cycle, timedelta()) / len(day_cycle)
|
294
|
+
eta = av_cycle_time * (number_of_days - current)
|
295
|
+
|
296
|
+
eta_str = f" ETA - {formatTimedelta(eta)}:"
|
297
|
+
|
298
|
+
|
299
|
+
else:
|
300
|
+
eta_str = ''
|
301
|
+
|
302
|
+
showProgress(current, number_of_days, barLength=20, message= f' >> current date: {day}/{month}/{year} - f{yr_to} {eta_str}')
|
303
|
+
|
304
|
+
previous_time = datetime.now()
|
305
|
+
elif "ntdll.dll" in line_parts:
|
306
|
+
print("\n! there was an error running SWAT+\n")
|
307
|
+
if counter < 10:
|
308
|
+
counter += 1
|
309
|
+
continue
|
310
|
+
|
311
|
+
if len(line_parts) < 2: break
|
312
|
+
|
313
|
+
showProgress(current, number_of_days, string_after= f' ')
|
314
|
+
print("\n")
|
315
|
+
|
316
|
+
os.chdir(finalDir)
|
317
|
+
|
318
|
+
|
319
|
+
def formatTimedelta(delta: timedelta) -> str:
|
320
|
+
"""Formats a timedelta duration to [N days] %H:%M:%S format"""
|
321
|
+
seconds = int(delta.total_seconds())
|
322
|
+
|
323
|
+
secs_in_a_day = 86400
|
324
|
+
secs_in_a_hour = 3600
|
325
|
+
secs_in_a_min = 60
|
326
|
+
|
327
|
+
days, seconds = divmod(seconds, secs_in_a_day)
|
328
|
+
hours, seconds = divmod(seconds, secs_in_a_hour)
|
329
|
+
minutes, seconds = divmod(seconds, secs_in_a_min)
|
330
|
+
|
331
|
+
time_fmt = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
|
332
|
+
|
333
|
+
if days > 0:
|
334
|
+
suffix = "s" if days > 1 else ""
|
335
|
+
return f"{days} day{suffix} {time_fmt}"
|
336
|
+
else:
|
337
|
+
return f"{time_fmt}"
|
338
|
+
|
339
|
+
|
239
340
|
def setMp3Metadata(fn, metadata, imagePath=None):
|
240
341
|
'''
|
241
342
|
This function takes a path to an mp3 and a metadata dictionary,
|
@@ -332,7 +433,7 @@ def deleteFile(filePath:str, v:bool = False) -> bool:
|
|
332
433
|
return deleted
|
333
434
|
|
334
435
|
|
335
|
-
def alert(message:str, server:str = "http://ntfy.sh", topic:str = "pythonAlerts", attachment:str = None, messageTitle:str = "info", priority:int = None, tags:list = [], printIt:bool = True, v:bool = False) -> bool:
|
436
|
+
def alert(message:str, server:str = "http://ntfy.sh", topic:str = "pythonAlerts", attachment:Optional[str] = None, messageTitle:str = "info", priority:int = None, tags:list = [], printIt:bool = True, v:bool = False) -> bool:
|
336
437
|
'''
|
337
438
|
This sends an alert to a given server in case you want to be notified of something
|
338
439
|
message : the message to send
|
@@ -357,7 +458,6 @@ def alert(message:str, server:str = "http://ntfy.sh", topic:str = "pythonAlerts"
|
|
357
458
|
if not attachment is None:
|
358
459
|
header_data["Filename"] = getFileBaseName(attachment)
|
359
460
|
requests.put( f"{server}/{topic}", data=open(attachment, 'rb'), headers=header_data )
|
360
|
-
return True
|
361
461
|
try: requests.post(f"{server}/{topic}",data=message, headers=header_data )
|
362
462
|
except: return False
|
363
463
|
except: return False
|
@@ -1037,61 +1137,81 @@ def ignoreWarnings(ignore:bool = True, v:bool = False) -> None:
|
|
1037
1137
|
return None
|
1038
1138
|
|
1039
1139
|
|
1040
|
-
def createGrid(
|
1041
|
-
|
1042
|
-
This function creates a grid of polygons based on a shapefile
|
1043
|
-
shapefile_path: path to the shapefile
|
1044
|
-
resolution: resolution of the grid
|
1045
|
-
useDegree: use degree (default is True)
|
1046
|
-
|
1047
|
-
return: xx, yy, polygons, within_mask, gdf.crs, minx, miny
|
1140
|
+
def createGrid(topLeft: list = None, bottomRight: list = None, resolution: float = None,
|
1141
|
+
inputShape: str = None, crs: str = "EPSG:4326", saveVector: str = None) -> geopandas.GeoDataFrame:
|
1048
1142
|
'''
|
1049
|
-
|
1050
|
-
gdf = geopandas.read_file(shapefile_path)
|
1051
|
-
|
1052
|
-
if useDegree:
|
1053
|
-
gdf = gdf.to_crs(epsg=4326)
|
1143
|
+
This function creates a grid of polygons based on either a shapefile or corner coordinates
|
1054
1144
|
|
1055
|
-
|
1056
|
-
|
1145
|
+
Parameters:
|
1146
|
+
topLeft: list [lon, lat] - top left corner coordinates
|
1147
|
+
bottomRight: list [lon, lat] - bottom right corner coordinates
|
1148
|
+
resolution: float - resolution of the grid
|
1149
|
+
inputShape: str - path to the shapefile (optional, if provided bounds will be taken from here)
|
1150
|
+
crs: str - coordinate reference system (default is "EPSG:4326")
|
1151
|
+
saveVector: str - path to save the generated grid (optional)
|
1152
|
+
|
1153
|
+
Returns:
|
1154
|
+
geopandas.GeoDataFrame - the generated grid
|
1155
|
+
'''
|
1156
|
+
# Input validation
|
1157
|
+
if inputShape is None and (topLeft is None or bottomRight is None or resolution is None):
|
1158
|
+
raise ValueError("Either provide inputShape OR provide topLeft, bottomRight, and resolution")
|
1159
|
+
|
1160
|
+
if inputShape is not None and resolution is None:
|
1161
|
+
raise ValueError("Resolution must be provided")
|
1162
|
+
|
1163
|
+
# Get bounds from shapefile or coordinates
|
1164
|
+
if inputShape is not None:
|
1165
|
+
# Read the shapefile and get bounds
|
1166
|
+
gdf = geopandas.read_file(inputShape)
|
1167
|
+
gdf = gdf.to_crs(crs)
|
1168
|
+
minx, miny, maxx, maxy = gdf.total_bounds
|
1169
|
+
reference_geometry = gdf.unary_union
|
1170
|
+
else:
|
1171
|
+
# Use provided corner coordinates [lon, lat]
|
1172
|
+
# Extract coordinates and determine actual bounds
|
1173
|
+
lon1, lat1 = topLeft[0], topLeft[1]
|
1174
|
+
lon2, lat2 = bottomRight[0], bottomRight[1]
|
1175
|
+
|
1176
|
+
# Determine actual min/max values
|
1177
|
+
minx = min(lon1, lon2)
|
1178
|
+
maxx = max(lon1, lon2)
|
1179
|
+
miny = min(lat1, lat2)
|
1180
|
+
maxy = max(lat1, lat2)
|
1181
|
+
reference_geometry = None
|
1057
1182
|
|
1058
1183
|
# Create a grid based on the bounds and resolution
|
1059
1184
|
x = numpy.arange(minx, maxx, resolution)
|
1060
1185
|
y = numpy.arange(miny, maxy, resolution)
|
1061
|
-
xx, yy = numpy.meshgrid(x, y)
|
1062
|
-
|
1063
|
-
# Create polygons for each grid cell, arranged in 2D array
|
1064
|
-
grid_shape = xx.shape
|
1065
|
-
polygons = numpy.empty(grid_shape, dtype=object)
|
1066
|
-
for i in range(grid_shape[0]):
|
1067
|
-
for j in range(grid_shape[1]):
|
1068
|
-
x0, y0 = xx[i, j], yy[i, j]
|
1069
|
-
x1, y1 = x0 + resolution, y0 + resolution
|
1070
|
-
polygons[i, j] = box(x0, y0, x1, y1)
|
1071
1186
|
|
1072
|
-
#
|
1073
|
-
|
1187
|
+
# Create polygons for each grid cell
|
1188
|
+
polygons = []
|
1189
|
+
for i in range(len(y)):
|
1190
|
+
for j in range(len(x)):
|
1191
|
+
x0, y0 = x[j], y[i]
|
1192
|
+
x1, y1 = x0 + resolution, y0 + resolution
|
1193
|
+
# Ensure we don't exceed the bounds
|
1194
|
+
x1 = min(x1, maxx)
|
1195
|
+
y1 = min(y1, maxy)
|
1196
|
+
polygons.append(box(x0, y0, x1, y1))
|
1074
1197
|
|
1075
1198
|
# Create a GeoDataFrame from the grid
|
1076
|
-
grid_gdf = geopandas.GeoDataFrame({'geometry':
|
1077
|
-
|
1078
|
-
minx, miny, maxx, maxy = grid_gdf.total_bounds
|
1079
|
-
print(" minx:", minx, "miny:", miny, "maxx:", maxx, "maxy:", maxy)
|
1080
|
-
|
1081
|
-
minx, miny, maxx, maxy = getVectorBounds(grid_gdf)
|
1199
|
+
grid_gdf = geopandas.GeoDataFrame({'geometry': polygons}, crs=crs)
|
1200
|
+
|
1082
1201
|
# Add a column to indicate if the cell intersects with the original shapefile
|
1083
|
-
|
1202
|
+
if reference_geometry is not None:
|
1203
|
+
grid_gdf['within'] = grid_gdf.intersects(reference_geometry)
|
1204
|
+
else:
|
1205
|
+
# For coordinate-based grids, set all cells as within
|
1206
|
+
grid_gdf['within'] = True
|
1084
1207
|
|
1085
|
-
#
|
1086
|
-
|
1208
|
+
# Save the grid if path is provided
|
1209
|
+
if saveVector is not None:
|
1210
|
+
grid_gdf.to_file(saveVector, driver="GPKG")
|
1211
|
+
print(f"Grid saved to {saveVector}")
|
1087
1212
|
|
1088
|
-
|
1089
|
-
reprojectedGrid = grid_gdf.to_crs(epsg=4326)
|
1213
|
+
return grid_gdf
|
1090
1214
|
|
1091
|
-
grid_gdf.to_file("generatedGrid4326.gpkg", driver="GPKG")
|
1092
|
-
reprojectedGrid.to_file("generatedGrid.gpkg", driver="GPKG")
|
1093
|
-
|
1094
|
-
return xx, yy, polygons, within_mask, gdf.crs, minx, miny
|
1095
1215
|
|
1096
1216
|
def setHomeDir(path:str) -> str:
|
1097
1217
|
'''
|
@@ -1148,7 +1268,7 @@ def netcdfVariableDimensions(ncFile: str, variable: str) -> dict:
|
|
1148
1268
|
|
1149
1269
|
return bands_info
|
1150
1270
|
|
1151
|
-
def netcdfExportTif(ncFile: str, variable: str, outputFile: str = None, band: int = None, v:bool = True) -> gdal.Dataset:
|
1271
|
+
def netcdfExportTif(ncFile: str, variable: str, outputFile: Optional[str] = None, band: int = None, v:bool = True) -> gdal.Dataset:
|
1152
1272
|
'''
|
1153
1273
|
Export a variable from a NetCDF file to a GeoTiff file
|
1154
1274
|
ncFile: NetCDF file
|
@@ -1391,7 +1511,7 @@ def showProgress(count: int, end: int, message: str, barLength: int = 100) -> No
|
|
1391
1511
|
message: message to display
|
1392
1512
|
barLength: length of the progress bar
|
1393
1513
|
'''
|
1394
|
-
percent =
|
1514
|
+
percent = float(count / end * 100)
|
1395
1515
|
percentStr = f'{percent:03.1f}'
|
1396
1516
|
filled = int(barLength * count / end)
|
1397
1517
|
bar = '█' * filled + '░' * (barLength - filled)
|
@@ -1469,7 +1589,7 @@ def createPointGeometry(coords: list, proj: str = "EPSG:4326") -> geopandas.GeoD
|
|
1469
1589
|
gdf.reset_index(inplace=True)
|
1470
1590
|
return gdf
|
1471
1591
|
|
1472
|
-
def calculateTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> dict:
|
1592
|
+
def calculateTimeseriesStats(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> dict:
|
1473
1593
|
'''
|
1474
1594
|
Calculate statistics for a timeseries
|
1475
1595
|
|
@@ -1613,7 +1733,7 @@ def calculateTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulat
|
|
1613
1733
|
}
|
1614
1734
|
|
1615
1735
|
|
1616
|
-
def getNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1736
|
+
def getNSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1617
1737
|
'''
|
1618
1738
|
this function is a wrapper for calculateTimeseriesStats specifically to return the NSE
|
1619
1739
|
|
@@ -1634,7 +1754,7 @@ def getNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1634
1754
|
|
1635
1755
|
return stats['NSE']
|
1636
1756
|
|
1637
|
-
def getKGE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1757
|
+
def getKGE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1638
1758
|
'''
|
1639
1759
|
this function is a wrapper for calculateTimeseriesStats specifically to return the KGE
|
1640
1760
|
|
@@ -1655,7 +1775,7 @@ def getKGE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1655
1775
|
|
1656
1776
|
return stats['KGE']
|
1657
1777
|
|
1658
|
-
def getPBIAS(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1778
|
+
def getPBIAS(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1659
1779
|
'''
|
1660
1780
|
this function is a wrapper for calculateTimeseriesStats specifically to return the PBIAS
|
1661
1781
|
|
@@ -1677,7 +1797,7 @@ def getPBIAS(data:pandas.DataFrame, observed:str = None, simulated:str = None, r
|
|
1677
1797
|
return stats['PBIAS']
|
1678
1798
|
|
1679
1799
|
|
1680
|
-
def getLNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1800
|
+
def getLNSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1681
1801
|
'''
|
1682
1802
|
this function is a wrapper for calculateTimeseriesStats specifically to return the LNSE
|
1683
1803
|
|
@@ -1698,7 +1818,7 @@ def getLNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, re
|
|
1698
1818
|
|
1699
1819
|
return stats['LNSE']
|
1700
1820
|
|
1701
|
-
def getR2(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1821
|
+
def getR2(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1702
1822
|
'''
|
1703
1823
|
this function is a wrapper for calculateTimeseriesStats specifically to return the R2
|
1704
1824
|
|
@@ -1719,7 +1839,7 @@ def getR2(data:pandas.DataFrame, observed:str = None, simulated:str = None, resa
|
|
1719
1839
|
|
1720
1840
|
return stats['R2']
|
1721
1841
|
|
1722
|
-
def getRMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1842
|
+
def getRMSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1723
1843
|
'''
|
1724
1844
|
this function is a wrapper for calculateTimeseriesStats specifically to return the RMSE
|
1725
1845
|
|
@@ -1740,7 +1860,7 @@ def getRMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, re
|
|
1740
1860
|
|
1741
1861
|
return stats['RMSE']
|
1742
1862
|
|
1743
|
-
def getMAE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1863
|
+
def getMAE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1744
1864
|
'''
|
1745
1865
|
this function is a wrapper for calculateTimeseriesStats specifically to return the MAE
|
1746
1866
|
|
@@ -1761,7 +1881,7 @@ def getMAE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1761
1881
|
|
1762
1882
|
return stats['MAE']
|
1763
1883
|
|
1764
|
-
def getMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1884
|
+
def getMSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1765
1885
|
'''
|
1766
1886
|
this function is a wrapper for calculateTimeseriesStats specifically to return the MSE
|
1767
1887
|
|
@@ -1782,7 +1902,7 @@ def getMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1782
1902
|
|
1783
1903
|
return stats['MSE']
|
1784
1904
|
|
1785
|
-
def getTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> dict:
|
1905
|
+
def getTimeseriesStats(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> dict:
|
1786
1906
|
'''
|
1787
1907
|
this function is a wrapper for calculateTimeseriesStats specifically to return all stats
|
1788
1908
|
|
@@ -1803,4 +1923,107 @@ def getTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str
|
|
1803
1923
|
|
1804
1924
|
return stats
|
1805
1925
|
|
1926
|
+
def readSWATPlusOutputs(filePath: str, column: Optional[str] = None, unit: Optional[int] = None, gis_id: Optional[int] = None, name: Optional[str] = None):
|
1927
|
+
'''
|
1928
|
+
Read SWAT+ output files and return a pandas DataFrame with proper date handling
|
1929
|
+
and optional filtering capabilities.
|
1930
|
+
|
1931
|
+
Parameters:
|
1932
|
+
-----------
|
1933
|
+
filePath: str
|
1934
|
+
Path to the SWAT+ output file
|
1935
|
+
column: str, optional
|
1936
|
+
Name of the column to extract. If not specified, returns all columns.
|
1937
|
+
If specified, returns first match, or specify multiple columns as comma-separated string
|
1938
|
+
unit: int, optional
|
1939
|
+
Filter by unit number. If not specified, returns all units
|
1940
|
+
gis_id: int, optional
|
1941
|
+
Filter by gis_id. If not specified, returns all gis_ids
|
1942
|
+
name: str, optional
|
1943
|
+
Filter by name. If not specified, returns all names
|
1944
|
+
|
1945
|
+
Returns:
|
1946
|
+
--------
|
1947
|
+
pandas.DataFrame or None
|
1948
|
+
DataFrame with date column and requested data, filtered as specified
|
1949
|
+
'''
|
1950
|
+
|
1951
|
+
if not exists(filePath):
|
1952
|
+
print('! SWAT+ result file does not exist')
|
1953
|
+
return None
|
1954
|
+
|
1955
|
+
# Read the header line (line 2, index 1)
|
1956
|
+
with open(filePath, 'r') as f:
|
1957
|
+
lines = f.readlines()
|
1958
|
+
|
1959
|
+
header_line = lines[1].strip()
|
1960
|
+
headers = header_line.split()
|
1961
|
+
|
1962
|
+
# Handle duplicate column names
|
1963
|
+
column_counts = defaultdict(int)
|
1964
|
+
modified_header = []
|
1965
|
+
for col_name in headers:
|
1966
|
+
column_counts[col_name] += 1
|
1967
|
+
if column_counts[col_name] > 1:
|
1968
|
+
modified_header.append(f"{col_name}_{column_counts[col_name]}")
|
1969
|
+
else:
|
1970
|
+
modified_header.append(col_name)
|
1971
|
+
|
1972
|
+
# Add extra columns to handle potential mismatches
|
1973
|
+
modified_header = modified_header + ['extra1', 'extra2']
|
1974
|
+
|
1975
|
+
try:
|
1976
|
+
df = pandas.read_csv(filePath, delim_whitespace=True, skiprows=3, names=modified_header, index_col=False)
|
1977
|
+
except:
|
1978
|
+
sys.stdout.write(f'\r! could not read {filePath} using pandas, check the number of columns\n')
|
1979
|
+
sys.stdout.flush()
|
1980
|
+
return None
|
1981
|
+
|
1982
|
+
# Remove extra columns
|
1983
|
+
df = df.drop(columns=['extra1', 'extra2'], errors='ignore')
|
1984
|
+
|
1985
|
+
# Convert all columns to numeric except 'name' (which is string)
|
1986
|
+
for col in df.columns:
|
1987
|
+
if col != 'name':
|
1988
|
+
df[col] = pandas.to_numeric(df[col], errors='coerce')
|
1989
|
+
|
1990
|
+
# Create date column from yr, mon, day
|
1991
|
+
try:
|
1992
|
+
df['date'] = pandas.to_datetime(pandas.DataFrame({'year': df.yr, 'month': df.mon, 'day': df.day}))
|
1993
|
+
except KeyError:
|
1994
|
+
# If some date columns are missing, create a simple index-based date
|
1995
|
+
df['date'] = pandas.date_range(start='2000-01-01', periods=len(df), freq='D')
|
1996
|
+
except:
|
1997
|
+
# If date creation fails for any other reason, use index-based date
|
1998
|
+
df['date'] = pandas.date_range(start='2000-01-01', periods=len(df), freq='D')
|
1999
|
+
|
2000
|
+
# Filter by unit if specified
|
2001
|
+
if unit is not None and 'unit' in df.columns:
|
2002
|
+
df = df[df['unit'] == unit]
|
2003
|
+
|
2004
|
+
# Filter by gis_id if specified
|
2005
|
+
if gis_id is not None and 'gis_id' in df.columns:
|
2006
|
+
df = df[df['gis_id'] == gis_id]
|
2007
|
+
|
2008
|
+
# Filter by name if specified
|
2009
|
+
if name is not None and 'name' in df.columns:
|
2010
|
+
df = df[df['name'] == name]
|
2011
|
+
|
2012
|
+
# Handle column selection
|
2013
|
+
if column is not None and column != "*":
|
2014
|
+
# Parse comma-separated columns
|
2015
|
+
requested_cols = [col.strip() for col in column.split(',')]
|
2016
|
+
|
2017
|
+
# Always include date column
|
2018
|
+
selected_cols = ['date']
|
2019
|
+
|
2020
|
+
# Add requested columns if they exist
|
2021
|
+
for req_col in requested_cols:
|
2022
|
+
if req_col in df.columns:
|
2023
|
+
selected_cols.append(req_col)
|
2024
|
+
|
2025
|
+
df = df[selected_cols]
|
2026
|
+
|
2027
|
+
return df
|
2028
|
+
|
1806
2029
|
ignoreWarnings()
|
@@ -29,7 +29,7 @@ class excel:
|
|
29
29
|
self.date_format = None
|
30
30
|
|
31
31
|
def create(self):
|
32
|
-
self.
|
32
|
+
self.createPath(os.path.dirname(self.path))
|
33
33
|
self.book = xlsxwriter.Workbook(self.path)
|
34
34
|
|
35
35
|
def addSheet(self, sheet_name):
|
@@ -42,7 +42,7 @@ class excel:
|
|
42
42
|
|
43
43
|
def writeDate(self, sheet_name, row, column, datetime_obj):
|
44
44
|
if self.date_format is None:
|
45
|
-
self.
|
45
|
+
self.setDateFormat()
|
46
46
|
|
47
47
|
self.sheet_names[sheet_name].write_datetime(
|
48
48
|
row, column, datetime_obj, self.date_format)
|
@@ -21,7 +21,7 @@ import geopandas
|
|
21
21
|
|
22
22
|
|
23
23
|
# classes
|
24
|
-
class
|
24
|
+
class mssqlConnection:
|
25
25
|
def __init__(self, server, username, password, driver, trust_server_ssl = True) -> None:
|
26
26
|
self.server = server
|
27
27
|
self.username = username
|
@@ -85,20 +85,49 @@ class mssql_connection:
|
|
85
85
|
self.cursor = self.connection.cursor()
|
86
86
|
self.cursor.execute(query)
|
87
87
|
tables = [row[0] for row in self.cursor.fetchall()]
|
88
|
-
print("> list of tables in the active database:")
|
89
|
-
for table in tables:
|
90
|
-
|
88
|
+
# print("> list of tables in the active database:")
|
89
|
+
# for table in tables:
|
90
|
+
# print(f"\t- {table}")
|
91
91
|
except pyodbc.Error as e:
|
92
92
|
print("Error occurred while fetching the list of tables:")
|
93
93
|
print(e)
|
94
94
|
|
95
95
|
return tables
|
96
96
|
|
97
|
+
def listColumns(self, tableName: str, dbName: str | None = None) -> list[str]:
|
98
|
+
if dbName:
|
99
|
+
self.connect_db(dbName)
|
100
|
+
|
101
|
+
schema, tbl = ('dbo', tableName) if '.' not in tableName else tableName.split('.', 1)
|
102
|
+
|
103
|
+
sql = """
|
104
|
+
SELECT column_name
|
105
|
+
FROM information_schema.columns
|
106
|
+
WHERE table_schema = ? AND table_name = ?
|
107
|
+
ORDER BY ordinal_position
|
108
|
+
"""
|
109
|
+
|
110
|
+
try:
|
111
|
+
with self.connection.cursor() as cur:
|
112
|
+
cur.execute(sql, (schema, tbl))
|
113
|
+
return [row[0] for row in cur.fetchall()]
|
114
|
+
except pyodbc.Error as e:
|
115
|
+
print(f"Could not list columns for {tableName}: {e}")
|
116
|
+
return []
|
117
|
+
|
97
118
|
|
98
119
|
def readTable(self, table_name:str, db_name:str = None, columns:list = None, geom_col:str = None, v = True):
|
99
120
|
if db_name is not None:
|
100
121
|
self.connect_db(db_name)
|
101
122
|
|
123
|
+
# ensure geometry column is not in columns if specified
|
124
|
+
if geom_col is not None:
|
125
|
+
if columns is None:
|
126
|
+
columns = self.listColumns(table_name, db_name)
|
127
|
+
|
128
|
+
columns = [col for col in columns if col != geom_col]
|
129
|
+
|
130
|
+
|
102
131
|
if columns is not None and geom_col is not None:
|
103
132
|
columns.append(f"{geom_col}.STAsText() as {geom_col}_wkt")
|
104
133
|
query = f"SELECT {','.join(columns)} FROM {table_name}"
|
@@ -221,7 +250,9 @@ class mssql_connection:
|
|
221
250
|
self.connection.close()
|
222
251
|
self.connection = None
|
223
252
|
self.cursor = None
|
224
|
-
|
253
|
+
if v: print("> connection closed...")
|
254
|
+
else:
|
255
|
+
if v: print("> no connection to close...")
|
225
256
|
|
226
257
|
def disconnect(self, v = True):
|
227
258
|
self.close(v = v)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|