ccfx 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ccfx/ccfx.py +282 -58
- ccfx/excel.py +2 -2
- ccfx/mssqlConnection.py +36 -5
- {ccfx-1.0.0.dist-info → ccfx-1.0.2.dist-info}/METADATA +1 -1
- ccfx-1.0.2.dist-info/RECORD +11 -0
- ccfx-1.0.0.dist-info/RECORD +0 -11
- {ccfx-1.0.0.dist-info → ccfx-1.0.2.dist-info}/WHEEL +0 -0
- {ccfx-1.0.0.dist-info → ccfx-1.0.2.dist-info}/licenses/LICENSE +0 -0
- {ccfx-1.0.0.dist-info → ccfx-1.0.2.dist-info}/top_level.txt +0 -0
ccfx/ccfx.py
CHANGED
@@ -23,6 +23,7 @@ import pickle
|
|
23
23
|
import time
|
24
24
|
from shapely.geometry import box, Point
|
25
25
|
import geopandas, pandas
|
26
|
+
from collections import defaultdict
|
26
27
|
import py7zr
|
27
28
|
import subprocess
|
28
29
|
import multiprocessing
|
@@ -34,9 +35,10 @@ import requests
|
|
34
35
|
from tqdm import tqdm
|
35
36
|
import yt_dlp
|
36
37
|
from typing import Optional
|
38
|
+
from datetime import datetime, timedelta
|
37
39
|
|
38
40
|
# functions
|
39
|
-
def listFiles(path: str, ext: str = None) -> list:
|
41
|
+
def listFiles(path: str, ext: Optional[str] = None) -> list:
|
40
42
|
'''
|
41
43
|
List all files in a directory with a specific extension
|
42
44
|
path: directory
|
@@ -132,7 +134,7 @@ def guessMimeType(imagePath):
|
|
132
134
|
return 'image/png'
|
133
135
|
|
134
136
|
|
135
|
-
def downloadYoutubeVideo(url: str, dstDir: str, audioOnly: bool = False, cookiesFile: str = None, dstFileName: Optional[str] = None ) -> str:
|
137
|
+
def downloadYoutubeVideo(url: str, dstDir: str, audioOnly: bool = False, cookiesFile: Optional[str] = None, dstFileName: Optional[str] = None ) -> str:
|
136
138
|
"""
|
137
139
|
Download from YouTube via yt-dlp.
|
138
140
|
|
@@ -236,6 +238,106 @@ def parseYoutubeChannelVideos(channelUrl: str, maxItems: Optional[int] = None) -
|
|
236
238
|
return [f"https://www.youtube.com/watch?v={e['id']}" for e in entries if e.get("id")]
|
237
239
|
|
238
240
|
|
241
|
+
|
242
|
+
def runSWATPlus(txtinoutDir: str, finalDir: str, executablePath: str = "swatplus", v: bool = True):
|
243
|
+
os.chdir(txtinoutDir)
|
244
|
+
|
245
|
+
if not v:
|
246
|
+
# Run the SWAT+ but ignore output and errors
|
247
|
+
subprocess.run([executablePath], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
248
|
+
else:
|
249
|
+
|
250
|
+
yrs_line = readFrom('time.sim')[2].strip().split()
|
251
|
+
|
252
|
+
yr_from = int(yrs_line[1])
|
253
|
+
yr_to = int(yrs_line[3])
|
254
|
+
|
255
|
+
delta = datetime(yr_to, 12, 31) - datetime(yr_from, 1, 1)
|
256
|
+
|
257
|
+
CREATE_NO_WINDOW = 0x08000000
|
258
|
+
|
259
|
+
if platform.system() == "Windows":
|
260
|
+
process = subprocess.Popen(executablePath, stdout=subprocess.PIPE, creationflags=CREATE_NO_WINDOW )
|
261
|
+
else:
|
262
|
+
process = subprocess.Popen(executablePath, stdout=subprocess.PIPE)
|
263
|
+
|
264
|
+
current = 0
|
265
|
+
number_of_days = delta.days + 1
|
266
|
+
|
267
|
+
day_cycle = []
|
268
|
+
previous_time = None
|
269
|
+
|
270
|
+
while True:
|
271
|
+
line = process.stdout.readline()
|
272
|
+
line_parts = str(line).strip().split()
|
273
|
+
if not "Simulation" in line_parts: pass
|
274
|
+
elif 'Simulation' in line_parts:
|
275
|
+
ref_index = str(line).strip().split().index("Simulation")
|
276
|
+
year = line_parts[ref_index + 3]
|
277
|
+
month = line_parts[ref_index + 1]
|
278
|
+
day = line_parts[ref_index + 2]
|
279
|
+
|
280
|
+
|
281
|
+
month = f"0{month}" if int(month) < 10 else month
|
282
|
+
day = f"0{day}" if int(day) < 10 else day
|
283
|
+
|
284
|
+
current += 1
|
285
|
+
|
286
|
+
if not previous_time is None:
|
287
|
+
day_cycle.append(datetime.now() - previous_time)
|
288
|
+
|
289
|
+
if len(day_cycle) > 40:
|
290
|
+
if len(day_cycle) > (7 * 365.25):
|
291
|
+
del day_cycle[0]
|
292
|
+
|
293
|
+
av_cycle_time = sum(day_cycle, timedelta()) / len(day_cycle)
|
294
|
+
eta = av_cycle_time * (number_of_days - current)
|
295
|
+
|
296
|
+
eta_str = f" ETA - {formatTimedelta(eta)}:"
|
297
|
+
|
298
|
+
|
299
|
+
else:
|
300
|
+
eta_str = ''
|
301
|
+
|
302
|
+
showProgress(current, number_of_days, bar_length=20, message= f' >> current date: {day}/{month}/{year} - f{yr_to} {eta_str}')
|
303
|
+
|
304
|
+
previous_time = datetime.now()
|
305
|
+
elif "ntdll.dll" in line_parts:
|
306
|
+
print("\n! there was an error running SWAT+\n")
|
307
|
+
if counter < 10:
|
308
|
+
counter += 1
|
309
|
+
continue
|
310
|
+
|
311
|
+
if len(line_parts) < 2: break
|
312
|
+
|
313
|
+
showProgress(current, number_of_days, string_after= f' ')
|
314
|
+
print("\n")
|
315
|
+
|
316
|
+
os.chdir(finalDir)
|
317
|
+
|
318
|
+
|
319
|
+
|
320
|
+
def formatTimedelta(delta: timedelta) -> str:
|
321
|
+
"""Formats a timedelta duration to [N days] %H:%M:%S format"""
|
322
|
+
seconds = int(delta.total_seconds())
|
323
|
+
|
324
|
+
secs_in_a_day = 86400
|
325
|
+
secs_in_a_hour = 3600
|
326
|
+
secs_in_a_min = 60
|
327
|
+
|
328
|
+
days, seconds = divmod(seconds, secs_in_a_day)
|
329
|
+
hours, seconds = divmod(seconds, secs_in_a_hour)
|
330
|
+
minutes, seconds = divmod(seconds, secs_in_a_min)
|
331
|
+
|
332
|
+
time_fmt = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
|
333
|
+
|
334
|
+
if days > 0:
|
335
|
+
suffix = "s" if days > 1 else ""
|
336
|
+
return f"{days} day{suffix} {time_fmt}"
|
337
|
+
else:
|
338
|
+
return f"{time_fmt}"
|
339
|
+
|
340
|
+
|
239
341
|
def setMp3Metadata(fn, metadata, imagePath=None):
|
240
342
|
'''
|
241
343
|
This function takes a path to an mp3 and a metadata dictionary,
|
@@ -332,7 +434,7 @@ def deleteFile(filePath:str, v:bool = False) -> bool:
|
|
332
434
|
return deleted
|
333
435
|
|
334
436
|
|
335
|
-
def alert(message:str, server:str = "http://ntfy.sh", topic:str = "pythonAlerts", attachment:str = None, messageTitle:str = "info", priority:int = None, tags:list = [], printIt:bool = True, v:bool = False) -> bool:
|
437
|
+
def alert(message:str, server:str = "http://ntfy.sh", topic:str = "pythonAlerts", attachment:Optional[str] = None, messageTitle:str = "info", priority:int = None, tags:list = [], printIt:bool = True, v:bool = False) -> bool:
|
336
438
|
'''
|
337
439
|
This sends an alert to a given server in case you want to be notified of something
|
338
440
|
message : the message to send
|
@@ -357,7 +459,6 @@ def alert(message:str, server:str = "http://ntfy.sh", topic:str = "pythonAlerts"
|
|
357
459
|
if not attachment is None:
|
358
460
|
header_data["Filename"] = getFileBaseName(attachment)
|
359
461
|
requests.put( f"{server}/{topic}", data=open(attachment, 'rb'), headers=header_data )
|
360
|
-
return True
|
361
462
|
try: requests.post(f"{server}/{topic}",data=message, headers=header_data )
|
362
463
|
except: return False
|
363
464
|
except: return False
|
@@ -1037,61 +1138,81 @@ def ignoreWarnings(ignore:bool = True, v:bool = False) -> None:
|
|
1037
1138
|
return None
|
1038
1139
|
|
1039
1140
|
|
1040
|
-
def createGrid(
|
1041
|
-
|
1042
|
-
This function creates a grid of polygons based on a shapefile
|
1043
|
-
shapefile_path: path to the shapefile
|
1044
|
-
resolution: resolution of the grid
|
1045
|
-
useDegree: use degree (default is True)
|
1046
|
-
|
1047
|
-
return: xx, yy, polygons, within_mask, gdf.crs, minx, miny
|
1141
|
+
def createGrid(topLeft: list = None, bottomRight: list = None, resolution: float = None,
|
1142
|
+
inputShape: str = None, crs: str = "EPSG:4326", saveVector: str = None) -> geopandas.GeoDataFrame:
|
1048
1143
|
'''
|
1049
|
-
|
1050
|
-
gdf = geopandas.read_file(shapefile_path)
|
1051
|
-
|
1052
|
-
if useDegree:
|
1053
|
-
gdf = gdf.to_crs(epsg=4326)
|
1144
|
+
This function creates a grid of polygons based on either a shapefile or corner coordinates
|
1054
1145
|
|
1055
|
-
|
1056
|
-
|
1146
|
+
Parameters:
|
1147
|
+
topLeft: list [lon, lat] - top left corner coordinates
|
1148
|
+
bottomRight: list [lon, lat] - bottom right corner coordinates
|
1149
|
+
resolution: float - resolution of the grid
|
1150
|
+
inputShape: str - path to the shapefile (optional, if provided bounds will be taken from here)
|
1151
|
+
crs: str - coordinate reference system (default is "EPSG:4326")
|
1152
|
+
saveVector: str - path to save the generated grid (optional)
|
1153
|
+
|
1154
|
+
Returns:
|
1155
|
+
geopandas.GeoDataFrame - the generated grid
|
1156
|
+
'''
|
1157
|
+
# Input validation
|
1158
|
+
if inputShape is None and (topLeft is None or bottomRight is None or resolution is None):
|
1159
|
+
raise ValueError("Either provide inputShape OR provide topLeft, bottomRight, and resolution")
|
1160
|
+
|
1161
|
+
if inputShape is not None and resolution is None:
|
1162
|
+
raise ValueError("Resolution must be provided")
|
1163
|
+
|
1164
|
+
# Get bounds from shapefile or coordinates
|
1165
|
+
if inputShape is not None:
|
1166
|
+
# Read the shapefile and get bounds
|
1167
|
+
gdf = geopandas.read_file(inputShape)
|
1168
|
+
gdf = gdf.to_crs(crs)
|
1169
|
+
minx, miny, maxx, maxy = gdf.total_bounds
|
1170
|
+
reference_geometry = gdf.unary_union
|
1171
|
+
else:
|
1172
|
+
# Use provided corner coordinates [lon, lat]
|
1173
|
+
# Extract coordinates and determine actual bounds
|
1174
|
+
lon1, lat1 = topLeft[0], topLeft[1]
|
1175
|
+
lon2, lat2 = bottomRight[0], bottomRight[1]
|
1176
|
+
|
1177
|
+
# Determine actual min/max values
|
1178
|
+
minx = min(lon1, lon2)
|
1179
|
+
maxx = max(lon1, lon2)
|
1180
|
+
miny = min(lat1, lat2)
|
1181
|
+
maxy = max(lat1, lat2)
|
1182
|
+
reference_geometry = None
|
1057
1183
|
|
1058
1184
|
# Create a grid based on the bounds and resolution
|
1059
1185
|
x = numpy.arange(minx, maxx, resolution)
|
1060
1186
|
y = numpy.arange(miny, maxy, resolution)
|
1061
|
-
xx, yy = numpy.meshgrid(x, y)
|
1062
|
-
|
1063
|
-
# Create polygons for each grid cell, arranged in 2D array
|
1064
|
-
grid_shape = xx.shape
|
1065
|
-
polygons = numpy.empty(grid_shape, dtype=object)
|
1066
|
-
for i in range(grid_shape[0]):
|
1067
|
-
for j in range(grid_shape[1]):
|
1068
|
-
x0, y0 = xx[i, j], yy[i, j]
|
1069
|
-
x1, y1 = x0 + resolution, y0 + resolution
|
1070
|
-
polygons[i, j] = box(x0, y0, x1, y1)
|
1071
1187
|
|
1072
|
-
#
|
1073
|
-
|
1188
|
+
# Create polygons for each grid cell
|
1189
|
+
polygons = []
|
1190
|
+
for i in range(len(y)):
|
1191
|
+
for j in range(len(x)):
|
1192
|
+
x0, y0 = x[j], y[i]
|
1193
|
+
x1, y1 = x0 + resolution, y0 + resolution
|
1194
|
+
# Ensure we don't exceed the bounds
|
1195
|
+
x1 = min(x1, maxx)
|
1196
|
+
y1 = min(y1, maxy)
|
1197
|
+
polygons.append(box(x0, y0, x1, y1))
|
1074
1198
|
|
1075
1199
|
# Create a GeoDataFrame from the grid
|
1076
|
-
grid_gdf = geopandas.GeoDataFrame({'geometry':
|
1077
|
-
|
1078
|
-
minx, miny, maxx, maxy = grid_gdf.total_bounds
|
1079
|
-
print(" minx:", minx, "miny:", miny, "maxx:", maxx, "maxy:", maxy)
|
1080
|
-
|
1081
|
-
minx, miny, maxx, maxy = getVectorBounds(grid_gdf)
|
1200
|
+
grid_gdf = geopandas.GeoDataFrame({'geometry': polygons}, crs=crs)
|
1201
|
+
|
1082
1202
|
# Add a column to indicate if the cell intersects with the original shapefile
|
1083
|
-
|
1203
|
+
if reference_geometry is not None:
|
1204
|
+
grid_gdf['within'] = grid_gdf.intersects(reference_geometry)
|
1205
|
+
else:
|
1206
|
+
# For coordinate-based grids, set all cells as within
|
1207
|
+
grid_gdf['within'] = True
|
1084
1208
|
|
1085
|
-
#
|
1086
|
-
|
1209
|
+
# Save the grid if path is provided
|
1210
|
+
if saveVector is not None:
|
1211
|
+
grid_gdf.to_file(saveVector, driver="GPKG")
|
1212
|
+
print(f"Grid saved to {saveVector}")
|
1087
1213
|
|
1088
|
-
|
1089
|
-
reprojectedGrid = grid_gdf.to_crs(epsg=4326)
|
1214
|
+
return grid_gdf
|
1090
1215
|
|
1091
|
-
grid_gdf.to_file("generatedGrid4326.gpkg", driver="GPKG")
|
1092
|
-
reprojectedGrid.to_file("generatedGrid.gpkg", driver="GPKG")
|
1093
|
-
|
1094
|
-
return xx, yy, polygons, within_mask, gdf.crs, minx, miny
|
1095
1216
|
|
1096
1217
|
def setHomeDir(path:str) -> str:
|
1097
1218
|
'''
|
@@ -1148,7 +1269,7 @@ def netcdfVariableDimensions(ncFile: str, variable: str) -> dict:
|
|
1148
1269
|
|
1149
1270
|
return bands_info
|
1150
1271
|
|
1151
|
-
def netcdfExportTif(ncFile: str, variable: str, outputFile: str = None, band: int = None, v:bool = True) -> gdal.Dataset:
|
1272
|
+
def netcdfExportTif(ncFile: str, variable: str, outputFile: Optional[str] = None, band: int = None, v:bool = True) -> gdal.Dataset:
|
1152
1273
|
'''
|
1153
1274
|
Export a variable from a NetCDF file to a GeoTiff file
|
1154
1275
|
ncFile: NetCDF file
|
@@ -1391,7 +1512,7 @@ def showProgress(count: int, end: int, message: str, barLength: int = 100) -> No
|
|
1391
1512
|
message: message to display
|
1392
1513
|
barLength: length of the progress bar
|
1393
1514
|
'''
|
1394
|
-
percent =
|
1515
|
+
percent = float(count / end * 100)
|
1395
1516
|
percentStr = f'{percent:03.1f}'
|
1396
1517
|
filled = int(barLength * count / end)
|
1397
1518
|
bar = '█' * filled + '░' * (barLength - filled)
|
@@ -1469,7 +1590,7 @@ def createPointGeometry(coords: list, proj: str = "EPSG:4326") -> geopandas.GeoD
|
|
1469
1590
|
gdf.reset_index(inplace=True)
|
1470
1591
|
return gdf
|
1471
1592
|
|
1472
|
-
def calculateTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> dict:
|
1593
|
+
def calculateTimeseriesStats(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> dict:
|
1473
1594
|
'''
|
1474
1595
|
Calculate statistics for a timeseries
|
1475
1596
|
|
@@ -1613,7 +1734,7 @@ def calculateTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulat
|
|
1613
1734
|
}
|
1614
1735
|
|
1615
1736
|
|
1616
|
-
def getNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1737
|
+
def getNSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1617
1738
|
'''
|
1618
1739
|
this function is a wrapper for calculateTimeseriesStats specifically to return the NSE
|
1619
1740
|
|
@@ -1634,7 +1755,7 @@ def getNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1634
1755
|
|
1635
1756
|
return stats['NSE']
|
1636
1757
|
|
1637
|
-
def getKGE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1758
|
+
def getKGE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1638
1759
|
'''
|
1639
1760
|
this function is a wrapper for calculateTimeseriesStats specifically to return the KGE
|
1640
1761
|
|
@@ -1655,7 +1776,7 @@ def getKGE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1655
1776
|
|
1656
1777
|
return stats['KGE']
|
1657
1778
|
|
1658
|
-
def getPBIAS(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1779
|
+
def getPBIAS(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1659
1780
|
'''
|
1660
1781
|
this function is a wrapper for calculateTimeseriesStats specifically to return the PBIAS
|
1661
1782
|
|
@@ -1677,7 +1798,7 @@ def getPBIAS(data:pandas.DataFrame, observed:str = None, simulated:str = None, r
|
|
1677
1798
|
return stats['PBIAS']
|
1678
1799
|
|
1679
1800
|
|
1680
|
-
def getLNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1801
|
+
def getLNSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1681
1802
|
'''
|
1682
1803
|
this function is a wrapper for calculateTimeseriesStats specifically to return the LNSE
|
1683
1804
|
|
@@ -1698,7 +1819,7 @@ def getLNSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, re
|
|
1698
1819
|
|
1699
1820
|
return stats['LNSE']
|
1700
1821
|
|
1701
|
-
def getR2(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1822
|
+
def getR2(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1702
1823
|
'''
|
1703
1824
|
this function is a wrapper for calculateTimeseriesStats specifically to return the R2
|
1704
1825
|
|
@@ -1719,7 +1840,7 @@ def getR2(data:pandas.DataFrame, observed:str = None, simulated:str = None, resa
|
|
1719
1840
|
|
1720
1841
|
return stats['R2']
|
1721
1842
|
|
1722
|
-
def getRMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1843
|
+
def getRMSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1723
1844
|
'''
|
1724
1845
|
this function is a wrapper for calculateTimeseriesStats specifically to return the RMSE
|
1725
1846
|
|
@@ -1740,7 +1861,7 @@ def getRMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, re
|
|
1740
1861
|
|
1741
1862
|
return stats['RMSE']
|
1742
1863
|
|
1743
|
-
def getMAE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1864
|
+
def getMAE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1744
1865
|
'''
|
1745
1866
|
this function is a wrapper for calculateTimeseriesStats specifically to return the MAE
|
1746
1867
|
|
@@ -1761,7 +1882,7 @@ def getMAE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1761
1882
|
|
1762
1883
|
return stats['MAE']
|
1763
1884
|
|
1764
|
-
def getMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> float:
|
1885
|
+
def getMSE(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> float:
|
1765
1886
|
'''
|
1766
1887
|
this function is a wrapper for calculateTimeseriesStats specifically to return the MSE
|
1767
1888
|
|
@@ -1782,7 +1903,7 @@ def getMSE(data:pandas.DataFrame, observed:str = None, simulated:str = None, res
|
|
1782
1903
|
|
1783
1904
|
return stats['MSE']
|
1784
1905
|
|
1785
|
-
def getTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str = None, resample:str = None ) -> dict:
|
1906
|
+
def getTimeseriesStats(data:pandas.DataFrame, observed:Optional[str] = None, simulated:Optional[str] = None, resample:Optional[str] = None ) -> dict:
|
1786
1907
|
'''
|
1787
1908
|
this function is a wrapper for calculateTimeseriesStats specifically to return all stats
|
1788
1909
|
|
@@ -1803,4 +1924,107 @@ def getTimeseriesStats(data:pandas.DataFrame, observed:str = None, simulated:str
|
|
1803
1924
|
|
1804
1925
|
return stats
|
1805
1926
|
|
1927
|
+
def readSWATPlusOutputs(filePath: str, column: Optional[str] = None, unit: Optional[int] = None, gis_id: Optional[int] = None, name: Optional[str] = None):
|
1928
|
+
'''
|
1929
|
+
Read SWAT+ output files and return a pandas DataFrame with proper date handling
|
1930
|
+
and optional filtering capabilities.
|
1931
|
+
|
1932
|
+
Parameters:
|
1933
|
+
-----------
|
1934
|
+
filePath: str
|
1935
|
+
Path to the SWAT+ output file
|
1936
|
+
column: str, optional
|
1937
|
+
Name of the column to extract. If not specified, returns all columns.
|
1938
|
+
If specified, returns first match, or specify multiple columns as comma-separated string
|
1939
|
+
unit: int, optional
|
1940
|
+
Filter by unit number. If not specified, returns all units
|
1941
|
+
gis_id: int, optional
|
1942
|
+
Filter by gis_id. If not specified, returns all gis_ids
|
1943
|
+
name: str, optional
|
1944
|
+
Filter by name. If not specified, returns all names
|
1945
|
+
|
1946
|
+
Returns:
|
1947
|
+
--------
|
1948
|
+
pandas.DataFrame or None
|
1949
|
+
DataFrame with date column and requested data, filtered as specified
|
1950
|
+
'''
|
1951
|
+
|
1952
|
+
if not exists(filePath):
|
1953
|
+
print('! SWAT+ result file does not exist')
|
1954
|
+
return None
|
1955
|
+
|
1956
|
+
# Read the header line (line 2, index 1)
|
1957
|
+
with open(filePath, 'r') as f:
|
1958
|
+
lines = f.readlines()
|
1959
|
+
|
1960
|
+
header_line = lines[1].strip()
|
1961
|
+
headers = header_line.split()
|
1962
|
+
|
1963
|
+
# Handle duplicate column names
|
1964
|
+
column_counts = defaultdict(int)
|
1965
|
+
modified_header = []
|
1966
|
+
for col_name in headers:
|
1967
|
+
column_counts[col_name] += 1
|
1968
|
+
if column_counts[col_name] > 1:
|
1969
|
+
modified_header.append(f"{col_name}_{column_counts[col_name]}")
|
1970
|
+
else:
|
1971
|
+
modified_header.append(col_name)
|
1972
|
+
|
1973
|
+
# Add extra columns to handle potential mismatches
|
1974
|
+
modified_header = modified_header + ['extra1', 'extra2']
|
1975
|
+
|
1976
|
+
try:
|
1977
|
+
df = pandas.read_csv(filePath, delim_whitespace=True, skiprows=3, names=modified_header, index_col=False)
|
1978
|
+
except:
|
1979
|
+
sys.stdout.write(f'\r! could not read {filePath} using pandas, check the number of columns\n')
|
1980
|
+
sys.stdout.flush()
|
1981
|
+
return None
|
1982
|
+
|
1983
|
+
# Remove extra columns
|
1984
|
+
df = df.drop(columns=['extra1', 'extra2'], errors='ignore')
|
1985
|
+
|
1986
|
+
# Convert all columns to numeric except 'name' (which is string)
|
1987
|
+
for col in df.columns:
|
1988
|
+
if col != 'name':
|
1989
|
+
df[col] = pandas.to_numeric(df[col], errors='coerce')
|
1990
|
+
|
1991
|
+
# Create date column from yr, mon, day
|
1992
|
+
try:
|
1993
|
+
df['date'] = pandas.to_datetime(pandas.DataFrame({'year': df.yr, 'month': df.mon, 'day': df.day}))
|
1994
|
+
except KeyError:
|
1995
|
+
# If some date columns are missing, create a simple index-based date
|
1996
|
+
df['date'] = pandas.date_range(start='2000-01-01', periods=len(df), freq='D')
|
1997
|
+
except:
|
1998
|
+
# If date creation fails for any other reason, use index-based date
|
1999
|
+
df['date'] = pandas.date_range(start='2000-01-01', periods=len(df), freq='D')
|
2000
|
+
|
2001
|
+
# Filter by unit if specified
|
2002
|
+
if unit is not None and 'unit' in df.columns:
|
2003
|
+
df = df[df['unit'] == unit]
|
2004
|
+
|
2005
|
+
# Filter by gis_id if specified
|
2006
|
+
if gis_id is not None and 'gis_id' in df.columns:
|
2007
|
+
df = df[df['gis_id'] == gis_id]
|
2008
|
+
|
2009
|
+
# Filter by name if specified
|
2010
|
+
if name is not None and 'name' in df.columns:
|
2011
|
+
df = df[df['name'] == name]
|
2012
|
+
|
2013
|
+
# Handle column selection
|
2014
|
+
if column is not None and column != "*":
|
2015
|
+
# Parse comma-separated columns
|
2016
|
+
requested_cols = [col.strip() for col in column.split(',')]
|
2017
|
+
|
2018
|
+
# Always include date column
|
2019
|
+
selected_cols = ['date']
|
2020
|
+
|
2021
|
+
# Add requested columns if they exist
|
2022
|
+
for req_col in requested_cols:
|
2023
|
+
if req_col in df.columns:
|
2024
|
+
selected_cols.append(req_col)
|
2025
|
+
|
2026
|
+
df = df[selected_cols]
|
2027
|
+
|
2028
|
+
return df
|
2029
|
+
|
1806
2030
|
ignoreWarnings()
|
ccfx/excel.py
CHANGED
@@ -29,7 +29,7 @@ class excel:
|
|
29
29
|
self.date_format = None
|
30
30
|
|
31
31
|
def create(self):
|
32
|
-
self.
|
32
|
+
self.createPath(os.path.dirname(self.path))
|
33
33
|
self.book = xlsxwriter.Workbook(self.path)
|
34
34
|
|
35
35
|
def addSheet(self, sheet_name):
|
@@ -42,7 +42,7 @@ class excel:
|
|
42
42
|
|
43
43
|
def writeDate(self, sheet_name, row, column, datetime_obj):
|
44
44
|
if self.date_format is None:
|
45
|
-
self.
|
45
|
+
self.setDateFormat()
|
46
46
|
|
47
47
|
self.sheet_names[sheet_name].write_datetime(
|
48
48
|
row, column, datetime_obj, self.date_format)
|
ccfx/mssqlConnection.py
CHANGED
@@ -21,7 +21,7 @@ import geopandas
|
|
21
21
|
|
22
22
|
|
23
23
|
# classes
|
24
|
-
class
|
24
|
+
class mssqlConnection:
|
25
25
|
def __init__(self, server, username, password, driver, trust_server_ssl = True) -> None:
|
26
26
|
self.server = server
|
27
27
|
self.username = username
|
@@ -85,20 +85,49 @@ class mssql_connection:
|
|
85
85
|
self.cursor = self.connection.cursor()
|
86
86
|
self.cursor.execute(query)
|
87
87
|
tables = [row[0] for row in self.cursor.fetchall()]
|
88
|
-
print("> list of tables in the active database:")
|
89
|
-
for table in tables:
|
90
|
-
|
88
|
+
# print("> list of tables in the active database:")
|
89
|
+
# for table in tables:
|
90
|
+
# print(f"\t- {table}")
|
91
91
|
except pyodbc.Error as e:
|
92
92
|
print("Error occurred while fetching the list of tables:")
|
93
93
|
print(e)
|
94
94
|
|
95
95
|
return tables
|
96
96
|
|
97
|
+
def listColumns(self, tableName: str, dbName: str | None = None) -> list[str]:
|
98
|
+
if dbName:
|
99
|
+
self.connect_db(dbName)
|
100
|
+
|
101
|
+
schema, tbl = ('dbo', tableName) if '.' not in tableName else tableName.split('.', 1)
|
102
|
+
|
103
|
+
sql = """
|
104
|
+
SELECT column_name
|
105
|
+
FROM information_schema.columns
|
106
|
+
WHERE table_schema = ? AND table_name = ?
|
107
|
+
ORDER BY ordinal_position
|
108
|
+
"""
|
109
|
+
|
110
|
+
try:
|
111
|
+
with self.connection.cursor() as cur:
|
112
|
+
cur.execute(sql, (schema, tbl))
|
113
|
+
return [row[0] for row in cur.fetchall()]
|
114
|
+
except pyodbc.Error as e:
|
115
|
+
print(f"Could not list columns for {tableName}: {e}")
|
116
|
+
return []
|
117
|
+
|
97
118
|
|
98
119
|
def readTable(self, table_name:str, db_name:str = None, columns:list = None, geom_col:str = None, v = True):
|
99
120
|
if db_name is not None:
|
100
121
|
self.connect_db(db_name)
|
101
122
|
|
123
|
+
# ensure geometry column is not in columns if specified
|
124
|
+
if geom_col is not None:
|
125
|
+
if columns is None:
|
126
|
+
columns = self.listColumns(table_name, db_name)
|
127
|
+
|
128
|
+
columns = [col for col in columns if col != geom_col]
|
129
|
+
|
130
|
+
|
102
131
|
if columns is not None and geom_col is not None:
|
103
132
|
columns.append(f"{geom_col}.STAsText() as {geom_col}_wkt")
|
104
133
|
query = f"SELECT {','.join(columns)} FROM {table_name}"
|
@@ -221,7 +250,9 @@ class mssql_connection:
|
|
221
250
|
self.connection.close()
|
222
251
|
self.connection = None
|
223
252
|
self.cursor = None
|
224
|
-
|
253
|
+
if v: print("> connection closed...")
|
254
|
+
else:
|
255
|
+
if v: print("> no connection to close...")
|
225
256
|
|
226
257
|
def disconnect(self, v = True):
|
227
258
|
self.close(v = v)
|
@@ -0,0 +1,11 @@
|
|
1
|
+
ccfx/__init__.py,sha256=UK62VcGS84SJyGVg1bK4FltZj7OkpdoyhoFWeXcKsX0,144
|
2
|
+
ccfx/ccfx.py,sha256=kBsVuHOZTR5J-upESSfo6Q9bYgeux6_jrbcXfcEyJvU,72663
|
3
|
+
ccfx/excel.py,sha256=vm_cm4huKKx4_Nstr5neJzhBLmoZjg8qxjzz4hcF5hg,4754
|
4
|
+
ccfx/mssqlConnection.py,sha256=C3HxzgZHmHy_de9EbMaXzR8NrkJxwHc8a00qzxQu_gs,8984
|
5
|
+
ccfx/sqliteConnection.py,sha256=jEJ94D5ySt84N7AeDpa27Rclt1NaKhkX6nYzidwApIg,11104
|
6
|
+
ccfx/word.py,sha256=AGa64jX5Zl5qotZh5L0QmrsjTnktIBhmj_ByRKZ88vw,3061
|
7
|
+
ccfx-1.0.2.dist-info/licenses/LICENSE,sha256=EuxaawJg_OOCLfikkCGgfXPZmxR-x_5PH7_2e9M-3eA,1099
|
8
|
+
ccfx-1.0.2.dist-info/METADATA,sha256=9NhF0qQsVrbyaIs65MQRc5G0Si1hzj4XFRHX-fOi9RA,11260
|
9
|
+
ccfx-1.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
10
|
+
ccfx-1.0.2.dist-info/top_level.txt,sha256=_cSvSA1WX2K8TgoV3iBJUdUZZqMKJbOPLNnKLYSLHaw,5
|
11
|
+
ccfx-1.0.2.dist-info/RECORD,,
|
ccfx-1.0.0.dist-info/RECORD
DELETED
@@ -1,11 +0,0 @@
|
|
1
|
-
ccfx/__init__.py,sha256=UK62VcGS84SJyGVg1bK4FltZj7OkpdoyhoFWeXcKsX0,144
|
2
|
-
ccfx/ccfx.py,sha256=hUeU8i3v3Y5PTfHhAfXNeoRjacdeWrYUeznwArXsEOk,64046
|
3
|
-
ccfx/excel.py,sha256=cQ4TQW49XqbMB3sSS0IOhO3-WArIolEBIrvOvhFyPtI,4757
|
4
|
-
ccfx/mssqlConnection.py,sha256=TwyZXhHHI7zy6BSfH1pszuHVJ5cmndRC5dVxvEtSTks,7904
|
5
|
-
ccfx/sqliteConnection.py,sha256=jEJ94D5ySt84N7AeDpa27Rclt1NaKhkX6nYzidwApIg,11104
|
6
|
-
ccfx/word.py,sha256=AGa64jX5Zl5qotZh5L0QmrsjTnktIBhmj_ByRKZ88vw,3061
|
7
|
-
ccfx-1.0.0.dist-info/licenses/LICENSE,sha256=EuxaawJg_OOCLfikkCGgfXPZmxR-x_5PH7_2e9M-3eA,1099
|
8
|
-
ccfx-1.0.0.dist-info/METADATA,sha256=Uoy4Br0JKKZxW8cd9Bj-dguujuMX2CRRVbiZ_lk-8QQ,11260
|
9
|
-
ccfx-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
10
|
-
ccfx-1.0.0.dist-info/top_level.txt,sha256=_cSvSA1WX2K8TgoV3iBJUdUZZqMKJbOPLNnKLYSLHaw,5
|
11
|
-
ccfx-1.0.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|