ras-commander 0.34.0__py3-none-any.whl → 0.36.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ras_commander/RasCmdr.py +362 -333
- ras_commander/RasExamples.py +329 -132
- ras_commander/RasGeo.py +38 -28
- ras_commander/RasGpt.py +142 -0
- ras_commander/RasHdf.py +1480 -109
- ras_commander/RasPlan.py +123 -167
- ras_commander/RasPrj.py +213 -136
- ras_commander/RasUnsteady.py +37 -22
- ras_commander/RasUtils.py +98 -82
- ras_commander/__init__.py +11 -13
- ras_commander/logging_config.py +80 -0
- {ras_commander-0.34.0.dist-info → ras_commander-0.36.0.dist-info}/METADATA +73 -13
- ras_commander-0.36.0.dist-info/RECORD +17 -0
- ras_commander-0.34.0.dist-info/RECORD +0 -15
- {ras_commander-0.34.0.dist-info → ras_commander-0.36.0.dist-info}/LICENSE +0 -0
- {ras_commander-0.34.0.dist-info → ras_commander-0.36.0.dist-info}/WHEEL +0 -0
- {ras_commander-0.34.0.dist-info → ras_commander-0.36.0.dist-info}/top_level.txt +0 -0
ras_commander/RasExamples.py
CHANGED
@@ -1,3 +1,28 @@
|
|
1
|
+
"""
|
2
|
+
RasExamples - Manage and load HEC-RAS example projects for testing and development
|
3
|
+
|
4
|
+
This module is part of the ras-commander library and uses a centralized logging configuration.
|
5
|
+
|
6
|
+
Logging Configuration:
|
7
|
+
- The logging is set up in the logging_config.py file.
|
8
|
+
- A @log_call decorator is available to automatically log function calls.
|
9
|
+
- Log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL
|
10
|
+
- Logs are written to both console and a rotating file handler.
|
11
|
+
- The default log file is 'ras_commander.log' in the 'logs' directory.
|
12
|
+
- The default log level is INFO.
|
13
|
+
|
14
|
+
To use logging in this module:
|
15
|
+
1. Use the @log_call decorator for automatic function call logging.
|
16
|
+
2. For additional logging, use logger.[level]() calls (e.g., logger.info(), logger.debug()).
|
17
|
+
3. Obtain the logger using: logger = logging.getLogger(__name__)
|
18
|
+
|
19
|
+
Example:
|
20
|
+
@log_call
|
21
|
+
def my_function():
|
22
|
+
logger = logging.getLogger(__name__)
|
23
|
+
logger.debug("Additional debug information")
|
24
|
+
# Function logic here
|
25
|
+
"""
|
1
26
|
import os
|
2
27
|
import requests
|
3
28
|
import zipfile
|
@@ -8,15 +33,12 @@ from typing import Union, List
|
|
8
33
|
import csv
|
9
34
|
from datetime import datetime
|
10
35
|
import logging
|
36
|
+
import re
|
37
|
+
from tqdm import tqdm
|
38
|
+
from ras_commander import get_logger
|
39
|
+
from ras_commander.logging_config import log_call
|
11
40
|
|
12
|
-
|
13
|
-
logging.basicConfig(
|
14
|
-
level=logging.INFO, # Set the logging level to INFO
|
15
|
-
format='%(asctime)s - %(levelname)s - %(message)s', # Log message format
|
16
|
-
handlers=[
|
17
|
-
logging.StreamHandler() # Log to stderr
|
18
|
-
]
|
19
|
-
)
|
41
|
+
logger = get_logger(__name__)
|
20
42
|
|
21
43
|
class RasExamples:
|
22
44
|
"""
|
@@ -24,46 +46,13 @@ class RasExamples:
|
|
24
46
|
|
25
47
|
This class provides functionality to download, extract, and manage HEC-RAS example projects.
|
26
48
|
It supports both default HEC-RAS example projects and custom projects from user-provided URLs.
|
27
|
-
|
28
|
-
|
29
|
-
ras-commander/
|
30
|
-
├── examples/ # This is examples_dir
|
31
|
-
│ ├── example_projects/ # This is projects_dir
|
32
|
-
│ │ ├── Balde Eagle Creek/ # Individual Projects from Zip file
|
33
|
-
│ │ ├── Muncie/
|
34
|
-
│ │ └── ...
|
35
|
-
│ ├── Example_Projects_6_5.zip # HEC-RAS Example Projects zip file will be downloaded here
|
36
|
-
│ ├── example_projects.csv # CSV file containing cached project metadata
|
37
|
-
│ └── 01_project_initialization.py # ras-commander library examples are also at this level
|
38
|
-
│ └── ...
|
39
|
-
└── ras_commander/ # Code for the ras-commander library
|
40
|
-
|
41
|
-
Attributes:
|
42
|
-
base_url (str): Base URL for downloading HEC-RAS example projects.
|
43
|
-
valid_versions (list): List of valid HEC-RAS versions for example projects.
|
44
|
-
base_dir (Path): Base directory for storing example projects.
|
45
|
-
examples_dir (Path): Directory for example projects and related files. (assumed to be parent )
|
46
|
-
projects_dir (Path): Directory where example projects are extracted.
|
47
|
-
zip_file_path (Path): Path to the downloaded zip file.
|
48
|
-
folder_df (pd.DataFrame): DataFrame containing folder structure information.
|
49
|
-
csv_file_path (Path): Path to the CSV file for caching project metadata.
|
50
|
-
|
51
|
-
Future Improvements:
|
52
|
-
- Implement the ability for user-provided example projects (provided as a zip file) for their own repeatable examples.
|
53
|
-
- If the zip file is in the same folder structure as the HEC-RAS example projects, simple replace Example_Projects_6_5.zip and the folder structure will be automatically extracted from the zip file.
|
54
|
-
- The actual RAS example projects haven't been updated much, but there is the structure here to handle future versions. Although this version of the code is probably fine for a few years, until HEC-RAS 2025 comes out.
|
49
|
+
Additionally, it includes functionality to download FEMA's Base Level Engineering (BLE) models
|
50
|
+
from CSV files provided by the FEMA Estimated Base Flood Elevation (BFE) Viewer.
|
55
51
|
"""
|
56
|
-
|
52
|
+
@log_call
|
57
53
|
def __init__(self):
|
58
54
|
"""
|
59
55
|
Initialize the RasExamples class.
|
60
|
-
|
61
|
-
This constructor sets up the necessary attributes and paths for managing HEC-RAS example projects.
|
62
|
-
It initializes the base URL for downloads, valid versions, directory paths, and other essential
|
63
|
-
attributes. It also creates the projects directory if it doesn't exist and loads the project data.
|
64
|
-
|
65
|
-
The method also logs the location of the example projects folder and calls _load_project_data()
|
66
|
-
to initialize the project data.
|
67
56
|
"""
|
68
57
|
self.base_url = 'https://github.com/HydrologicEngineeringCenter/hec-downloads/releases/download/'
|
69
58
|
self.valid_versions = [
|
@@ -79,56 +68,56 @@ class RasExamples:
|
|
79
68
|
self.csv_file_path = self.examples_dir / 'example_projects.csv'
|
80
69
|
|
81
70
|
self.projects_dir.mkdir(parents=True, exist_ok=True)
|
82
|
-
|
71
|
+
logger.info(f"Example projects folder: {self.projects_dir}")
|
83
72
|
self._load_project_data()
|
84
73
|
|
74
|
+
@log_call
|
85
75
|
def _load_project_data(self):
|
86
76
|
"""
|
87
77
|
Load project data from CSV if up-to-date, otherwise extract from zip.
|
88
|
-
|
89
|
-
Checks for existing CSV file and compares modification times with zip file.
|
90
|
-
Extracts folder structure if necessary and saves to CSV.
|
91
78
|
"""
|
92
79
|
self._find_zip_file()
|
93
80
|
|
94
81
|
if not self.zip_file_path:
|
95
|
-
|
82
|
+
logger.info("No example projects zip file found. Downloading...")
|
96
83
|
self.get_example_projects()
|
97
84
|
|
98
85
|
try:
|
99
86
|
zip_modified_time = os.path.getmtime(self.zip_file_path)
|
100
87
|
except FileNotFoundError:
|
101
|
-
|
88
|
+
logger.error(f"Zip file not found at {self.zip_file_path}.")
|
102
89
|
return
|
103
90
|
|
104
91
|
if self.csv_file_path.exists():
|
105
92
|
csv_modified_time = os.path.getmtime(self.csv_file_path)
|
106
93
|
|
107
94
|
if csv_modified_time >= zip_modified_time:
|
108
|
-
|
95
|
+
logger.info("Loading project data from CSV...")
|
109
96
|
try:
|
110
97
|
self.folder_df = pd.read_csv(self.csv_file_path)
|
111
|
-
|
98
|
+
logger.info(f"Loaded {len(self.folder_df)} projects from CSV. Use list_categories() and list_projects() to explore them.")
|
112
99
|
except Exception as e:
|
113
|
-
|
100
|
+
logger.error(f"Failed to read CSV file: {e}")
|
114
101
|
self.folder_df = None
|
115
102
|
return
|
116
103
|
|
117
|
-
|
104
|
+
logger.info("Extracting folder structure from zip file...")
|
118
105
|
self._extract_folder_structure()
|
119
106
|
self._save_to_csv()
|
120
107
|
|
108
|
+
@log_call
|
121
109
|
def _find_zip_file(self):
|
122
110
|
"""Locate the example projects zip file in the examples directory."""
|
123
111
|
for version in self.valid_versions:
|
124
112
|
potential_zip = self.examples_dir / f"Example_Projects_{version.replace('.', '_')}.zip"
|
125
113
|
if potential_zip.exists():
|
126
114
|
self.zip_file_path = potential_zip
|
127
|
-
|
115
|
+
logger.info(f"Found zip file: {self.zip_file_path}")
|
128
116
|
break
|
129
117
|
else:
|
130
|
-
|
118
|
+
logger.warning("No existing example projects zip file found.")
|
131
119
|
|
120
|
+
@log_call
|
132
121
|
def _extract_folder_structure(self):
|
133
122
|
"""
|
134
123
|
Extract folder structure from the zip file.
|
@@ -145,45 +134,38 @@ class RasExamples:
|
|
145
134
|
'Category': parts[1],
|
146
135
|
'Project': parts[2]
|
147
136
|
})
|
148
|
-
|
137
|
+
|
149
138
|
self.folder_df = pd.DataFrame(folder_data).drop_duplicates()
|
150
|
-
|
151
|
-
|
139
|
+
logger.info(f"Extracted {len(self.folder_df)} projects.")
|
140
|
+
logger.debug(f"folder_df:\n{self.folder_df}")
|
152
141
|
except zipfile.BadZipFile:
|
153
|
-
|
142
|
+
logger.error(f"The file {self.zip_file_path} is not a valid zip file.")
|
154
143
|
self.folder_df = pd.DataFrame(columns=['Category', 'Project'])
|
155
144
|
except Exception as e:
|
156
|
-
|
145
|
+
logger.error(f"An error occurred while extracting the folder structure: {str(e)}")
|
157
146
|
self.folder_df = pd.DataFrame(columns=['Category', 'Project'])
|
158
147
|
|
148
|
+
@log_call
|
159
149
|
def _save_to_csv(self):
|
160
150
|
"""Save the extracted folder structure to CSV file."""
|
161
151
|
if self.folder_df is not None and not self.folder_df.empty:
|
162
152
|
try:
|
163
153
|
self.folder_df.to_csv(self.csv_file_path, index=False)
|
164
|
-
|
154
|
+
logger.info(f"Saved project data to {self.csv_file_path}")
|
165
155
|
except Exception as e:
|
166
|
-
|
156
|
+
logger.error(f"Failed to save project data to CSV: {e}")
|
167
157
|
else:
|
168
|
-
|
158
|
+
logger.warning("No folder data to save to CSV.")
|
169
159
|
|
160
|
+
@log_call
|
170
161
|
def get_example_projects(self, version_number='6.5'):
|
171
162
|
"""
|
172
163
|
Download and extract HEC-RAS example projects for a specified version.
|
173
|
-
|
174
|
-
Args:
|
175
|
-
version_number (str): HEC-RAS version number. Defaults to '6.5'.
|
176
|
-
|
177
|
-
Returns:
|
178
|
-
Path: Path to the extracted example projects.
|
179
|
-
|
180
|
-
Raises:
|
181
|
-
ValueError: If an invalid version number is provided.
|
182
164
|
"""
|
183
|
-
|
165
|
+
logger.info(f"Getting example projects for version {version_number}")
|
184
166
|
if version_number not in self.valid_versions:
|
185
167
|
error_msg = f"Invalid version number. Valid versions are: {', '.join(self.valid_versions)}"
|
186
|
-
|
168
|
+
logger.error(error_msg)
|
187
169
|
raise ValueError(error_msg)
|
188
170
|
|
189
171
|
zip_url = f"{self.base_url}1.0.31/Example_Projects_{version_number.replace('.', '_')}.zip"
|
@@ -193,69 +175,54 @@ class RasExamples:
|
|
193
175
|
self.zip_file_path = self.examples_dir / f"Example_Projects_{version_number.replace('.', '_')}.zip"
|
194
176
|
|
195
177
|
if not self.zip_file_path.exists():
|
196
|
-
|
178
|
+
logger.info(f"Downloading HEC-RAS Example Projects from {zip_url}. \nThe file is over 400 MB, so it may take a few minutes to download....")
|
197
179
|
try:
|
198
180
|
response = requests.get(zip_url, stream=True)
|
199
181
|
response.raise_for_status()
|
200
182
|
with open(self.zip_file_path, 'wb') as file:
|
201
183
|
shutil.copyfileobj(response.raw, file)
|
202
|
-
|
184
|
+
logger.info(f"Downloaded to {self.zip_file_path}")
|
203
185
|
except requests.exceptions.RequestException as e:
|
204
|
-
|
186
|
+
logger.error(f"Failed to download the zip file: {e}")
|
205
187
|
raise
|
206
188
|
else:
|
207
|
-
|
189
|
+
logger.info("HEC-RAS Example Projects zip file already exists. Skipping download.")
|
208
190
|
|
209
191
|
self._load_project_data()
|
210
192
|
return self.projects_dir
|
211
193
|
|
194
|
+
@log_call
|
212
195
|
def list_categories(self):
|
213
196
|
"""
|
214
197
|
List all categories of example projects.
|
215
|
-
|
216
|
-
Returns:
|
217
|
-
list: Available categories.
|
218
198
|
"""
|
219
199
|
if self.folder_df is None or 'Category' not in self.folder_df.columns:
|
220
|
-
|
200
|
+
logger.warning("No categories available. Make sure the zip file is properly loaded.")
|
221
201
|
return []
|
222
202
|
categories = self.folder_df['Category'].unique()
|
223
|
-
|
203
|
+
logger.info(f"Available categories: {', '.join(categories)}")
|
224
204
|
return categories.tolist()
|
225
205
|
|
206
|
+
@log_call
|
226
207
|
def list_projects(self, category=None):
|
227
208
|
"""
|
228
209
|
List all projects or projects in a specific category.
|
229
|
-
|
230
|
-
Args:
|
231
|
-
category (str, optional): Category to filter projects.
|
232
|
-
|
233
|
-
Returns:
|
234
|
-
list: List of project names.
|
235
210
|
"""
|
236
211
|
if self.folder_df is None:
|
237
|
-
|
212
|
+
logger.warning("No projects available. Make sure the zip file is properly loaded.")
|
238
213
|
return []
|
239
214
|
if category:
|
240
215
|
projects = self.folder_df[self.folder_df['Category'] == category]['Project'].unique()
|
241
|
-
|
216
|
+
logger.info(f"Projects in category '{category}': {', '.join(projects)}")
|
242
217
|
else:
|
243
218
|
projects = self.folder_df['Project'].unique()
|
244
|
-
|
219
|
+
logger.info(f"All available projects: {', '.join(projects)}")
|
245
220
|
return projects.tolist()
|
246
221
|
|
222
|
+
@log_call
|
247
223
|
def extract_project(self, project_names: Union[str, List[str]]):
|
248
224
|
"""
|
249
|
-
Extract one or more specific projects from the zip file.
|
250
|
-
|
251
|
-
Args:
|
252
|
-
project_names (str or List[str]): Name(s) of the project(s) to extract.
|
253
|
-
|
254
|
-
Returns:
|
255
|
-
List[Path]: List of paths to the extracted project(s).
|
256
|
-
|
257
|
-
Raises:
|
258
|
-
ValueError: If any project is not found.
|
225
|
+
Extract one or more specific HEC-RAS projects from the zip file.
|
259
226
|
"""
|
260
227
|
if isinstance(project_names, str):
|
261
228
|
project_names = [project_names]
|
@@ -263,28 +230,28 @@ class RasExamples:
|
|
263
230
|
extracted_paths = []
|
264
231
|
|
265
232
|
for project_name in project_names:
|
266
|
-
|
267
|
-
|
233
|
+
logger.info("----- RasExamples Extracting Project -----")
|
234
|
+
logger.info(f"Extracting project '{project_name}'")
|
268
235
|
project_path = self.projects_dir / project_name
|
269
236
|
|
270
237
|
if project_path.exists():
|
271
|
-
|
238
|
+
logger.info(f"Project '{project_name}' already exists. Deleting existing folder...")
|
272
239
|
try:
|
273
240
|
shutil.rmtree(project_path)
|
274
|
-
|
241
|
+
logger.info(f"Existing folder for project '{project_name}' has been deleted.")
|
275
242
|
except Exception as e:
|
276
|
-
|
243
|
+
logger.error(f"Failed to delete existing project folder '{project_name}': {e}")
|
277
244
|
continue
|
278
245
|
|
279
246
|
if self.folder_df is None or self.folder_df.empty:
|
280
247
|
error_msg = "No project information available. Make sure the zip file is properly loaded."
|
281
|
-
|
248
|
+
logger.error(error_msg)
|
282
249
|
raise ValueError(error_msg)
|
283
250
|
|
284
251
|
project_info = self.folder_df[self.folder_df['Project'] == project_name]
|
285
252
|
if project_info.empty:
|
286
253
|
error_msg = f"Project '{project_name}' not found in the zip file."
|
287
|
-
|
254
|
+
logger.error(error_msg)
|
288
255
|
raise ValueError(error_msg)
|
289
256
|
|
290
257
|
category = project_info['Category'].iloc[0]
|
@@ -307,48 +274,278 @@ class RasExamples:
|
|
307
274
|
with zip_ref.open(file) as source, open(extract_path, "wb") as target:
|
308
275
|
shutil.copyfileobj(source, target)
|
309
276
|
|
310
|
-
|
277
|
+
logger.info(f"Successfully extracted project '{project_name}' to {project_path}")
|
311
278
|
extracted_paths.append(project_path)
|
312
279
|
except zipfile.BadZipFile:
|
313
|
-
|
280
|
+
logger.error(f"Error: The file {self.zip_file_path} is not a valid zip file.")
|
314
281
|
except FileNotFoundError:
|
315
|
-
|
282
|
+
logger.error(f"Error: The file {self.zip_file_path} was not found.")
|
316
283
|
except Exception as e:
|
317
|
-
|
318
|
-
|
284
|
+
logger.error(f"An unexpected error occurred while extracting the project: {str(e)}")
|
285
|
+
logger.info("----- RasExamples Extraction Complete -----")
|
319
286
|
return extracted_paths
|
320
287
|
|
288
|
+
@log_call
|
321
289
|
def is_project_extracted(self, project_name):
|
322
290
|
"""
|
323
291
|
Check if a specific project is already extracted.
|
324
|
-
|
325
|
-
Args:
|
326
|
-
project_name (str): Name of the project to check.
|
327
|
-
|
328
|
-
Returns:
|
329
|
-
bool: True if the project is extracted, False otherwise.
|
330
292
|
"""
|
331
293
|
project_path = self.projects_dir / project_name
|
332
294
|
is_extracted = project_path.exists()
|
333
|
-
|
295
|
+
logger.info(f"Project '{project_name}' extracted: {is_extracted}")
|
334
296
|
return is_extracted
|
335
297
|
|
298
|
+
@log_call
|
336
299
|
def clean_projects_directory(self):
|
337
300
|
"""Remove all extracted projects from the example_projects directory."""
|
338
|
-
|
301
|
+
logger.info(f"Cleaning projects directory: {self.projects_dir}")
|
339
302
|
if self.projects_dir.exists():
|
340
303
|
try:
|
341
304
|
shutil.rmtree(self.projects_dir)
|
342
|
-
|
305
|
+
logger.info("All projects have been removed.")
|
343
306
|
except Exception as e:
|
344
|
-
|
307
|
+
logger.error(f"Failed to remove projects directory: {e}")
|
345
308
|
else:
|
346
|
-
|
309
|
+
logger.warning("Projects directory does not exist.")
|
347
310
|
self.projects_dir.mkdir(parents=True, exist_ok=True)
|
348
|
-
|
311
|
+
logger.info("Projects directory cleaned and recreated.")
|
312
|
+
|
313
|
+
|
314
|
+
@log_call
|
315
|
+
def download_fema_ble_model(self, csv_file: Union[str, Path], output_base_dir: Union[str, Path] = None):
|
316
|
+
"""
|
317
|
+
Download a single FEMA Base Level Engineering (BLE) model from a CSV file and organize it into folders.
|
318
|
+
|
319
|
+
This function performs the following steps:
|
320
|
+
1. Reads the specified CSV file to get the download URLs.
|
321
|
+
2. Creates a folder for the region (e.g., `LowerPearl`, `BogueChitto`, etc.).
|
322
|
+
3. Downloads the zip files to the same folder as the CSV.
|
323
|
+
4. Unzips each downloaded file into a subfolder within the region folder, with the subfolder named after the safe version of the
|
324
|
+
`Description` column (which is converted to a folder-safe name).
|
325
|
+
5. Leaves the zip files in place in the CSV folder.
|
326
|
+
6. Does not download files again if they already exist in the CSV folder.
|
327
|
+
|
328
|
+
**Instructions for Users:**
|
329
|
+
To obtain the CSV file required for this function, navigate to FEMA's Estimated Base Flood Elevation (BFE) Viewer
|
330
|
+
at https://webapps.usgs.gov/infrm/estBFE/. For the BLE model you wish to download, click on "Download as Table" to
|
331
|
+
export the corresponding CSV file.
|
332
|
+
|
333
|
+
Args:
|
334
|
+
csv_file (str or Path): Path to the CSV file containing the BLE model information.
|
335
|
+
output_base_dir (str or Path, optional): Path to the base directory where the BLE model will be organized.
|
336
|
+
Defaults to a subdirectory of the current working directory named "FEMA_BLE_Models".
|
337
|
+
|
338
|
+
Raises:
|
339
|
+
FileNotFoundError: If the specified CSV file does not exist.
|
340
|
+
Exception: For any other exceptions that occur during the download and extraction process.
|
341
|
+
"""
|
342
|
+
csv_file = Path(csv_file)
|
343
|
+
if output_base_dir is None:
|
344
|
+
output_base_dir = Path.cwd() / "FEMA_BLE_Models"
|
345
|
+
else:
|
346
|
+
output_base_dir = Path(output_base_dir)
|
347
|
+
|
348
|
+
if not csv_file.exists() or not csv_file.is_file():
|
349
|
+
logger.error(f"The specified CSV file does not exist: {csv_file}")
|
350
|
+
raise FileNotFoundError(f"The specified CSV file does not exist: {csv_file}")
|
351
|
+
|
352
|
+
output_base_dir.mkdir(parents=True, exist_ok=True)
|
353
|
+
logger.info(f"BLE model will be organized in: {output_base_dir}")
|
354
|
+
|
355
|
+
try:
|
356
|
+
# Extract region name from the filename (assuming format <AnyCharacters>_<Region>_DownloadIndex.csv)
|
357
|
+
match = re.match(r'.+?_(.+?)_DownloadIndex\.csv', csv_file.name)
|
358
|
+
if not match:
|
359
|
+
logger.warning(f"Filename does not match expected pattern and will be skipped: {csv_file.name}")
|
360
|
+
return
|
361
|
+
region = match.group(1)
|
362
|
+
logger.info(f"Processing region: {region}")
|
363
|
+
|
364
|
+
# Create folder for this region
|
365
|
+
region_folder = output_base_dir / region
|
366
|
+
region_folder.mkdir(parents=True, exist_ok=True)
|
367
|
+
logger.info(f"Created/verified region folder: {region_folder}")
|
368
|
+
|
369
|
+
# Read the CSV file
|
370
|
+
try:
|
371
|
+
df = pd.read_csv(csv_file, comment='#')
|
372
|
+
except pd.errors.ParserError as e:
|
373
|
+
logger.error(f"Error parsing CSV file {csv_file.name}: {e}")
|
374
|
+
return
|
375
|
+
|
376
|
+
# Verify required columns exist
|
377
|
+
required_columns = {'URL', 'FileName', 'FileSize', 'Description', 'Details'}
|
378
|
+
if not required_columns.issubset(df.columns):
|
379
|
+
logger.warning(f"CSV file {csv_file.name} is missing required columns and will be skipped.")
|
380
|
+
return
|
381
|
+
|
382
|
+
# Process each row in the CSV
|
383
|
+
for index, row in tqdm(df.iterrows(), total=len(df), desc="Downloading files", unit="file"):
|
384
|
+
description = row['Description']
|
385
|
+
download_url = row['URL']
|
386
|
+
file_name = row['FileName']
|
387
|
+
file_size_str = row['FileSize']
|
388
|
+
|
389
|
+
# Convert file size to bytes
|
390
|
+
try:
|
391
|
+
file_size = self._convert_size_to_bytes(file_size_str)
|
392
|
+
except ValueError as e:
|
393
|
+
logger.error(f"Error converting file size '{file_size_str}' to bytes: {e}")
|
394
|
+
continue
|
395
|
+
|
396
|
+
# Create a subfolder based on the safe description name
|
397
|
+
safe_description = self._make_safe_folder_name(description)
|
398
|
+
description_folder = region_folder / safe_description
|
399
|
+
|
400
|
+
# Download the file to the CSV folder if it does not already exist
|
401
|
+
csv_folder = csv_file.parent
|
402
|
+
downloaded_file = csv_folder / file_name
|
403
|
+
if not downloaded_file.exists():
|
404
|
+
try:
|
405
|
+
logger.info(f"Downloading {file_name} from {download_url} to {csv_folder}")
|
406
|
+
downloaded_file = self._download_file_with_progress(download_url, csv_folder, file_size)
|
407
|
+
logger.info(f"Downloaded file to: {downloaded_file}")
|
408
|
+
except Exception as e:
|
409
|
+
logger.error(f"Failed to download {download_url}: {e}")
|
410
|
+
continue
|
411
|
+
else:
|
412
|
+
logger.info(f"File {file_name} already exists in {csv_folder}, skipping download.")
|
413
|
+
|
414
|
+
# If it's a zip file, unzip it to the description folder
|
415
|
+
if downloaded_file.suffix == '.zip':
|
416
|
+
# If the folder exists, delete it
|
417
|
+
if description_folder.exists():
|
418
|
+
logger.info(f"Folder {description_folder} already exists. Deleting it.")
|
419
|
+
shutil.rmtree(description_folder)
|
420
|
+
|
421
|
+
description_folder.mkdir(parents=True, exist_ok=True)
|
422
|
+
logger.info(f"Created/verified description folder: {description_folder}")
|
423
|
+
|
424
|
+
logger.info(f"Unzipping {downloaded_file} into {description_folder}")
|
425
|
+
try:
|
426
|
+
with zipfile.ZipFile(downloaded_file, 'r') as zip_ref:
|
427
|
+
zip_ref.extractall(description_folder)
|
428
|
+
logger.info(f"Unzipped {downloaded_file} successfully.")
|
429
|
+
except Exception as e:
|
430
|
+
logger.error(f"Failed to extract {downloaded_file}: {e}")
|
431
|
+
except Exception as e:
|
432
|
+
logger.error(f"An error occurred while processing {csv_file.name}: {e}")
|
349
433
|
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
434
|
+
@log_call
|
435
|
+
def _make_safe_folder_name(self, name: str) -> str:
|
436
|
+
"""
|
437
|
+
Convert a string to a safe folder name by replacing unsafe characters with underscores.
|
438
|
+
"""
|
439
|
+
safe_name = re.sub(r'[^a-zA-Z0-9_\-]', '_', name)
|
440
|
+
logger.debug(f"Converted '{name}' to safe folder name '{safe_name}'")
|
441
|
+
return safe_name
|
442
|
+
|
443
|
+
@log_call
|
444
|
+
def _download_file_with_progress(self, url: str, dest_folder: Path, file_size: int) -> Path:
|
445
|
+
"""
|
446
|
+
Download a file from a URL to a specified destination folder with progress bar.
|
447
|
+
"""
|
448
|
+
local_filename = dest_folder / url.split('/')[-1]
|
449
|
+
try:
|
450
|
+
with requests.get(url, stream=True) as r:
|
451
|
+
r.raise_for_status()
|
452
|
+
with open(local_filename, 'wb') as f, tqdm(
|
453
|
+
desc=local_filename.name,
|
454
|
+
total=file_size,
|
455
|
+
unit='iB',
|
456
|
+
unit_scale=True,
|
457
|
+
unit_divisor=1024,
|
458
|
+
) as progress_bar:
|
459
|
+
for chunk in r.iter_content(chunk_size=8192):
|
460
|
+
size = f.write(chunk)
|
461
|
+
progress_bar.update(size)
|
462
|
+
logger.info(f"Successfully downloaded {url} to {local_filename}")
|
463
|
+
return local_filename
|
464
|
+
except requests.exceptions.RequestException as e:
|
465
|
+
logger.error(f"Request failed for {url}: {e}")
|
466
|
+
raise
|
467
|
+
except Exception as e:
|
468
|
+
logger.error(f"Failed to write file {local_filename}: {e}")
|
469
|
+
raise
|
470
|
+
|
471
|
+
@log_call
|
472
|
+
def _convert_size_to_bytes(self, size_str: str) -> int:
|
473
|
+
"""
|
474
|
+
Convert a human-readable file size to bytes.
|
475
|
+
"""
|
476
|
+
units = {'B': 1, 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3, 'TB': 1024**4}
|
477
|
+
size_str = size_str.upper().replace(' ', '')
|
478
|
+
if not re.match(r'^\d+(\.\d+)?[BKMGT]B?$', size_str):
|
479
|
+
raise ValueError(f"Invalid size string: {size_str}")
|
480
|
+
|
481
|
+
number, unit = float(re.findall(r'[\d\.]+', size_str)[0]), re.findall(r'[BKMGT]B?', size_str)[0]
|
482
|
+
return int(number * units[unit])
|
483
|
+
|
484
|
+
# Example usage:
|
485
|
+
# ras_examples = RasExamples()
|
486
|
+
# ras_examples.download_fema_ble_models('/path/to/csv/files', '/path/to/output/folder')
|
487
|
+
# extracted_paths = ras_examples.extract_project(["Bald Eagle Creek", "BaldEagleCrkMulti2D", "Muncie"])
|
488
|
+
# for path in extracted_paths:
|
489
|
+
# logger.info(f"Extracted to: {path}")
|
490
|
+
|
491
|
+
|
492
|
+
"""
|
493
|
+
### How to Use the Revised `RasExamples` Class
|
494
|
+
|
495
|
+
1. **Instantiate the Class:**
|
496
|
+
```python
|
497
|
+
ras_examples = RasExamples()
|
498
|
+
```
|
499
|
+
|
500
|
+
2. **Download FEMA BLE Models:**
|
501
|
+
- Ensure you have the required CSV files by visiting [FEMA's Estimated Base Flood Elevation (BFE) Viewer](https://webapps.usgs.gov/infrm/estBFE/) and using the "Download as Table" option for each BLE model you wish to access.
|
502
|
+
- Call the `download_fema_ble_models` method with the appropriate paths:
|
503
|
+
```python
|
504
|
+
ras_examples.download_fema_ble_models('/path/to/csv/files', '/path/to/output/folder')
|
505
|
+
```
|
506
|
+
- Replace `'/path/to/csv/files'` with the directory containing your CSV files.
|
507
|
+
- Replace `'/path/to/output/folder'` with the directory where you want the BLE models to be downloaded and organized.
|
508
|
+
|
509
|
+
3. **Extract Projects (If Needed):**
|
510
|
+
- After downloading, you can extract specific projects using the existing `extract_project` method:
|
511
|
+
```python
|
512
|
+
extracted_paths = ras_examples.extract_project(["Bald Eagle Creek", "BaldEagleCrkMulti2D", "Muncie"])
|
513
|
+
for path in extracted_paths:
|
514
|
+
logging.info(f"Extracted to: {path}")
|
515
|
+
```
|
516
|
+
|
517
|
+
4. **Explore Projects and Categories:**
|
518
|
+
- List available categories:
|
519
|
+
```python
|
520
|
+
categories = ras_examples.list_categories()
|
521
|
+
```
|
522
|
+
- List projects within a specific category:
|
523
|
+
```python
|
524
|
+
projects = ras_examples.list_projects(category='SomeCategory')
|
525
|
+
```
|
526
|
+
|
527
|
+
5. **Clean Projects Directory (If Needed):**
|
528
|
+
- To remove all extracted projects:
|
529
|
+
```python
|
530
|
+
ras_examples.clean_projects_directory()
|
531
|
+
```
|
532
|
+
|
533
|
+
### Dependencies
|
534
|
+
|
535
|
+
Ensure that the following Python packages are installed:
|
536
|
+
|
537
|
+
- `pandas`
|
538
|
+
- `requests`
|
539
|
+
|
540
|
+
You can install them using `pip`:
|
541
|
+
|
542
|
+
```bash
|
543
|
+
pip install pandas requests
|
544
|
+
```
|
545
|
+
|
546
|
+
### Notes
|
547
|
+
|
548
|
+
- The class uses Python's `logging` module to provide detailed information about its operations. Ensure that the logging level is set appropriately to capture the desired amount of detail.
|
549
|
+
- The `download_fema_ble_models` method handles large file downloads by streaming data in chunks, which is memory-efficient.
|
550
|
+
- All folder names are sanitized to prevent filesystem errors due to unsafe characters.
|
551
|
+
"""
|