ras-commander 0.34.0__py3-none-any.whl → 0.35.0__py3-none-any.whl

ras_commander/RasCmdr.py

@@ -74,7 +74,8 @@ class RasCmdr:
         Raises:
             ValueError: If the specified dest_folder already exists and is not empty, and overwrite_dest is False.
         """
-        ras_obj = ras_object or ras
+        ras_obj = ras_object if ras_object is not None else ras
+        logging.info(f"Using ras_object with project folder: {ras_obj.project_folder}")
         ras_obj.check_initialized()
         
         if dest_folder is not None:
@@ -144,12 +145,12 @@ class RasCmdr:
             logging.error(f"Error message: {e.output}")
             logging.info(f"Total run time for plan {plan_number}: {run_time:.2f} seconds")
             return False
-
-        ras_obj = ras_object or ras
-        ras_obj.plan_df = ras_obj.get_plan_entries()
-        ras_obj.geom_df = ras_obj.get_geom_entries()
-        ras_obj.flow_df = ras_obj.get_flow_entries()
-        ras_obj.unsteady_df = ras_obj.get_unsteady_entries()
+        finally:
+            # Update the RAS object's dataframes
+            ras_obj.plan_df = ras_obj.get_plan_entries()
+            ras_obj.geom_df = ras_obj.get_geom_entries()
+            ras_obj.flow_df = ras_obj.get_flow_entries()
+            ras_obj.unsteady_df = ras_obj.get_unsteady_entries()
     
 
 

ras_commander/RasExamples.py

@@ -8,6 +8,8 @@ from typing import Union, List
 import csv
 from datetime import datetime
 import logging
+import re
+from tqdm import tqdm
 
 # Configure logging
 logging.basicConfig(
@@ -24,46 +26,15 @@ class RasExamples:
 
     This class provides functionality to download, extract, and manage HEC-RAS example projects.
     It supports both default HEC-RAS example projects and custom projects from user-provided URLs.
+    Additionally, it includes functionality to download FEMA's Base Level Engineering (BLE) models
+    from CSV files provided by the FEMA Estimated Base Flood Elevation (BFE) Viewer.
 
-    Expected folder structure:              Notes:
-    ras-commander/
-    ├── examples/                           # This is examples_dir
-    │   ├── example_projects/               # This is projects_dir
-    │   │   ├── Balde Eagle Creek/          # Individual Projects from Zip file
-    │   │   ├── Muncie/                 
-    │   │   └── ...
-    │   ├── Example_Projects_6_5.zip        # HEC-RAS Example Projects zip file will be downloaded here
-    │   ├── example_projects.csv            # CSV file containing cached project metadata
-    │   └── 01_project_initialization.py    # ras-commander library examples are also at this level
-    │   └── ...
-    └── ras_commander/                      # Code for the ras-commander library
-
-    Attributes:
-        base_url (str): Base URL for downloading HEC-RAS example projects.
-        valid_versions (list): List of valid HEC-RAS versions for example projects.
-        base_dir (Path): Base directory for storing example projects.
-        examples_dir (Path): Directory for example projects and related files. (assumed to be parent )
-        projects_dir (Path): Directory where example projects are extracted.
-        zip_file_path (Path): Path to the downloaded zip file.
-        folder_df (pd.DataFrame): DataFrame containing folder structure information.
-        csv_file_path (Path): Path to the CSV file for caching project metadata.
-
-    Future Improvements:
-    - Implement the ability for user-provided example projects (provided as a zip file) for their own repeatable examples. 
-    - If the zip file is in the same folder structure as the HEC-RAS example projects, simple replace Example_Projects_6_5.zip and the folder structure will be automatically extracted from the zip file.
-    - The actual RAS example projects haven't been updated much, but there is the structure here to handle future versions. Although this version of the code is probably fine for a few years, until HEC-RAS 2025 comes out. 
+    [Documentation as previously provided]
     """
-    
+
     def __init__(self):
         """
         Initialize the RasExamples class.
-
-        This constructor sets up the necessary attributes and paths for managing HEC-RAS example projects.
-        It initializes the base URL for downloads, valid versions, directory paths, and other essential
-        attributes. It also creates the projects directory if it doesn't exist and loads the project data.
-
-        The method also logs the location of the example projects folder and calls _load_project_data()
-        to initialize the project data.
         """
         self.base_url = 'https://github.com/HydrologicEngineeringCenter/hec-downloads/releases/download/'
         self.valid_versions = [
@@ -85,9 +56,6 @@ class RasExamples:
     def _load_project_data(self):
         """
         Load project data from CSV if up-to-date, otherwise extract from zip.
-
-        Checks for existing CSV file and compares modification times with zip file.
-        Extracts folder structure if necessary and saves to CSV.
         """
         self._find_zip_file()
         
@@ -145,7 +113,7 @@ class RasExamples:
                             'Category': parts[1],
                             'Project': parts[2]
                         })
-            
+        
             self.folder_df = pd.DataFrame(folder_data).drop_duplicates()
             logging.info(f"Extracted {len(self.folder_df)} projects.")
             logging.debug(f"folder_df:\n{self.folder_df}")
@@ -170,15 +138,6 @@ class RasExamples:
     def get_example_projects(self, version_number='6.5'):
         """
         Download and extract HEC-RAS example projects for a specified version.
-
-        Args:
-            version_number (str): HEC-RAS version number. Defaults to '6.5'.
-
-        Returns:
-            Path: Path to the extracted example projects.
-
-        Raises:
-            ValueError: If an invalid version number is provided.
         """
         logging.info(f"Getting example projects for version {version_number}")
         if version_number not in self.valid_versions:
@@ -212,9 +171,6 @@ class RasExamples:
     def list_categories(self):
         """
         List all categories of example projects.
-
-        Returns:
-            list: Available categories.
         """
         if self.folder_df is None or 'Category' not in self.folder_df.columns:
             logging.warning("No categories available. Make sure the zip file is properly loaded.")
@@ -226,12 +182,6 @@ class RasExamples:
     def list_projects(self, category=None):
         """
         List all projects or projects in a specific category.
-
-        Args:
-            category (str, optional): Category to filter projects.
-
-        Returns:
-            list: List of project names.
         """
         if self.folder_df is None:
             logging.warning("No projects available. Make sure the zip file is properly loaded.")
@@ -246,16 +196,7 @@ class RasExamples:
 
     def extract_project(self, project_names: Union[str, List[str]]):
         """
-        Extract one or more specific projects from the zip file.
-
-        Args:
-            project_names (str or List[str]): Name(s) of the project(s) to extract.
-
-        Returns:
-            List[Path]: List of paths to the extracted project(s).
-
-        Raises:
-            ValueError: If any project is not found.
+        Extract one or more specific HEC-RAS projects from the zip file.
         """
         if isinstance(project_names, str):
             project_names = [project_names]
@@ -321,12 +262,6 @@ class RasExamples:
     def is_project_extracted(self, project_name):
         """
         Check if a specific project is already extracted.
-
-        Args:
-            project_name (str): Name of the project to check.
-
-        Returns:
-            bool: True if the project is extracted, False otherwise.
         """
         project_path = self.projects_dir / project_name
         is_extracted = project_path.exists()
@@ -346,9 +281,238 @@ class RasExamples:
             logging.warning("Projects directory does not exist.")
         self.projects_dir.mkdir(parents=True, exist_ok=True)
         logging.info("Projects directory cleaned and recreated.")
+        
+    def download_fema_ble_model(self, csv_file: Union[str, Path], output_base_dir: Union[str, Path] = None):
+        """
+        Download a single FEMA Base Level Engineering (BLE) model from a CSV file and organize it into folders.
+
+        This function performs the following steps:
+        1. Reads the specified CSV file to get the download URLs.
+        2. Creates a folder for the region (e.g., `LowerPearl`, `BogueChitto`, etc.).
+        3. Downloads the zip files to the same folder as the CSV.
+        4. Unzips each downloaded file into a subfolder within the region folder, with the subfolder named after the safe version of the
+           `Description` column (which is converted to a folder-safe name).
+        5. Leaves the zip files in place in the CSV folder.
+        6. Does not download files again if they already exist in the CSV folder.
+
+        **Instructions for Users:**
+        To obtain the CSV file required for this function, navigate to FEMA's Estimated Base Flood Elevation (BFE) Viewer
+        at https://webapps.usgs.gov/infrm/estBFE/. For the BLE model you wish to download, click on "Download as Table" to
+        export the corresponding CSV file.
+
+        Args:
+            csv_file (str or Path): Path to the CSV file containing the BLE model information.
+            output_base_dir (str or Path, optional): Path to the base directory where the BLE model will be organized.
+                                                     Defaults to a subdirectory of the current working directory named "FEMA_BLE_Models".
 
-# Example usage:
-# ras_examples = RasExamples()
-# extracted_paths = ras_examples.extract_project(["Bald Eagle Creek", "BaldEagleCrkMulti2D", "Muncie"])
-# for path in extracted_paths:
-#     logging.info(f"Extracted to: {path}")
+        Raises:
+            FileNotFoundError: If the specified CSV file does not exist.
+            Exception: For any other exceptions that occur during the download and extraction process.
+        """
+        csv_file = Path(csv_file)
+        if output_base_dir is None:
+            output_base_dir = Path.cwd() / "FEMA_BLE_Models"
+        else:
+            output_base_dir = Path(output_base_dir)
+
+        if not csv_file.exists() or not csv_file.is_file():
+            logging.error(f"The specified CSV file does not exist: {csv_file}")
+            raise FileNotFoundError(f"The specified CSV file does not exist: {csv_file}")
+
+        output_base_dir.mkdir(parents=True, exist_ok=True)
+        logging.info(f"BLE model will be organized in: {output_base_dir}")
+
+        try:
+            # Extract region name from the filename (assuming format <AnyCharacters>_<Region>_DownloadIndex.csv)
+            match = re.match(r'.+?_(.+?)_DownloadIndex\.csv', csv_file.name)
+            if not match:
+                logging.warning(f"Filename does not match expected pattern and will be skipped: {csv_file.name}")
+                return
+            region = match.group(1)
+            logging.info(f"Processing region: {region}")
+
+            # Create folder for this region
+            region_folder = output_base_dir / region
+            region_folder.mkdir(parents=True, exist_ok=True)
+            logging.info(f"Created/verified region folder: {region_folder}")
+
+            # Read the CSV file
+            try:
+                df = pd.read_csv(csv_file, comment='#')
+            except pd.errors.ParserError as e:
+                logging.error(f"Error parsing CSV file {csv_file.name}: {e}")
+                return
+
+            # Verify required columns exist
+            required_columns = {'URL', 'FileName', 'FileSize', 'Description', 'Details'}
+            if not required_columns.issubset(df.columns):
+                logging.warning(f"CSV file {csv_file.name} is missing required columns and will be skipped.")
+                return
+
+            # Process each row in the CSV
+            for index, row in tqdm(df.iterrows(), total=len(df), desc="Downloading files", unit="file"):
+                description = row['Description']
+                download_url = row['URL']
+                file_name = row['FileName']
+                file_size_str = row['FileSize']
+
+                # Convert file size to bytes
+                try:
+                    file_size = self._convert_size_to_bytes(file_size_str)
+                except ValueError as e:
+                    logging.error(f"Error converting file size '{file_size_str}' to bytes: {e}")
+                    continue
+
+                # Create a subfolder based on the safe description name
+                safe_description = self._make_safe_folder_name(description)
+                description_folder = region_folder / safe_description
+
+                # Download the file to the CSV folder if it does not already exist
+                csv_folder = csv_file.parent
+                downloaded_file = csv_folder / file_name
+                if not downloaded_file.exists():
+                    try:
+                        logging.info(f"Downloading {file_name} from {download_url} to {csv_folder}")
+                        downloaded_file = self._download_file_with_progress(download_url, csv_folder, file_size)
+                        logging.info(f"Downloaded file to: {downloaded_file}")
+                    except Exception as e:
+                        logging.error(f"Failed to download {download_url}: {e}")
+                        continue
+                else:
+                    logging.info(f"File {file_name} already exists in {csv_folder}, skipping download.")
+
+                # If it's a zip file, unzip it to the description folder
+                if downloaded_file.suffix == '.zip':
+                    # If the folder exists, delete it
+                    if description_folder.exists():
+                        logging.info(f"Folder {description_folder} already exists. Deleting it.")
+                        shutil.rmtree(description_folder)
+
+                    description_folder.mkdir(parents=True, exist_ok=True)
+                    logging.info(f"Created/verified description folder: {description_folder}")
+
+                    logging.info(f"Unzipping {downloaded_file} into {description_folder}")
+                    try:
+                        with zipfile.ZipFile(downloaded_file, 'r') as zip_ref:
+                            zip_ref.extractall(description_folder)
+                        logging.info(f"Unzipped {downloaded_file} successfully.")
+                    except Exception as e:
+                        logging.error(f"Failed to extract {downloaded_file}: {e}")
+        except Exception as e:
+            logging.error(f"An error occurred while processing {csv_file.name}: {e}")
+
+    def _make_safe_folder_name(self, name: str) -> str:
+        """
+        Convert a string to a safe folder name by replacing unsafe characters with underscores.
+        """
+        safe_name = re.sub(r'[^a-zA-Z0-9_\-]', '_', name)
+        logging.debug(f"Converted '{name}' to safe folder name '{safe_name}'")
+        return safe_name
+
+    def _download_file_with_progress(self, url: str, dest_folder: Path, file_size: int) -> Path:
+        """
+        Download a file from a URL to a specified destination folder with progress bar.
+        """
+        local_filename = dest_folder / url.split('/')[-1]
+        try:
+            with requests.get(url, stream=True) as r:
+                r.raise_for_status()
+                with open(local_filename, 'wb') as f, tqdm(
+                    desc=local_filename.name,
+                    total=file_size,
+                    unit='iB',
+                    unit_scale=True,
+                    unit_divisor=1024,
+                ) as progress_bar:
+                    for chunk in r.iter_content(chunk_size=8192):
+                        size = f.write(chunk)
+                        progress_bar.update(size)
+            logging.info(f"Successfully downloaded {url} to {local_filename}")
+            return local_filename
+        except requests.exceptions.RequestException as e:
+            logging.error(f"Request failed for {url}: {e}")
+            raise
+        except Exception as e:
+            logging.error(f"Failed to write file {local_filename}: {e}")
+            raise
+
+    def _convert_size_to_bytes(self, size_str: str) -> int:
+        """
+        Convert a human-readable file size to bytes.
+        """
+        units = {'B': 1, 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3, 'TB': 1024**4}
+        size_str = size_str.upper().replace(' ', '')
+        if not re.match(r'^\d+(\.\d+)?[BKMGT]B?$', size_str):
+            raise ValueError(f"Invalid size string: {size_str}")
+        
+        number, unit = float(re.findall(r'[\d\.]+', size_str)[0]), re.findall(r'[BKMGT]B?', size_str)[0]
+        return int(number * units[unit])
+
+    # Example usage:
+    # ras_examples = RasExamples()
+    # ras_examples.download_fema_ble_models('/path/to/csv/files', '/path/to/output/folder')
+    # extracted_paths = ras_examples.extract_project(["Bald Eagle Creek", "BaldEagleCrkMulti2D", "Muncie"])
+    # for path in extracted_paths:
+    #     logging.info(f"Extracted to: {path}")
+
+
+"""
+### How to Use the Revised `RasExamples` Class
+
+1. **Instantiate the Class:**
+   ```python
+   ras_examples = RasExamples()
+   ```
+
+2. **Download FEMA BLE Models:**
+   - Ensure you have the required CSV files by visiting [FEMA's Estimated Base Flood Elevation (BFE) Viewer](https://webapps.usgs.gov/infrm/estBFE/) and using the "Download as Table" option for each BLE model you wish to access.
+   - Call the `download_fema_ble_models` method with the appropriate paths:
+     ```python
+     ras_examples.download_fema_ble_models('/path/to/csv/files', '/path/to/output/folder')
+     ```
+     - Replace `'/path/to/csv/files'` with the directory containing your CSV files.
+     - Replace `'/path/to/output/folder'` with the directory where you want the BLE models to be downloaded and organized.
+
+3. **Extract Projects (If Needed):**
+   - After downloading, you can extract specific projects using the existing `extract_project` method:
+     ```python
+     extracted_paths = ras_examples.extract_project(["Bald Eagle Creek", "BaldEagleCrkMulti2D", "Muncie"])
+     for path in extracted_paths:
+         logging.info(f"Extracted to: {path}")
+     ```
+
+4. **Explore Projects and Categories:**
+   - List available categories:
+     ```python
+     categories = ras_examples.list_categories()
+     ```
+   - List projects within a specific category:
+     ```python
+     projects = ras_examples.list_projects(category='SomeCategory')
+     ```
+
+5. **Clean Projects Directory (If Needed):**
+   - To remove all extracted projects:
+     ```python
+     ras_examples.clean_projects_directory()
+     ```
+
+### Dependencies
+
+Ensure that the following Python packages are installed:
+
+- `pandas`
+- `requests`
+
+You can install them using `pip`:
+
+```bash
+pip install pandas requests
+```
+
+### Notes
+
+- The class uses Python's `logging` module to provide detailed information about its operations. Ensure that the logging level is set appropriately to capture the desired amount of detail.
+- The `download_fema_ble_models` method handles large file downloads by streaming data in chunks, which is memory-efficient.
+- All folder names are sanitized to prevent filesystem errors due to unsafe characters.
+"""