tunned-geobr 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

tunned_geobr/read_existent_pch.py

@@ -0,0 +1,168 @@
+import geopandas as gpd
+import os
+import tempfile
+import urllib.parse
+import requests
+import shutil
+from zipfile import ZipFile
+from pathlib import Path
+from io import BytesIO
+import warnings
+import json
+
+def read_existent_pch(simplified=False, verbose=False):
+    """Download Existing Small Hydroelectric Power Plants (PCH) data from EPE.
+    
+    This function downloads and processes existing small hydroelectric power plants (PCH) data from EPE 
+    (Energy Research Company). The data includes information about existing small hydroelectric 
+    power generation projects across Brazil.
+    Original source: EPE (Empresa de Pesquisa Energética)
+    
+    Parameters
+    ----------
+    simplified : boolean, by default False
+        If True, returns a simplified version of the dataset with fewer columns
+    verbose : boolean, by default False
+        If True, prints detailed information about the download process
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Geodataframe with existing small hydroelectric power plants data
+        
+    Example
+    -------
+    >>> from tunned_geobr import read_existent_pch
+    
+    # Read existing small hydroelectric power plants data
+    >>> existent_pch = read_existent_pch()
+    """
+    
+    # URL for the EPE geoserver
+    url = r"https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22PCH%20-%20Base%20Existente%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+ 
+    try:
+        # Disable SSL verification warning
+        warnings.filterwarnings('ignore', message='Unverified HTTPS request')
+        
+        if verbose:
+            print("Requesting data from EPE server...")
+            
+        response = requests.get(url, timeout=60, verify=False)
+        if not response.ok:
+            raise Exception(f"Error getting JSON response: {response.status_code}")
+
+        json_response = response.json()
+        
+        if verbose:
+            print(f"JSON response received: {json.dumps(json_response, indent=2)[:500]}...")
+            
+        if 'results' not in json_response or len(json_response['results']) == 0:
+            raise Exception("Invalid JSON response structure")
+            
+        if 'value' not in json_response['results'][0] or 'url' not in json_response['results'][0]['value']:
+            raise Exception("URL not found in JSON response")
+            
+        file_url = json_response['results'][0]['value']['url']
+        
+        if verbose:
+            print(f"Downloading file from: {file_url}")
+        
+        file_response = requests.get(file_url, stream=True, timeout=60, verify=False)
+        if not file_response.ok:
+            raise Exception(f"Error downloading file: {file_response.status_code}")
+        
+        # Check if content is actually a zip file
+        content = file_response.content
+        if len(content) < 100:
+            if verbose:
+                print(f"Warning: Downloaded content is very small ({len(content)} bytes)")
+                print(f"Content preview: {content[:100]}")
+            
+        # Create a temporary directory to extract the files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if verbose:
+                print(f"Extracting files to temporary directory: {temp_dir}")
+                
+            try:
+                # Extract the zip file
+                with ZipFile(BytesIO(content)) as zip_ref:
+                    zip_ref.extractall(temp_dir)
+                    
+                    if verbose:
+                        print(f"Files in zip: {zip_ref.namelist()}")
+            except Exception as zip_error:
+                if verbose:
+                    print(f"Error extracting zip: {str(zip_error)}")
+                    print(f"Saving content to debug.zip for inspection")
+                    with open("debug.zip", "wb") as f:
+                        f.write(content)
+                raise Exception(f"Failed to extract zip file: {str(zip_error)}")
+            
+            # Find the shapefile
+            all_files = os.listdir(temp_dir)
+            if verbose:
+                print(f"Files in temp directory: {all_files}")
+                
+            shp_files = [f for f in all_files if f.endswith('.shp')]
+            if not shp_files:
+                # Try looking in subdirectories
+                for root, dirs, files in os.walk(temp_dir):
+                    shp_files.extend([os.path.join(root, f) for f in files if f.endswith('.shp')])
+                
+                if not shp_files:
+                    raise Exception("No shapefile found in the downloaded data")
+            
+            # Read the shapefile
+            shp_path = shp_files[0] if os.path.isabs(shp_files[0]) else os.path.join(temp_dir, shp_files[0])
+            if verbose:
+                print(f"Reading shapefile: {shp_path}")
+                
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            gdf = gdf.to_crs(4674)
+            
+            if verbose:
+                print(f"Data loaded successfully with {len(gdf)} records")
+                print(f"Columns: {gdf.columns.tolist()}")
+            
+            if simplified:
+                # Keep only the most relevant columns
+                columns_to_keep = [
+                    'geometry',
+                    'nome',            # Power plant name
+                    'potencia',        # Capacity in MW
+                    'rio',             # River name
+                    'bacia',           # Basin
+                    'sub_bacia',       # Sub-basin
+                    'uf',              # State
+                    'municipio',       # Municipality
+                    'situacao'         # Status
+                ]
+                
+                # Filter columns that actually exist in the dataset
+                existing_columns = ['geometry'] + [col for col in columns_to_keep[1:] if col in gdf.columns]
+                if len(existing_columns) <= 1:
+                    if verbose:
+                        print("Warning: No matching columns found for simplified version. Returning all columns.")
+                else:
+                    gdf = gdf[existing_columns]
+    
+    except Exception as e:
+        raise Exception(f"Error downloading or processing existing small hydroelectric power plants data: {str(e)}")
+        
+    return gdf
+
+if __name__ == '__main__':
+    try:
+        pch_data = read_existent_pch(verbose=True)
+        print(f"Downloaded existing small hydroelectric power plants data with {len(pch_data)} records and {len(pch_data.columns)} columns")
+        
+        # Test simplified version
+        simplified_data = read_existent_pch(simplified=True)
+        print(f"Simplified data has {len(simplified_data.columns)} columns: {simplified_data.columns.tolist()}")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        import traceback
+        traceback.print_exc()

tunned_geobr/read_existent_solar.py

@@ -0,0 +1,165 @@
+import geopandas as gpd
+import os
+import tempfile
+import urllib.parse
+import requests
+import shutil
+from zipfile import ZipFile
+from pathlib import Path
+from io import BytesIO
+import warnings
+import json
+
+def read_existent_solar(simplified=False, verbose=False):
+    """Download Existing Solar Power Plants data from EPE.
+    
+    This function downloads and processes existing solar power plants data from EPE 
+    (Energy Research Company). The data includes information about operational solar power 
+    generation facilities across Brazil.
+    Original source: EPE (Empresa de Pesquisa Energética)
+    
+    Parameters
+    ----------
+    simplified : boolean, by default False
+        If True, returns a simplified version of the dataset with fewer columns
+    verbose : boolean, by default False
+        If True, prints detailed information about the download process
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Geodataframe with existing solar power plants data
+        
+    Example
+    -------
+    >>> from tunned_geobr import read_existent_solar
+    
+    # Read existing solar power plants data
+    >>> existent_solar = read_existent_solar()
+    """
+    
+    # URL for the EPE geoserver WFS service
+    url = r"https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22UFV%20-%20Base%20Existente%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8202935.359686549%2C-4447637.219321614%5D%2C%5B-8202935.359686549%2C1119424.4247428626%5D%2C%5B-3487076.4626055676%2C1119424.4247428626%5D%2C%5B-3487076.4626055676%2C-4447637.219321614%5D%2C%5B-8202935.359686549%2C-4447637.219321614%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+ 
+    try:
+        # Disable SSL verification warning
+        warnings.filterwarnings('ignore', message='Unverified HTTPS request')
+        
+        if verbose:
+            print("Requesting data from EPE server...")
+            
+        response = requests.get(url, timeout=60, verify=False)
+        if not response.ok:
+            raise Exception(f"Error getting JSON response: {response.status_code}")
+
+        json_response = response.json()
+        
+        if verbose:
+            print(f"JSON response received: {json.dumps(json_response, indent=2)[:500]}...")
+            
+        if 'results' not in json_response or len(json_response['results']) == 0:
+            raise Exception("Invalid JSON response structure")
+            
+        if 'value' not in json_response['results'][0] or 'url' not in json_response['results'][0]['value']:
+            raise Exception("URL not found in JSON response")
+            
+        file_url = json_response['results'][0]['value']['url']
+        
+        if verbose:
+            print(f"Downloading file from: {file_url}")
+        
+        file_response = requests.get(file_url, stream=True, timeout=60, verify=False)
+        if not file_response.ok:
+            raise Exception(f"Error downloading file: {file_response.status_code}")
+        
+        # Check if content is actually a zip file
+        content = file_response.content
+        if len(content) < 100:
+            if verbose:
+                print(f"Warning: Downloaded content is very small ({len(content)} bytes)")
+                print(f"Content preview: {content[:100]}")
+            
+        # Create a temporary directory to extract the files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if verbose:
+                print(f"Extracting files to temporary directory: {temp_dir}")
+                
+            try:
+                # Extract the zip file
+                with ZipFile(BytesIO(content)) as zip_ref:
+                    zip_ref.extractall(temp_dir)
+                    
+                    if verbose:
+                        print(f"Files in zip: {zip_ref.namelist()}")
+            except Exception as zip_error:
+                if verbose:
+                    print(f"Error extracting zip: {str(zip_error)}")
+                    print(f"Saving content to debug.zip for inspection")
+                    with open("debug.zip", "wb") as f:
+                        f.write(content)
+                raise Exception(f"Failed to extract zip file: {str(zip_error)}")
+            
+            # Find the shapefile
+            all_files = os.listdir(temp_dir)
+            if verbose:
+                print(f"Files in temp directory: {all_files}")
+                
+            shp_files = [f for f in all_files if f.endswith('.shp')]
+            if not shp_files:
+                # Try looking in subdirectories
+                for root, dirs, files in os.walk(temp_dir):
+                    shp_files.extend([os.path.join(root, f) for f in files if f.endswith('.shp')])
+                
+                if not shp_files:
+                    raise Exception("No shapefile found in the downloaded data")
+            
+            # Read the shapefile
+            shp_path = shp_files[0] if os.path.isabs(shp_files[0]) else os.path.join(temp_dir, shp_files[0])
+            if verbose:
+                print(f"Reading shapefile: {shp_path}")
+                
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            gdf = gdf.to_crs(4674)
+            
+            if verbose:
+                print(f"Data loaded successfully with {len(gdf)} records")
+                print(f"Columns: {gdf.columns.tolist()}")
+            
+            if simplified:
+                # Keep only the most relevant columns based on actual data structure
+                columns_to_keep = [
+                    'geometry',
+                    'nome',       # Plant name
+                    'potencia',   # Power capacity
+                    'leilao',     # Auction
+                    'ceg',        # CEG code
+                    'ano_prev'    # Expected year
+                ]
+                
+                # Filter columns that actually exist in the dataset
+                existing_columns = ['geometry'] + [col for col in columns_to_keep[1:] if col in gdf.columns]
+                if len(existing_columns) <= 1:
+                    if verbose:
+                        print("Warning: No matching columns found for simplified version. Returning all columns.")
+                else:
+                    gdf = gdf[existing_columns]
+    
+    except Exception as e:
+        raise Exception(f"Error downloading or processing existing solar power plants data: {str(e)}")
+        
+    return gdf
+
+if __name__ == '__main__':
+    try:
+        solar_data = read_existent_solar(verbose=True)
+        print(f"Downloaded existing solar power plants data with {len(solar_data)} records and {len(solar_data.columns)} columns")
+        
+        # Test simplified version
+        simplified_data = read_existent_solar(simplified=True)
+        print(f"Simplified data has {len(simplified_data.columns)} columns: {simplified_data.columns.tolist()}")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        import traceback
+        traceback.print_exc()

tunned_geobr/read_existent_substations.py

@@ -0,0 +1,128 @@
+import geopandas as gpd
+import requests
+import shutil
+import zipfile
+import tempfile
+import warnings
+import os
+from shapely.geometry.point import Point
+
+
+def read_existent_substations(simplified=False, verbose=False):
+    """Download data of existing electrical substations in Brazil.
+
+    This function downloads and returns data of existing electrical substations
+    in Brazil as a GeoPandas GeoDataFrame. The data comes from EPE (Energy Research Company).
+
+    Parameters
+    ----------
+    simplified : bool, optional
+        If True, returns a simplified version of the dataset with only the most
+        important columns. If False, returns the complete dataset. Default is False.
+    verbose : bool, optional
+        If True, displays detailed messages about the download and processing
+        steps. Default is False.
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        A GeoDataFrame containing data on existing electrical substations in Brazil.
+
+    Raises
+    ------
+    Exception
+        If the download or processing of the data fails.
+
+    Example
+    -------
+    >>> from tunned_geobr import read_existent_substations
+    >>> 
+    >>> # Read the data
+    >>> substations = read_existent_substations()
+    >>> 
+    >>> # Plot the data
+    >>> substations.plot()
+    """
+    
+    if verbose:
+        print("Downloading data of existing electrical substations in Brazil")
+    
+    # Define the URL for the API request
+    url = "https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22Subesta%C3%A7%C3%B5es%20-%20Base%20Existente%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+    
+    try:
+        # Make the API request
+        response = requests.get(url)
+        response.raise_for_status()
+        
+        # Parse the JSON response
+        data = response.json()
+        
+        # Extract the URL for the zip file
+        if 'results' in data and len(data['results']) > 0 and 'value' in data['results'][0]:
+            download_url = data['results'][0]['value']['url']
+        else:
+            raise Exception("Failed to extract download URL from API response")
+        
+        # Create a temporary directory to store the downloaded files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Download the zip file
+            zip_path = os.path.join(temp_dir, "substations.zip")
+            if verbose:
+                print("Downloading zip file")
+            
+            response = requests.get(download_url, stream=True)
+            response.raise_for_status()
+            
+            with open(zip_path, 'wb') as f:
+                response.raw.decode_content = True
+                shutil.copyfileobj(response.raw, f)
+            
+            # Extract the zip file
+            if verbose:
+                print("Extracting files")
+            
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                zip_ref.extractall(temp_dir)
+            
+            # Find the shapefile in the extracted files
+            shp_files = [f for f in os.listdir(temp_dir) if f.endswith('.shp')]
+            
+            if not shp_files:
+                raise Exception("No shapefile found in the downloaded zip file")
+            
+            # Read the shapefile
+            if verbose:
+                print("Reading shapefile")
+            
+            shp_path = os.path.join(temp_dir, shp_files[0])
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            if verbose:
+                print("Converting to SIRGAS 2000 (EPSG:4674)")
+            
+            gdf = gdf.to_crs(epsg=4674)
+            
+            # Simplify the dataset if requested
+            if simplified:
+                if verbose:
+                    print("Simplifying the dataset")
+                
+                # Select only the most important columns
+                # Adjust these columns based on the actual data structure
+                cols_to_keep = ['NOME', 'TENSAO', 'PROPRIETAR', 'UF', 'MUNICIPIO', 'geometry']
+                cols_available = [col for col in cols_to_keep if col in gdf.columns]
+                
+                if not cols_available:
+                    warnings.warn("None of the specified columns for simplification are available. Returning the full dataset.")
+                else:
+                    gdf = gdf[cols_available]
+            
+            if verbose:
+                print("Finished processing existing electrical substations data")
+            
+            return gdf
+    
+    except Exception as e:
+        raise Exception(f"Failed to download or process existing electrical substations data: {str(e)}")

tunned_geobr/read_existent_transmission_lines.py

@@ -0,0 +1,128 @@
+import geopandas as gpd
+import requests
+import shutil
+import zipfile
+import tempfile
+import warnings
+import os
+from shapely.geometry.point import Point
+
+
+def read_existent_transmission_lines(simplified=False, verbose=False):
+    """Download data of existing transmission lines in Brazil.
+
+    This function downloads and returns data of existing transmission lines
+    in Brazil as a GeoPandas GeoDataFrame. The data comes from EPE (Energy Research Company).
+
+    Parameters
+    ----------
+    simplified : bool, optional
+        If True, returns a simplified version of the dataset with only the most
+        important columns. If False, returns the complete dataset. Default is False.
+    verbose : bool, optional
+        If True, displays detailed messages about the download and processing
+        steps. Default is False.
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        A GeoDataFrame containing data on existing transmission lines in Brazil.
+
+    Raises
+    ------
+    Exception
+        If the download or processing of the data fails.
+
+    Example
+    -------
+    >>> from tunned_geobr import read_existent_transmission_lines
+    >>> 
+    >>> # Read the data
+    >>> transmission_lines = read_existent_transmission_lines()
+    >>> 
+    >>> # Plot the data
+    >>> transmission_lines.plot()
+    """
+    
+    if verbose:
+        print("Downloading data of existing transmission lines in Brazil")
+    
+    # Define the URL for the API request
+    url = "https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22Linhas%20de%20Transmiss%C3%A3o%20-%20Base%20Existente%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+    
+    try:
+        # Make the API request
+        response = requests.get(url)
+        response.raise_for_status()
+        
+        # Parse the JSON response
+        data = response.json()
+        
+        # Extract the URL for the zip file
+        if 'results' in data and len(data['results']) > 0 and 'value' in data['results'][0]:
+            download_url = data['results'][0]['value']['url']
+        else:
+            raise Exception("Failed to extract download URL from API response")
+        
+        # Create a temporary directory to store the downloaded files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Download the zip file
+            zip_path = os.path.join(temp_dir, "transmission_lines.zip")
+            if verbose:
+                print("Downloading zip file")
+            
+            response = requests.get(download_url, stream=True)
+            response.raise_for_status()
+            
+            with open(zip_path, 'wb') as f:
+                response.raw.decode_content = True
+                shutil.copyfileobj(response.raw, f)
+            
+            # Extract the zip file
+            if verbose:
+                print("Extracting files")
+            
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                zip_ref.extractall(temp_dir)
+            
+            # Find the shapefile in the extracted files
+            shp_files = [f for f in os.listdir(temp_dir) if f.endswith('.shp')]
+            
+            if not shp_files:
+                raise Exception("No shapefile found in the downloaded zip file")
+            
+            # Read the shapefile
+            if verbose:
+                print("Reading shapefile")
+            
+            shp_path = os.path.join(temp_dir, shp_files[0])
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            if verbose:
+                print("Converting to SIRGAS 2000 (EPSG:4674)")
+            
+            gdf = gdf.to_crs(epsg=4674)
+            
+            # Simplify the dataset if requested
+            if simplified:
+                if verbose:
+                    print("Simplifying the dataset")
+                
+                # Select only the most important columns
+                # Adjust these columns based on the actual data structure
+                cols_to_keep = ['NOME', 'TENSAO', 'EXTENSAO', 'PROPRIETAR', 'UF', 'geometry']
+                cols_available = [col for col in cols_to_keep if col in gdf.columns]
+                
+                if not cols_available:
+                    warnings.warn("None of the specified columns for simplification are available. Returning the full dataset.")
+                else:
+                    gdf = gdf[cols_available]
+            
+            if verbose:
+                print("Finished processing existing transmission lines data")
+            
+            return gdf
+    
+    except Exception as e:
+        raise Exception(f"Failed to download or process existing transmission lines data: {str(e)}")