tunned-geobr 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

tunned_geobr/read_planned_transmission_lines.py

@@ -0,0 +1,165 @@
+import geopandas as gpd
+import os
+import tempfile
+import urllib.parse
+import requests
+import shutil
+from zipfile import ZipFile
+from pathlib import Path
+from io import BytesIO
+import warnings
+import json
+
+def read_planned_transmission_lines(simplified=False, verbose=False):
+    """Download Planned Transmission Lines data from EPE.
+    
+    This function downloads and processes planned transmission lines data from EPE 
+    (Energy Research Company). The data includes information about planned transmission 
+    lines across Brazil.
+    Original source: EPE (Empresa de Pesquisa Energética)
+    
+    Parameters
+    ----------
+    simplified : boolean, by default False
+        If True, returns a simplified version of the dataset with fewer columns
+    verbose : boolean, by default False
+        If True, prints detailed information about the download process
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Geodataframe with planned transmission lines data
+        
+    Example
+    -------
+    >>> from tunned_geobr import read_planned_transmission_lines
+    
+    # Read planned transmission lines data
+    >>> planned_lines = read_planned_transmission_lines()
+    """
+    
+    # URL for the EPE geoserver
+    url = r"https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22Linhas%20de%20Transmiss%C3%A3o%20-%20Expans%C3%A3o%20Planejada%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+ 
+    try:
+        # Disable SSL verification warning
+        warnings.filterwarnings('ignore', message='Unverified HTTPS request')
+        
+        if verbose:
+            print("Requesting data from EPE server...")
+            
+        response = requests.get(url, timeout=60, verify=False)
+        if not response.ok:
+            raise Exception(f"Error getting JSON response: {response.status_code}")
+
+        json_response = response.json()
+        
+        if verbose:
+            print(f"JSON response received: {json.dumps(json_response, indent=2)[:500]}...")
+            
+        if 'results' not in json_response or len(json_response['results']) == 0:
+            raise Exception("Invalid JSON response structure")
+            
+        if 'value' not in json_response['results'][0] or 'url' not in json_response['results'][0]['value']:
+            raise Exception("URL not found in JSON response")
+            
+        file_url = json_response['results'][0]['value']['url']
+        
+        if verbose:
+            print(f"Downloading file from: {file_url}")
+        
+        file_response = requests.get(file_url, stream=True, timeout=60, verify=False)
+        if not file_response.ok:
+            raise Exception(f"Error downloading file: {file_response.status_code}")
+        
+        # Check if content is actually a zip file
+        content = file_response.content
+        if len(content) < 100:
+            if verbose:
+                print(f"Warning: Downloaded content is very small ({len(content)} bytes)")
+                print(f"Content preview: {content[:100]}")
+            
+        # Create a temporary directory to extract the files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if verbose:
+                print(f"Extracting files to temporary directory: {temp_dir}")
+                
+            try:
+                # Extract the zip file
+                with ZipFile(BytesIO(content)) as zip_ref:
+                    zip_ref.extractall(temp_dir)
+                    
+                    if verbose:
+                        print(f"Files in zip: {zip_ref.namelist()}")
+            except Exception as zip_error:
+                if verbose:
+                    print(f"Error extracting zip: {str(zip_error)}")
+                    print(f"Saving content to debug.zip for inspection")
+                    with open("debug.zip", "wb") as f:
+                        f.write(content)
+                raise Exception(f"Failed to extract zip file: {str(zip_error)}")
+            
+            # Find the shapefile
+            all_files = os.listdir(temp_dir)
+            if verbose:
+                print(f"Files in temp directory: {all_files}")
+                
+            shp_files = [f for f in all_files if f.endswith('.shp')]
+            if not shp_files:
+                # Try looking in subdirectories
+                for root, dirs, files in os.walk(temp_dir):
+                    shp_files.extend([os.path.join(root, f) for f in files if f.endswith('.shp')])
+                
+                if not shp_files:
+                    raise Exception("No shapefile found in the downloaded data")
+            
+            # Read the shapefile
+            shp_path = shp_files[0] if os.path.isabs(shp_files[0]) else os.path.join(temp_dir, shp_files[0])
+            if verbose:
+                print(f"Reading shapefile: {shp_path}")
+                
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            gdf = gdf.to_crs(4674)
+            
+            if verbose:
+                print(f"Data loaded successfully with {len(gdf)} records")
+                print(f"Columns: {gdf.columns.tolist()}")
+            
+            if simplified:
+                # Keep only the most relevant columns
+                columns_to_keep = [
+                    'geometry',
+                    'nome',            # Line name
+                    'tensao',          # Voltage level
+                    'extensao',        # Length
+                    'ano_prev',        # Expected year
+                    'situacao'         # Status
+                ]
+                
+                # Filter columns that actually exist in the dataset
+                existing_columns = ['geometry'] + [col for col in columns_to_keep[1:] if col in gdf.columns]
+                if len(existing_columns) <= 1:
+                    if verbose:
+                        print("Warning: No matching columns found for simplified version. Returning all columns.")
+                else:
+                    gdf = gdf[existing_columns]
+    
+    except Exception as e:
+        raise Exception(f"Error downloading or processing planned transmission lines data: {str(e)}")
+        
+    return gdf
+
+if __name__ == '__main__':
+    try:
+        lines_data = read_planned_transmission_lines(verbose=True)
+        print(f"Downloaded planned transmission lines data with {len(lines_data)} records and {len(lines_data.columns)} columns")
+        
+        # Test simplified version
+        simplified_data = read_planned_transmission_lines(simplified=True)
+        print(f"Simplified data has {len(simplified_data.columns)} columns: {simplified_data.columns.tolist()}")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        import traceback
+        traceback.print_exc()

tunned_geobr/read_planned_uhe.py

@@ -0,0 +1,166 @@
+import geopandas as gpd
+import os
+import tempfile
+import urllib.parse
+import requests
+import shutil
+from zipfile import ZipFile
+from pathlib import Path
+from io import BytesIO
+import warnings
+import json
+
+def read_planned_uhe(simplified=False, verbose=False):
+    """Download Planned Large Hydroelectric Power Plants (UHE) data from EPE.
+    
+    This function downloads and processes planned large hydroelectric power plants (UHE) data from EPE 
+    (Energy Research Company). The data includes information about planned large hydroelectric 
+    power generation projects across Brazil.
+    Original source: EPE (Empresa de Pesquisa Energética)
+    
+    Parameters
+    ----------
+    simplified : boolean, by default False
+        If True, returns a simplified version of the dataset with fewer columns
+    verbose : boolean, by default False
+        If True, prints detailed information about the download process
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Geodataframe with planned large hydroelectric power plants data
+        
+    Example
+    -------
+    >>> from tunned_geobr import read_planned_uhe
+    
+    # Read planned large hydroelectric power plants data
+    >>> planned_uhe = read_planned_uhe()
+    """
+    
+    # URL for the EPE geoserver
+    url = r"https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22UHE%20-%20Expans%C3%A3o%20Planejada%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+ 
+    try:
+        # Disable SSL verification warning
+        warnings.filterwarnings('ignore', message='Unverified HTTPS request')
+        
+        if verbose:
+            print("Requesting data from EPE server...")
+            
+        response = requests.get(url, timeout=60, verify=False)
+        if not response.ok:
+            raise Exception(f"Error getting JSON response: {response.status_code}")
+
+        json_response = response.json()
+        
+        if verbose:
+            print(f"JSON response received: {json.dumps(json_response, indent=2)[:500]}...")
+            
+        if 'results' not in json_response or len(json_response['results']) == 0:
+            raise Exception("Invalid JSON response structure")
+            
+        if 'value' not in json_response['results'][0] or 'url' not in json_response['results'][0]['value']:
+            raise Exception("URL not found in JSON response")
+            
+        file_url = json_response['results'][0]['value']['url']
+        
+        if verbose:
+            print(f"Downloading file from: {file_url}")
+        
+        file_response = requests.get(file_url, stream=True, timeout=60, verify=False)
+        if not file_response.ok:
+            raise Exception(f"Error downloading file: {file_response.status_code}")
+        
+        # Check if content is actually a zip file
+        content = file_response.content
+        if len(content) < 100:
+            if verbose:
+                print(f"Warning: Downloaded content is very small ({len(content)} bytes)")
+                print(f"Content preview: {content[:100]}")
+            
+        # Create a temporary directory to extract the files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if verbose:
+                print(f"Extracting files to temporary directory: {temp_dir}")
+                
+            try:
+                # Extract the zip file
+                with ZipFile(BytesIO(content)) as zip_ref:
+                    zip_ref.extractall(temp_dir)
+                    
+                    if verbose:
+                        print(f"Files in zip: {zip_ref.namelist()}")
+            except Exception as zip_error:
+                if verbose:
+                    print(f"Error extracting zip: {str(zip_error)}")
+                    print(f"Saving content to debug.zip for inspection")
+                    with open("debug.zip", "wb") as f:
+                        f.write(content)
+                raise Exception(f"Failed to extract zip file: {str(zip_error)}")
+            
+            # Find the shapefile
+            all_files = os.listdir(temp_dir)
+            if verbose:
+                print(f"Files in temp directory: {all_files}")
+                
+            shp_files = [f for f in all_files if f.endswith('.shp')]
+            if not shp_files:
+                # Try looking in subdirectories
+                for root, dirs, files in os.walk(temp_dir):
+                    shp_files.extend([os.path.join(root, f) for f in files if f.endswith('.shp')])
+                
+                if not shp_files:
+                    raise Exception("No shapefile found in the downloaded data")
+            
+            # Read the shapefile
+            shp_path = shp_files[0] if os.path.isabs(shp_files[0]) else os.path.join(temp_dir, shp_files[0])
+            if verbose:
+                print(f"Reading shapefile: {shp_path}")
+                
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            gdf = gdf.to_crs(4674)
+            
+            if verbose:
+                print(f"Data loaded successfully with {len(gdf)} records")
+                print(f"Columns: {gdf.columns.tolist()}")
+            
+            if simplified:
+                # Keep only the most relevant columns
+                columns_to_keep = [
+                    'geometry',
+                    'nome',            # Power plant name
+                    'potencia',        # Capacity in MW
+                    'rio',             # River name
+                    'bacia',           # Basin
+                    'sub_bacia',       # Sub-basin
+                    'ano_prev'         # Expected year
+                ]
+                
+                # Filter columns that actually exist in the dataset
+                existing_columns = ['geometry'] + [col for col in columns_to_keep[1:] if col in gdf.columns]
+                if len(existing_columns) <= 1:
+                    if verbose:
+                        print("Warning: No matching columns found for simplified version. Returning all columns.")
+                else:
+                    gdf = gdf[existing_columns]
+    
+    except Exception as e:
+        raise Exception(f"Error downloading or processing planned large hydroelectric power plants data: {str(e)}")
+        
+    return gdf
+
+if __name__ == '__main__':
+    try:
+        uhe_data = read_planned_uhe(verbose=True)
+        print(f"Downloaded planned large hydroelectric power plants data with {len(uhe_data)} records and {len(uhe_data.columns)} columns")
+        
+        # Test simplified version
+        simplified_data = read_planned_uhe(simplified=True)
+        print(f"Simplified data has {len(simplified_data.columns)} columns: {simplified_data.columns.tolist()}")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        import traceback
+        traceback.print_exc()

tunned_geobr/read_processing_facilities.py

@@ -0,0 +1,128 @@
+import geopandas as gpd
+import requests
+import shutil
+import zipfile
+import tempfile
+import warnings
+import os
+from shapely.geometry.point import Point
+
+
+def read_processing_facilities(simplified=False, verbose=False):
+    """Download data of oil and gas processing facilities in Brazil.
+
+    This function downloads and returns data of oil and gas processing facilities
+    in Brazil as a GeoPandas GeoDataFrame. The data comes from EPE (Energy Research Company).
+
+    Parameters
+    ----------
+    simplified : bool, optional
+        If True, returns a simplified version of the dataset with only the most
+        important columns. If False, returns the complete dataset. Default is False.
+    verbose : bool, optional
+        If True, displays detailed messages about the download and processing
+        steps. Default is False.
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        A GeoDataFrame containing data on processing facilities in Brazil.
+
+    Raises
+    ------
+    Exception
+        If the download or processing of the data fails.
+
+    Example
+    -------
+    >>> from tunned_geobr import read_processing_facilities
+    >>> 
+    >>> # Read the data
+    >>> processing_facilities = read_processing_facilities()
+    >>> 
+    >>> # Plot the data
+    >>> processing_facilities.plot()
+    """
+    
+    if verbose:
+        print("Downloading data of processing facilities in Brazil")
+    
+    # Define the URL for the API request
+    url = "https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22Instala%C3%A7%C3%B5es%20de%20Processamento%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+    
+    try:
+        # Make the API request
+        response = requests.get(url)
+        response.raise_for_status()
+        
+        # Parse the JSON response
+        data = response.json()
+        
+        # Extract the URL for the zip file
+        if 'results' in data and len(data['results']) > 0 and 'value' in data['results'][0]:
+            download_url = data['results'][0]['value']['url']
+        else:
+            raise Exception("Failed to extract download URL from API response")
+        
+        # Create a temporary directory to store the downloaded files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Download the zip file
+            zip_path = os.path.join(temp_dir, "processing_facilities.zip")
+            if verbose:
+                print("Downloading zip file")
+            
+            response = requests.get(download_url, stream=True)
+            response.raise_for_status()
+            
+            with open(zip_path, 'wb') as f:
+                response.raw.decode_content = True
+                shutil.copyfileobj(response.raw, f)
+            
+            # Extract the zip file
+            if verbose:
+                print("Extracting files")
+            
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                zip_ref.extractall(temp_dir)
+            
+            # Find the shapefile in the extracted files
+            shp_files = [f for f in os.listdir(temp_dir) if f.endswith('.shp')]
+            
+            if not shp_files:
+                raise Exception("No shapefile found in the downloaded zip file")
+            
+            # Read the shapefile
+            if verbose:
+                print("Reading shapefile")
+            
+            shp_path = os.path.join(temp_dir, shp_files[0])
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            if verbose:
+                print("Converting to SIRGAS 2000 (EPSG:4674)")
+            
+            gdf = gdf.to_crs(epsg=4674)
+            
+            # Simplify the dataset if requested
+            if simplified:
+                if verbose:
+                    print("Simplifying the dataset")
+                
+                # Select only the most important columns
+                # Adjust these columns based on the actual data structure
+                cols_to_keep = ['NOME', 'EMPRESA', 'TIPO', 'CAPACIDADE', 'UF', 'MUNICIPIO', 'geometry']
+                cols_available = [col for col in cols_to_keep if col in gdf.columns]
+                
+                if not cols_available:
+                    warnings.warn("None of the specified columns for simplification are available. Returning the full dataset.")
+                else:
+                    gdf = gdf[cols_available]
+            
+            if verbose:
+                print("Finished processing facilities data")
+            
+            return gdf
+    
+    except Exception as e:
+        raise Exception(f"Failed to download or process processing facilities data: {str(e)}")

tunned_geobr/read_sedimentary_basins.py

@@ -0,0 +1,119 @@
+import geopandas as gpd
+import requests
+import zipfile
+import tempfile
+import os
+import warnings
+import shutil
+
+
+def read_sedimentary_basins(simplified=True, verbose=False):
+    """Download data for Sedimentary Basins in Brazil.
+
+    This function downloads, processes, and returns data for Sedimentary Basins
+    in Brazil as a geopandas GeoDataFrame.
+
+    Parameters
+    ----------
+    simplified : bool, optional
+        If True, returns a simplified version of the dataset with only essential columns.
+        If False, returns the complete dataset with all columns.
+        Default is True.
+    verbose : bool, optional
+        If True, prints detailed information about the data download and processing.
+        Default is False.
+
+    Returns
+    -------
+    geopandas.GeoDataFrame
+        A GeoDataFrame containing Sedimentary Basins data.
+
+    Examples
+    --------
+    >>> # Download Sedimentary Basins data
+    >>> df = read_sedimentary_basins()
+    >>> df.head()
+    """
+
+    url = "https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22Bacias%20Sedimentares%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-9237395.881983705%2C-4650539.310904562%5D%2C%5B-9237395.881983705%2C1219824.4613954136%5D%2C%5B-2349502.3891517334%2C1219824.4613954136%5D%2C%5B-2349502.3891517334%2C-4650539.310904562%5D%2C%5B-9237395.881983705%2C-4650539.310904562%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+
+    if verbose:
+        print("Downloading data...")
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        response_json = response.json()
+
+        if "value" not in response_json or not response_json["value"]:
+            raise ValueError("No data found in the response")
+
+        download_url = response_json["value"]["itemUrl"]
+        
+        if verbose:
+            print(f"Download URL: {download_url}")
+            print("Downloading zip file...")
+
+        zip_response = requests.get(download_url)
+        zip_response.raise_for_status()
+
+        # Create a temporary directory to extract the files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            zip_path = os.path.join(temp_dir, "data.zip")
+            
+            # Save the zip file
+            with open(zip_path, "wb") as f:
+                f.write(zip_response.content)
+            
+            if verbose:
+                print(f"Zip file saved to {zip_path}")
+                print("Extracting files...")
+            
+            # Extract the zip file
+            with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                zip_ref.extractall(temp_dir)
+            
+            # Find the shapefile
+            shp_files = [f for f in os.listdir(temp_dir) if f.endswith(".shp")]
+            
+            if not shp_files:
+                raise FileNotFoundError("No shapefile found in the downloaded zip file")
+            
+            shp_path = os.path.join(temp_dir, shp_files[0])
+            
+            if verbose:
+                print(f"Reading shapefile from {shp_path}")
+            
+            # Read the shapefile
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            gdf = gdf.to_crs(epsg=4674)
+            
+            if simplified:
+                # Select only essential columns
+                if verbose:
+                    print("Simplifying the dataset...")
+                
+                # Identify the essential columns
+                essential_cols = ["geometry"]
+                
+                # Add any other essential columns that exist in the dataset
+                for col in ["NOME", "DESCRICAO", "AREA_KM2"]:
+                    if col in gdf.columns:
+                        essential_cols.append(col)
+                
+                # Select only the essential columns
+                gdf = gdf[essential_cols]
+            
+            return gdf
+    
+    except requests.exceptions.RequestException as e:
+        warnings.warn(f"Error downloading data: {e}")
+        return None
+    except (ValueError, FileNotFoundError, zipfile.BadZipFile) as e:
+        warnings.warn(f"Error processing data: {e}")
+        return None
+    except Exception as e:
+        warnings.warn(f"Unexpected error: {e}")
+        return None