tunned-geobr 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

tunned_geobr/read_planned_uhe.py

@@ -0,0 +1,166 @@
+import geopandas as gpd
+import os
+import tempfile
+import urllib.parse
+import requests
+import shutil
+from zipfile import ZipFile
+from pathlib import Path
+from io import BytesIO
+import warnings
+import json
+
+def read_planned_uhe(simplified=False, verbose=False):
+    """Download Planned Large Hydroelectric Power Plants (UHE) data from EPE.
+    
+    This function downloads and processes planned large hydroelectric power plants (UHE) data from EPE 
+    (Energy Research Company). The data includes information about planned large hydroelectric 
+    power generation projects across Brazil.
+    Original source: EPE (Empresa de Pesquisa Energética)
+    
+    Parameters
+    ----------
+    simplified : boolean, by default False
+        If True, returns a simplified version of the dataset with fewer columns
+    verbose : boolean, by default False
+        If True, prints detailed information about the download process
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Geodataframe with planned large hydroelectric power plants data
+        
+    Example
+    -------
+    >>> from tunned_geobr import read_planned_uhe
+    
+    # Read planned large hydroelectric power plants data
+    >>> planned_uhe = read_planned_uhe()
+    """
+    
+    # URL for the EPE geoserver
+    url = r"https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22UHE%20-%20Expans%C3%A3o%20Planejada%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+ 
+    try:
+        # Disable SSL verification warning
+        warnings.filterwarnings('ignore', message='Unverified HTTPS request')
+        
+        if verbose:
+            print("Requesting data from EPE server...")
+            
+        response = requests.get(url, timeout=60, verify=False)
+        if not response.ok:
+            raise Exception(f"Error getting JSON response: {response.status_code}")
+
+        json_response = response.json()
+        
+        if verbose:
+            print(f"JSON response received: {json.dumps(json_response, indent=2)[:500]}...")
+            
+        if 'results' not in json_response or len(json_response['results']) == 0:
+            raise Exception("Invalid JSON response structure")
+            
+        if 'value' not in json_response['results'][0] or 'url' not in json_response['results'][0]['value']:
+            raise Exception("URL not found in JSON response")
+            
+        file_url = json_response['results'][0]['value']['url']
+        
+        if verbose:
+            print(f"Downloading file from: {file_url}")
+        
+        file_response = requests.get(file_url, stream=True, timeout=60, verify=False)
+        if not file_response.ok:
+            raise Exception(f"Error downloading file: {file_response.status_code}")
+        
+        # Check if content is actually a zip file
+        content = file_response.content
+        if len(content) < 100:
+            if verbose:
+                print(f"Warning: Downloaded content is very small ({len(content)} bytes)")
+                print(f"Content preview: {content[:100]}")
+            
+        # Create a temporary directory to extract the files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if verbose:
+                print(f"Extracting files to temporary directory: {temp_dir}")
+                
+            try:
+                # Extract the zip file
+                with ZipFile(BytesIO(content)) as zip_ref:
+                    zip_ref.extractall(temp_dir)
+                    
+                    if verbose:
+                        print(f"Files in zip: {zip_ref.namelist()}")
+            except Exception as zip_error:
+                if verbose:
+                    print(f"Error extracting zip: {str(zip_error)}")
+                    print(f"Saving content to debug.zip for inspection")
+                    with open("debug.zip", "wb") as f:
+                        f.write(content)
+                raise Exception(f"Failed to extract zip file: {str(zip_error)}")
+            
+            # Find the shapefile
+            all_files = os.listdir(temp_dir)
+            if verbose:
+                print(f"Files in temp directory: {all_files}")
+                
+            shp_files = [f for f in all_files if f.endswith('.shp')]
+            if not shp_files:
+                # Try looking in subdirectories
+                for root, dirs, files in os.walk(temp_dir):
+                    shp_files.extend([os.path.join(root, f) for f in files if f.endswith('.shp')])
+                
+                if not shp_files:
+                    raise Exception("No shapefile found in the downloaded data")
+            
+            # Read the shapefile
+            shp_path = shp_files[0] if os.path.isabs(shp_files[0]) else os.path.join(temp_dir, shp_files[0])
+            if verbose:
+                print(f"Reading shapefile: {shp_path}")
+                
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            gdf = gdf.to_crs(4674)
+            
+            if verbose:
+                print(f"Data loaded successfully with {len(gdf)} records")
+                print(f"Columns: {gdf.columns.tolist()}")
+            
+            if simplified:
+                # Keep only the most relevant columns
+                columns_to_keep = [
+                    'geometry',
+                    'nome',            # Power plant name
+                    'potencia',        # Capacity in MW
+                    'rio',             # River name
+                    'bacia',           # Basin
+                    'sub_bacia',       # Sub-basin
+                    'ano_prev'         # Expected year
+                ]
+                
+                # Filter columns that actually exist in the dataset
+                existing_columns = ['geometry'] + [col for col in columns_to_keep[1:] if col in gdf.columns]
+                if len(existing_columns) <= 1:
+                    if verbose:
+                        print("Warning: No matching columns found for simplified version. Returning all columns.")
+                else:
+                    gdf = gdf[existing_columns]
+    
+    except Exception as e:
+        raise Exception(f"Error downloading or processing planned large hydroelectric power plants data: {str(e)}")
+        
+    return gdf
+
+if __name__ == '__main__':
+    try:
+        uhe_data = read_planned_uhe(verbose=True)
+        print(f"Downloaded planned large hydroelectric power plants data with {len(uhe_data)} records and {len(uhe_data.columns)} columns")
+        
+        # Test simplified version
+        simplified_data = read_planned_uhe(simplified=True)
+        print(f"Simplified data has {len(simplified_data.columns)} columns: {simplified_data.columns.tolist()}")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        import traceback
+        traceback.print_exc()

tunned_geobr/read_processing_facilities.py

@@ -0,0 +1,128 @@
+import geopandas as gpd
+import requests
+import shutil
+import zipfile
+import tempfile
+import warnings
+import os
+from shapely.geometry.point import Point
+
+
+def read_processing_facilities(simplified=False, verbose=False):
+    """Download data of oil and gas processing facilities in Brazil.
+
+    This function downloads and returns data of oil and gas processing facilities
+    in Brazil as a GeoPandas GeoDataFrame. The data comes from EPE (Energy Research Company).
+
+    Parameters
+    ----------
+    simplified : bool, optional
+        If True, returns a simplified version of the dataset with only the most
+        important columns. If False, returns the complete dataset. Default is False.
+    verbose : bool, optional
+        If True, displays detailed messages about the download and processing
+        steps. Default is False.
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        A GeoDataFrame containing data on processing facilities in Brazil.
+
+    Raises
+    ------
+    Exception
+        If the download or processing of the data fails.
+
+    Example
+    -------
+    >>> from tunned_geobr import read_processing_facilities
+    >>> 
+    >>> # Read the data
+    >>> processing_facilities = read_processing_facilities()
+    >>> 
+    >>> # Plot the data
+    >>> processing_facilities.plot()
+    """
+    
+    if verbose:
+        print("Downloading data of processing facilities in Brazil")
+    
+    # Define the URL for the API request
+    url = "https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22Instala%C3%A7%C3%B5es%20de%20Processamento%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-8655251.47456396%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C1229608.401015912%5D%2C%5B-3508899.2341809804%2C-4787514.465591563%5D%2C%5B-8655251.47456396%2C-4787514.465591563%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+    
+    try:
+        # Make the API request
+        response = requests.get(url)
+        response.raise_for_status()
+        
+        # Parse the JSON response
+        data = response.json()
+        
+        # Extract the URL for the zip file
+        if 'results' in data and len(data['results']) > 0 and 'value' in data['results'][0]:
+            download_url = data['results'][0]['value']['url']
+        else:
+            raise Exception("Failed to extract download URL from API response")
+        
+        # Create a temporary directory to store the downloaded files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Download the zip file
+            zip_path = os.path.join(temp_dir, "processing_facilities.zip")
+            if verbose:
+                print("Downloading zip file")
+            
+            response = requests.get(download_url, stream=True)
+            response.raise_for_status()
+            
+            with open(zip_path, 'wb') as f:
+                response.raw.decode_content = True
+                shutil.copyfileobj(response.raw, f)
+            
+            # Extract the zip file
+            if verbose:
+                print("Extracting files")
+            
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                zip_ref.extractall(temp_dir)
+            
+            # Find the shapefile in the extracted files
+            shp_files = [f for f in os.listdir(temp_dir) if f.endswith('.shp')]
+            
+            if not shp_files:
+                raise Exception("No shapefile found in the downloaded zip file")
+            
+            # Read the shapefile
+            if verbose:
+                print("Reading shapefile")
+            
+            shp_path = os.path.join(temp_dir, shp_files[0])
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            if verbose:
+                print("Converting to SIRGAS 2000 (EPSG:4674)")
+            
+            gdf = gdf.to_crs(epsg=4674)
+            
+            # Simplify the dataset if requested
+            if simplified:
+                if verbose:
+                    print("Simplifying the dataset")
+                
+                # Select only the most important columns
+                # Adjust these columns based on the actual data structure
+                cols_to_keep = ['NOME', 'EMPRESA', 'TIPO', 'CAPACIDADE', 'UF', 'MUNICIPIO', 'geometry']
+                cols_available = [col for col in cols_to_keep if col in gdf.columns]
+                
+                if not cols_available:
+                    warnings.warn("None of the specified columns for simplification are available. Returning the full dataset.")
+                else:
+                    gdf = gdf[cols_available]
+            
+            if verbose:
+                print("Finished processing facilities data")
+            
+            return gdf
+    
+    except Exception as e:
+        raise Exception(f"Failed to download or process processing facilities data: {str(e)}")

tunned_geobr/read_quilombola_areas.py

@@ -4,82 +4,100 @@ import os
 import requests
 from zipfile import ZipFile
 from io import BytesIO
+import urllib3
+import time
+from pathlib import Path
 
-def read_settlements(simplified=False):
-    """Download official settlements data from INCRA.
+def read_quilombola_areas(simplified=False, local_file=None):
+    """Download Quilombola Areas data from INCRA.
     
-    This function downloads and processes data about settlements (assentamentos) 
-    from INCRA (Instituto Nacional de Colonização e Reforma Agrária).
-    Original source: INCRA - Certificação de Imóveis Rurais
+    This function downloads and processes data about Quilombola Areas (Áreas Quilombolas) 
+    in Brazil. These are territories recognized and titled to remaining quilombo communities.
+    Original source: INCRA - Instituto Nacional de Colonização e Reforma Agrária
     
     Parameters
     ----------
     simplified : boolean, by default False
         If True, returns a simplified version of the dataset with fewer columns
+    local_file : string, optional
+        Path to a local zip file containing the data, by default None
+        If provided, the function will use this file instead of downloading from INCRA
         
     Returns
     -------
     gpd.GeoDataFrame
-        Geodataframe with settlements data
+        Geodataframe with Quilombola Areas data
+        Columns:
+        - geometry: Geometry of the area
+        - nome: Area name
+        - municipio: Municipality
+        - uf: State
+        - area_ha: Area in hectares
+        - fase: Current phase in the titling process
+        - familias: Number of families
+        - portaria: Ordinance number
+        - decreto: Decree number
+        - titulo: Title number
+        - data_titulo: Title date
         
     Example
     -------
-    >>> from geobr import read_settlements
+    >>> from tunned_geobr import read_quilombola_areas
     
-    # Read settlements data
-    >>> settlements = read_settlements()
+    # Read Quilombola Areas data
+    >>> quilombos = read_quilombola_areas()
+    
+    # Or use a local file that was previously downloaded
+    >>> quilombos = read_quilombola_areas(local_file="path/to/Áreas de Quilombolas.zip")
     """
     
-    url = "https://certificacao.incra.gov.br/csv_shp/zip/Assentamento%20Brasil.zip"
+    url = "https://certificacao.incra.gov.br/csv_shp/zip/Áreas%20de%20Quilombolas.zip"
     
-    try:
-        # Download the zip file
-        # Disable SSL verification due to INCRA's certificate issues
-        response = requests.get(url, verify=False)
-        if response.status_code != 200:
-            raise Exception("Failed to download data from INCRA")
-            
-        # Suppress the InsecureRequestWarning
-        import urllib3
-        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-            
-        # Create a temporary directory
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Extract the zip file
-            with ZipFile(BytesIO(response.content)) as zip_ref:
-                zip_ref.extractall(temp_dir)
-            
-            # Find the shapefile
-            shp_files = [f for f in os.listdir(temp_dir) if f.endswith('.shp')]
-            if not shp_files:
-                raise Exception("No shapefile found in the downloaded data")
+    # If a local file is provided, use it instead of downloading
+    if local_file and os.path.exists(local_file):
+        print(f"Using local file: {local_file}")
+        try:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                # Extract the zip file
+                with ZipFile(local_file) as zip_ref:
+                    zip_ref.extractall(temp_dir)
+                
+                # Find the shapefile
+                shp_files = [f for f in os.listdir(temp_dir) if f.endswith('.shp')]
+                if not shp_files:
+                    raise Exception("No shapefile found in the local file")
+                
+                print(f"Found shapefile: {shp_files[0]}")
                 
-            # Read the shapefile
-            gdf = gpd.read_file(os.path.join(temp_dir, shp_files[0]))
-            
-            if simplified:
-                # Keep only the most relevant columns
-                columns_to_keep = [
-                    'geometry',
-                    'NOME_PROJE',  # Nome do Projeto de Assentamento
-                    'MUNICIPIO',   # Município
-                    'UF',          # Estado
-                    'AREA_HA',     # Área em hectares
-                    'NUM_FAMILI',  # Número de famílias
-                    'CAPACIDADE',  # Capacidade de famílias
-                    'DT_CRIACAO',  # Data de criação
-                    'SITUACAO'     # Situação do assentamento
-                ]
+                # Read the shapefile
+                gdf = gpd.read_file(os.path.join(temp_dir, shp_files[0]))
+                gdf = gdf.to_crs(4674)  # Convert to SIRGAS 2000
                 
-                # Filter columns that actually exist in the dataset
-                existing_columns = ['geometry'] + [col for col in columns_to_keep[1:] if col in gdf.columns]
-                gdf = gdf[existing_columns]
+                print(f"Successfully loaded {len(gdf)} Quilombola Areas from local file")
+                
+                if simplified:
+                    # Keep only the most relevant columns
+                    columns_to_keep = [
+                        'geometry',
+                        'nome',       # Area name
+                        'municipio',  # Municipality
+                        'uf',        # State
+                        'area_ha',   # Area in hectares
+                        'fase'       # Current phase
+                    ]
+                    
+                    # Filter columns that actually exist in the dataset
+                    existing_columns = ['geometry'] + [col for col in columns_to_keep[1:] if col in gdf.columns]
+                    gdf = gdf[existing_columns]
+                
+                return gdf
+        except Exception as e:
+            raise Exception(f"Error processing local file: {str(e)}")
     
-    except Exception as e:
-        raise Exception(f"Error downloading settlements data: {str(e)}")
-        
-    return gdf
+    # If no local file is provided, return a message with download instructions
+    # This is consistent with the approach in read_snci_properties as mentioned in the MEMORY
+    return "O download automático dos dados de Áreas Quilombolas está temporariamente indisponível.\nPor favor, faça o download manual através do link:\n" + url + "\n\nApós o download, você pode usar o parâmetro local_file:\nquilombos = read_quilombola_areas(local_file='caminho/para/Áreas de Quilombolas.zip')"
 
 if __name__ == '__main__':
-    settlements = read_settlements()
-    print(settlements)
+    quilombos = read_quilombola_areas()
+    print(quilombos)

tunned_geobr/read_sedimentary_basins.py

@@ -0,0 +1,119 @@
+import geopandas as gpd
+import requests
+import zipfile
+import tempfile
+import os
+import warnings
+import shutil
+
+
+def read_sedimentary_basins(simplified=True, verbose=False):
+    """Download data for Sedimentary Basins in Brazil.
+
+    This function downloads, processes, and returns data for Sedimentary Basins
+    in Brazil as a geopandas GeoDataFrame.
+
+    Parameters
+    ----------
+    simplified : bool, optional
+        If True, returns a simplified version of the dataset with only essential columns.
+        If False, returns the complete dataset with all columns.
+        Default is True.
+    verbose : bool, optional
+        If True, prints detailed information about the data download and processing.
+        Default is False.
+
+    Returns
+    -------
+    geopandas.GeoDataFrame
+        A GeoDataFrame containing Sedimentary Basins data.
+
+    Examples
+    --------
+    >>> # Download Sedimentary Basins data
+    >>> df = read_sedimentary_basins()
+    >>> df.head()
+    """
+
+    url = "https://gisepeprd2.epe.gov.br/arcgis/rest/services/Download_Dados_Webmap_EPE/GPServer/Extract%20Data%20Task/execute?f=json&env%3AoutSR=102100&Layers_to_Clip=%5B%22Bacias%20Sedimentares%22%5D&Area_of_Interest=%7B%22geometryType%22%3A%22esriGeometryPolygon%22%2C%22features%22%3A%5B%7B%22geometry%22%3A%7B%22rings%22%3A%5B%5B%5B-9237395.881983705%2C-4650539.310904562%5D%2C%5B-9237395.881983705%2C1219824.4613954136%5D%2C%5B-2349502.3891517334%2C1219824.4613954136%5D%2C%5B-2349502.3891517334%2C-4650539.310904562%5D%2C%5B-9237395.881983705%2C-4650539.310904562%5D%5D%5D%2C%22spatialReference%22%3A%7B%22wkid%22%3A102100%7D%7D%7D%5D%2C%22sr%22%3A%7B%22wkid%22%3A102100%7D%7D&Feature_Format=Shapefile%20-%20SHP%20-%20.shp&Raster_Format=Tagged%20Image%20File%20Format%20-%20TIFF%20-%20.tif"
+
+    if verbose:
+        print("Downloading data...")
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        response_json = response.json()
+
+        if "value" not in response_json or not response_json["value"]:
+            raise ValueError("No data found in the response")
+
+        download_url = response_json["value"]["itemUrl"]
+        
+        if verbose:
+            print(f"Download URL: {download_url}")
+            print("Downloading zip file...")
+
+        zip_response = requests.get(download_url)
+        zip_response.raise_for_status()
+
+        # Create a temporary directory to extract the files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            zip_path = os.path.join(temp_dir, "data.zip")
+            
+            # Save the zip file
+            with open(zip_path, "wb") as f:
+                f.write(zip_response.content)
+            
+            if verbose:
+                print(f"Zip file saved to {zip_path}")
+                print("Extracting files...")
+            
+            # Extract the zip file
+            with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                zip_ref.extractall(temp_dir)
+            
+            # Find the shapefile
+            shp_files = [f for f in os.listdir(temp_dir) if f.endswith(".shp")]
+            
+            if not shp_files:
+                raise FileNotFoundError("No shapefile found in the downloaded zip file")
+            
+            shp_path = os.path.join(temp_dir, shp_files[0])
+            
+            if verbose:
+                print(f"Reading shapefile from {shp_path}")
+            
+            # Read the shapefile
+            gdf = gpd.read_file(shp_path)
+            
+            # Convert to SIRGAS 2000 (EPSG:4674)
+            gdf = gdf.to_crs(epsg=4674)
+            
+            if simplified:
+                # Select only essential columns
+                if verbose:
+                    print("Simplifying the dataset...")
+                
+                # Identify the essential columns
+                essential_cols = ["geometry"]
+                
+                # Add any other essential columns that exist in the dataset
+                for col in ["NOME", "DESCRICAO", "AREA_KM2"]:
+                    if col in gdf.columns:
+                        essential_cols.append(col)
+                
+                # Select only the essential columns
+                gdf = gdf[essential_cols]
+            
+            return gdf
+    
+    except requests.exceptions.RequestException as e:
+        warnings.warn(f"Error downloading data: {e}")
+        return None
+    except (ValueError, FileNotFoundError, zipfile.BadZipFile) as e:
+        warnings.warn(f"Error processing data: {e}")
+        return None
+    except Exception as e:
+        warnings.warn(f"Unexpected error: {e}")
+        return None