tunned-geobr 0.1.0__py3-none-any.whl

tunned_geobr/read_weighting_area.py

@@ -0,0 +1,74 @@
+from cursed_geobr.utils import select_metadata, download_gpkg
+
+
+def read_weighting_area(
+    code_weighting="all", year=2010, simplified=True, verbose=False
+):
+    """Download shape files of Census Weighting Areas (area de ponderacao) of the Brazilian Population Census.
+
+     Only 2010 data is currently available.
+
+    Parameters
+    ----------
+    code_weighting:
+        The 7-digit code of a Municipality. If the two-digit code or a two-letter uppercase abbreviation of
+        a state is passed, (e.g. 33 or "RJ") the function will load all weighting areas of that state.
+        If code_weighting="all", all weighting areas of the country are loaded.
+    year : int, optional
+        Year of the data, by default 2010
+    simplified: boolean, by default True
+        Data 'type', indicating whether the function returns the 'original' dataset
+        with high resolution or a dataset with 'simplified' borders (Default)
+    verbose : bool, optional
+        by default False
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Metadata and geopackage of selected states
+
+    Raises
+    ------
+    Exception
+        If parameters are not found or not well defined
+
+    Example
+    -------
+    >>> from cursed_geobr import read_weighting_area
+
+    # Read specific state at a given year
+    >>> df = read_weighting_area(year=2010)
+    """
+
+    metadata = select_metadata("weighting_area", year=year, simplified=simplified)
+
+    if code_weighting == "all":
+
+        if verbose:
+            print("Loading data for the whole country. This might take a few minutes.")
+
+        return download_gpkg(metadata)
+
+    metadata = metadata[
+        metadata[["code", "code_abbrev"]].apply(
+            lambda x: str(code_weighting)[:2] in str(x["code"])
+            or str(code_weighting)[:2]  # if number e.g. 12
+            in str(x["code_abbrev"]),  # if UF e.g. RO
+            1,
+        )
+    ]
+
+    if not len(metadata):
+        raise Exception("Invalid Value to argument code_weighting.")
+
+    gdf = download_gpkg(metadata)
+
+    if len(str(code_weighting)) == 2:
+        return gdf
+
+    elif code_weighting in gdf["code_muni"].tolist():
+        return gdf.query(f"code_muni == {code_weighting}")
+
+    elif code_weighting in gdf["code_weighting"].tolist():
+        return gdf.query(f"code_weighting == {code_weighting}")
+    return gdf

tunned_geobr/utils.py

@@ -0,0 +1,326 @@
+import os
+from functools import lru_cache
+from urllib.error import HTTPError
+import tempfile
+
+import geopandas as gpd
+import pandas as pd
+import requests
+import unicodedata
+from io import StringIO
+
+from cursed_geobr.constants import DataTypes
+
+MIRRORS = ["https://github.com/ipeaGIT/geobr/releases/download/v1.7.0/"]
+
+
+def _get_unique_values(_df, column):
+
+    return ", ".join([str(i) for i in _df[column].unique()])
+
+
+def url_solver(url):
+
+    file_id = url.split("/")[-1]
+    urls = [url] + [mirror + file_id for mirror in MIRRORS]
+
+    for url in urls:
+
+        try:
+            response = requests.get(url)
+            if response.status_code == 200:
+                return response
+        except:
+            continue
+
+    raise ConnectionError(
+        "No mirrors are active. Please report to https://github.com/ipeaGIT/geobr/issues"
+    )
+
+
+@lru_cache(maxsize=124)
+def download_metadata(
+    url="http://www.ipea.gov.br/geobr/metadata/metadata_1.7.0_gpkg.csv",
+):
+    """Support function to download metadata internally used in geobr.
+
+    It caches the metadata file to avoid reloading it in the same session.
+
+    Parameters
+    ----------
+    url : str, optional
+        Metadata url, by default 'http://www.ipea.gov.br/geobr/metadata/metadata_1.7.0_gpkg.csv'
+
+    Returns
+    -------
+    pd.DataFrame
+        Table with all metadata of geopackages
+
+    Raises
+    ------
+    Exception
+        Leads user to Github issue page if metadata url is not found
+
+    Examples
+    --------
+    >>> metadata = download_metadata()
+    >>> metadata.head(1)
+                  geo  year code                                      download_path      code_abbrev
+    0  amazonia_legal  2012   am  http://www.ipea.gov.br/geobr/data_gpkg/amazoni...  amazonia_legal
+    """
+
+    try:
+        return pd.read_csv(StringIO(url_solver(url).text))
+
+    except HTTPError:
+        raise Exception(
+            "Perhaps this is an internet connection problem."
+            "If this is not a connection problem in your network, "
+            " please try geobr again in a few minutes. "
+            "Please report to https://github.com/ipeaGIT/geobr/issues"
+        )
+
+
+def select_year(metadata, year):
+    """Apply year to metadata and checks its existence.
+
+    If it do not exist, raises an informative error.
+
+    Parameters
+    ----------
+    metadata : pd.DataFrame
+        Filtered metadata table
+    year : int
+        Year selected by user
+
+    Returns
+    -------
+    pd.DataFrame
+        Filtered dataframe by year.
+
+    Raises
+    ------
+    Exception
+        If year does not exists, raises exception with available years.
+    """
+
+    if year is None:
+        year = max(metadata["year"])
+
+    elif year not in list(metadata["year"]):
+
+        years = ", ".join([str(i) for i in metadata["year"].unique()])
+
+        raise Exception(
+            "Error: Invalid Value to argument 'year/date'. "
+            "It must be one of the following: "
+            f'{_get_unique_values(metadata, "year")}'
+        )
+
+    return metadata.query(f"year == {year}")
+
+
+def select_simplified(metadata, simplified):
+    """Filter metadata by data type. It can be simplified or normal.
+    If 'simplified' is True, it returns a simplified version of the shapefiles.
+    'normal' returns the complete version. Usually, the complete version
+    if heavier than the simplified, demanding more resources.
+
+    Parameters
+    ----------
+    metadata : pd.DataFrame
+        Filtered metadata table
+    simplified : boolean
+        Data type, either True for 'simplified' or False for 'normal'
+
+    Returns
+    -------
+    pd.DataFrame
+        Filtered metadata table by type
+
+    """
+
+    if simplified:
+        return metadata[metadata["download_path"].str.contains("simplified")]
+
+    else:
+        return metadata[~metadata["download_path"].str.contains("simplified")]
+
+
+@lru_cache(maxsize=1240)
+def load_gpkg(url):
+    """Internal function to donwload and convert to geopandas one url.
+
+    It caches url result for the active session.
+
+    Parameters
+    ----------
+    url : str
+        Address with gpkg
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+         Table with metadata and shapefiles contained in url.
+    """
+
+    try:
+        content = url_solver(url).content
+
+    except Exception as e:
+
+        raise Exception(
+            "Some internal url is broken."
+            "Please report to https://github.com/ipeaGIT/geobr/issues"
+        ) from e
+
+    # This below does not work in Windows -- see the Docs
+    # Whether the name can be used to open the file a second time, while the named temporary file is still open,
+    # varies across platforms (it can be so used on Unix; it cannot on Windows NT or later).
+    # https://docs.python.org/2/library/tempfile.html
+
+    # Create a temporary file with .gpkg extension that is automatically deleted when closed
+    with tempfile.NamedTemporaryFile(suffix=".gpkg", delete=False) as fp:
+        fp.write(content)
+        # Need to close file before reading on Windows
+        fp.close()
+        gdf = gpd.read_file(fp.name)
+        # Clean up temp file
+        os.unlink(fp.name)
+
+    return gdf
+
+
+def enforce_types(df):
+    """Enforce correct datatypes according to DataTypes constant
+
+    Parameters
+    ----------
+    df : gpd.GeoDataFrame
+        Raw output data
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Output data with correct types
+    """
+
+    for column in df.columns:
+
+        if column in DataTypes.__members__.keys():
+
+            df[column] = df[column].astype(DataTypes[column].value)
+
+    return df
+
+
+def download_gpkg(metadata):
+    """Generalizes gpkg dowload and conversion to geopandas
+    for one or many urls
+
+    Parameters
+    ----------
+    metadata : pd.DataFrame
+        Filtered metadata
+
+    Returns
+    -------
+    gpd.GeoDataFrame
+        Table with metadata and shapefiles contained in urls.
+    """
+
+    urls = metadata["download_path"].tolist()
+
+    gpkgs = [load_gpkg(url) for url in urls]
+
+    df = gpd.GeoDataFrame(pd.concat(gpkgs, ignore_index=True))
+
+    df = enforce_types(df)
+
+    return df
+
+
+def select_metadata(geo, simplified=None, year=False):
+    """Downloads and filters metadata given `geo`, `simplified` and `year`.
+
+    Parameters
+    ----------
+    geo : str
+        Shapefile category. I.e: state, biome, etc...
+    simplified : boolean
+        `simplified` or `normal` shapefiles
+    year : int
+        Year of the data
+
+    Returns
+    -------
+    pd.DataFrame
+        Filtered metadata
+
+    Raises
+    ------
+    Exception
+        if a parameter is not found in metadata table
+    """
+
+    # Get metadata with data addresses
+    metadata = download_metadata()
+
+    if len(metadata.query(f'geo == "{geo}"')) == 0:
+        raise Exception(
+            f"The `geo` argument {geo} does not exist."
+            "Please, use one of the following:"
+            f'{_get_unique_values(metadata, "geo")}'
+        )
+
+    # Select geo
+    metadata = metadata.query(f'geo == "{geo}"')
+
+    if simplified is not None:
+        # Select data type
+        metadata = select_simplified(metadata, simplified)
+
+    if year != False:
+        # Verify year input
+        metadata = select_year(metadata, year)
+
+    return metadata
+
+
+def change_type_list(lst, astype=str):
+    return [astype(l) for l in lst]
+
+
+def test_options(choosen, name, allowed=None, not_allowed=None):
+
+    if allowed is not None:
+        if choosen not in allowed:
+            raise Exception(
+                f"Invalid value to argument '{name}'. "
+                f"It must be either {' or '.join(change_type_list(allowed))}"
+            )
+
+    if not_allowed is not None:
+        if choosen in not_allowed:
+            raise Exception(
+                f"Invalid value to argument '{name}'. "
+                f"It cannot be {' or '.join(change_type_list(allowed))}"
+            )
+
+
+def strip_accents(text):
+    """
+    Strip accents from input String.
+
+    Parameters
+    ----------
+    text: str, The input string
+
+    Returns
+    ----------
+    str, The processed string
+    """
+    text = unicodedata.normalize("NFD", text)
+    text = text.encode("ascii", "ignore")
+    text = text.decode("utf-8")
+    return str(text)

tunned_geobr-0.1.0.dist-info/METADATA

@@ -0,0 +1,103 @@
+Metadata-Version: 2.1
+Name: tunned-geobr
+Version: 0.1.0
+Summary: Fork personalizado do geobr com funcionalidades extras como download de dados da ANM
+Author: Anderson Stolfi
+License: MIT
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Topic :: Scientific/Engineering :: GIS
+Classifier: Topic :: Scientific/Engineering :: Visualization
+Classifier: Programming Language :: Python
+Project-URL: homepage, https://github.com/popogis24/tunned_geobr
+Project-URL: repository, https://github.com/popogis24/tunned_geobr
+Requires-Python: <4.0,>=3.9
+Requires-Dist: geopandas<=1.1,>=1.0.0
+Requires-Dist: shapely<=2.1.0,>=1.7.0
+Requires-Dist: requests<3.0.0,>=2.25.1
+Requires-Dist: lxml<6.0.0,>=5.1.0
+Requires-Dist: html5lib==1.1
+Requires-Dist: geobr<0.3.0,>=0.2.2
+Requires-Dist: patool>=1.15.0
+Description-Content-Type: text/markdown
+
+# geobr: Download Official Spatial Data Sets of Brazil 
+
+<img align="right" src="https://github.com/ipeaGIT/geobr/blob/master/r-package/man/figures/geobr_logo_b.png?raw=true" alt="logo" width="140"> 
+<img align="right" src="https://github.com/ipeaGIT/geobr/blob/master/r-package/man/figures/geobr_logo_y.png?raw=true" alt="logo" width="140">
+<p align="justify">geobr is a computational package to download official spatial data sets of Brazil. The package includes a wide range of geospatial data in geopackage format (like shapefiles but better), available at various geographic scales and for various years with harmonized attributes, projection and topology (see detailed list of available data sets below). </p> 
+
+## [READ FULL DOCS](https://github.com/ipeaGIT/geobr)
+
+## Contribute
+
+To start the development environment run
+
+```sh
+uv sync
+```
+
+Test with
+
+`uv run pytest -n auto`
+
+You can use a helper to translate a function from R.
+If you want to add `read_biomes`, just run
+
+```sh
+uv run python helpers/translate_from_R.py read_biomes
+```
+
+It will scrape the original R function to get documentation and metadata.
+It adds:
+- default year
+- function name
+- documentation one liner
+- larger documentation
+- very basic tests
+
+! Be aware that if the function that you are adding is more complicated than the template. So, always double check !
+
+
+
+
+## Translation Status
+
+| Function                  | Translated? | Easy? |
+| ------------------------- | ----------- | ----- |
+| read_amazon               | Yes         | Super |
+| read_biomes               | Yes         | Super |
+| read_census_tract         | Yes         | No    |
+| read_comparable_areas     | Yes         | Yes   |
+| read_conservation_units   | Yes         | Super |
+| read_country              | Yes         | Super |
+| read_disaster_risk_area   | Yes         | Super |
+| read_health_facilities    | Yes         | Super |
+| read_health_region        | Yes         | Super |
+| read_immediate_region     | Yes         | Yes   |
+| read_indigenous_land      | Yes         | Super |
+| read_intermediate_region  | Yes         | Yes   |
+| read_meso_region          | Yes         | No    |
+| read_metro_area           | Yes         | Super |
+| read_micro_region         | Yes         | No    |
+| read_municipal_seat       | Yes         | Super |
+| read_municipality         | Yes         | No    |
+| read_region               | Yes         | Super |
+| read_semiarid             | Yes         | Super |
+| read_state                | Yes         | Super |
+| read_statistical_grid     | Yes         | No    |
+| read_urban_area           | Yes         | Super |
+| read_urban_concentrations | Yes         | Super |
+| read_weighting_area       | Yes         | No    |
+| list_geobr                | Yes         | Yes   |
+| lookup_muni               | Yes         | No    |
+| read_neighborhood         | Yes         | Yes   |
+
+
+# Release new version
+
+```
+poetry version [patch|minor|major]
+poetry publish --build
+```

tunned_geobr-0.1.0.dist-info/RECORD

@@ -0,0 +1,46 @@
+tunned_geobr-0.1.0.dist-info/METADATA,sha256=Rq_0FSHR6SbXupwxbHI_3poqz-sToBb_JurPUy9Z-9Y,3896
+tunned_geobr-0.1.0.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+tunned_geobr-0.1.0.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+tunned_geobr-0.1.0.dist-info/licenses/LICENSE.txt,sha256=mECZRcbde3HssOKe1Co4zgqBLGVN0OWpTsEy3LIbcRA,75
+tunned_geobr/__init__.py,sha256=zeKiOihNUT5yqeOzWo84E_rLZSZOwxgyEsrOuAdqT4M,1872
+tunned_geobr/constants.py,sha256=ZHj4pKtrxoUMFFgw-4ikuFcCkxEjzIbWL_gzhutGDB4,262
+tunned_geobr/data/grid_state_correspondence_table.csv,sha256=FpkBuX_-lRXQ1yBrQODxQgG9oha9Fd8A8zGKfdsDAmk,2660
+tunned_geobr/list_geobr.py,sha256=uIH11FOltrcjIQOqFk6uHgHj2moCWH_0vWyxbMj-xtA,1252
+tunned_geobr/lookup_muni.py,sha256=ny1zU4i6OagvL4Mrc6XQWPgn2RrJa_mXlKXh81oVYsM,3462
+tunned_geobr/read_amazon.py,sha256=CpUDhNEeZPAXHEuCUl0fsKi6KyeVBFOvZrFAcLe8OEg,1299
+tunned_geobr/read_amazon_ibas.py,sha256=RtOo5wPfc26S2HYJCLylNCPM5cHBOLGTP4uKEtGC3Bw,3500
+tunned_geobr/read_atlantic_forest_ibas.py,sha256=GjaNy8bKMOYRA4G5IR7gfmyAGm8EltojqdqBHZDOg7U,3577
+tunned_geobr/read_biomes.py,sha256=5W2toakngPWG0V7c9QP_xon__38gIkJr2xrKc2mYaN0,1351
+tunned_geobr/read_census_tract.py,sha256=lFIlgDqsz6zkiVWEBE97SlxgdmxuI6_fgqgWhaXcgYg,3272
+tunned_geobr/read_climate_aggressiveness.py,sha256=r7dNvFQ7mo0B5PUWEv28x4aYAD-jF6JiSV7Oydq6N-0,2811
+tunned_geobr/read_comparable_areas.py,sha256=NUyHfGaKoqNBfQUID2uZc-CrZLTxGN_5gJog-I7XZS0,2122
+tunned_geobr/read_conservation_units.py,sha256=-RYV3x06LZDqQKAtmiDLwEV07TxKxCbXI4mKENxJZbk,1393
+tunned_geobr/read_country.py,sha256=wDfG95wFz16iy0WpSxyrezO3hqr3v5xJRK94pDJAneM,1371
+tunned_geobr/read_disaster_risk_area.py,sha256=r6Ccf8yr-zr2IZSPc5jHNsECvesnlG0Yfek1Q5-S5Rw,1859
+tunned_geobr/read_geology.py,sha256=q6o3E8DQ5NBc2wJ9rJfGLN5qSI9_0vQG48xil-f7jzM,2773
+tunned_geobr/read_geomorphology.py,sha256=h2Si0OXHBBna1eMN9LzgiV5IttRyzcnCSe2tqx6frYk,2850
+tunned_geobr/read_health_facilities.py,sha256=ujfg6nbuoefUvhGV69sETKXgog90s6r3wwU8JQa-jUM,2026
+tunned_geobr/read_health_region.py,sha256=3j-Y4NTTrnUuTpFWhWfJDtsfmlRzJBRBv3n9QCvOEtU,1846
+tunned_geobr/read_immediate_region.py,sha256=nIDFssnaPzmvAPYrNuRhLIqczampQct4l8JaoWv90L0,2568
+tunned_geobr/read_indigenous_land.py,sha256=ByE_VUwHhVf1V-rOBzuyCmwKnHdpk2AcSWZBQ3UsWQo,1473
+tunned_geobr/read_intermediate_region.py,sha256=oo9lvrttInePaaSpHPZSrxDlx80u900HA5BvrVP1tTU,2199
+tunned_geobr/read_meso_region.py,sha256=LDW2lufe8yqToE79intlHBWmXWB5PS2j3C4Zh8ElOhg,2615
+tunned_geobr/read_metro_area.py,sha256=xvpdASJJamFN7qikPKjrytMKoNwIVRfRoR37lUsQo0g,1511
+tunned_geobr/read_micro_region.py,sha256=n0oAeIdyZGo2tkoSYzcP8OfvAkwTRTkH4rMOnfKzgI4,2523
+tunned_geobr/read_mining_processes.py,sha256=oVhCqhIQwLTXmaL6RRCmF-i81OLj-0Lb2k7SHiYgaf8,2576
+tunned_geobr/read_municipal_seat.py,sha256=jiDn6jbULbl5KQlmpowqZTLW2P0z1H9P8zIzMcPETKk,1156
+tunned_geobr/read_municipality.py,sha256=RrCg8MULpCgTwX3ngcIH0YpHcI0GR4CubF4AIgXdwqE,2607
+tunned_geobr/read_neighborhood.py,sha256=rk7U9ZamkkdKrYcd18TNu0Pjhr7OQN4RY8BUVfa9Xcw,1098
+tunned_geobr/read_pedology.py,sha256=yqzn6lEBS__6_SIUtneJzJVCc3l0XGs3vqO68ylDUSs,2790
+tunned_geobr/read_pop_arrangements.py,sha256=NN4zvU4P4G6_3PDshQFpmYgRlcCqwPqD8DdKyw6a714,1399
+tunned_geobr/read_region.py,sha256=osFZJI-DPndNABs3dqtZamcJGW5hNRx4CgQmU9Mo7cA,1118
+tunned_geobr/read_schools.py,sha256=RZbb_glUfppPF2RxJwKIMhHB7BAZzfY2tnVkyIE9HuQ,1378
+tunned_geobr/read_semiarid.py,sha256=9zNQepAnC1Sc89E-1YMd_QSFz5dH0Z-I7W7469-0jHo,1371
+tunned_geobr/read_settlements.py,sha256=jh7hbXX63NBnG-q1meO4ed0NvBfLzdY17_PT7p-lbDs,3067
+tunned_geobr/read_state.py,sha256=cM44s3hBOknwbOQxAP9J7SRWuSrA97YAW1GEw_G5bFE,2712
+tunned_geobr/read_statistical_grid.py,sha256=2ZoT-kbuvVkKEoTIfIiAc2jLM789cWWE3E8-HA_S-VA,4493
+tunned_geobr/read_urban_area.py,sha256=3n31dj-21glSA2qQI88ELz00-nRGcpNunBpPYb0h3RQ,1377
+tunned_geobr/read_urban_concentrations.py,sha256=mPEZIDyyGUxriqhtDwD_cGbVaJNYX11_11Vj42s-cls,1441
+tunned_geobr/read_weighting_area.py,sha256=ggXpUnNtXBGsXdBjhwuaCWgKReQjb1wNkFFtB2bkuNk,2338
+tunned_geobr/utils.py,sha256=by8mOgQlpY6uXX2dtstTLdEgjMcI0lnLzQ8deA3DYus,8182
+tunned_geobr-0.1.0.dist-info/RECORD,,

tunned_geobr-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: pdm-backend (2.4.3)
+Root-Is-Purelib: true
+Tag: py3-none-any

tunned_geobr-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+
+[gui_scripts]
+

tunned_geobr-0.1.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,2 @@
+YEAR: 2020
+COPYRIGHT HOLDER: Institute for Appled Economic Research (Ipea)