pyFBDS 0.1.0__tar.gz

pyfbds-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Open Geodata
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyfbds-0.1.0/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.4
+Name: pyFBDS
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.32.5
+Requires-Dist: requests-cache>=1.2.1
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: lxml>=6.0.0
+Requires-Dist: ipywidgets>=8.1.8
+Requires-Dist: aiohttp>=3.13.2
+Requires-Dist: nest-asyncio>=1.6.0
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: selenium>=4.38.0
+Dynamic: license-file
+
+# pyFBDS
+
+[![Repo](https://img.shields.io/badge/GitHub-repo-blue?logo=github&logoColor=f5f5f5)](https://github.com/open-geodata/pyFBDS)
+[![PyPI - Version](https://img.shields.io/pypi/v/pyfbds?logo=pypi&label=PyPI&color=blue)](https://pypi.org/project/pyfbds/)<br>
+[![Read the Docs](https://img.shields.io/readthedocs/pyFBDS?logo=ReadTheDocs&label=Read%20The%20Docs)](https://pyFBDS.readthedocs.io/)
+[![Publish Python to PyPI](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml/badge.svg)](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml)
+
+_Scripts_ para obter dados espaciais do [**repositório público de mapas e _shapefiles_ para _download_**](https://geo.fbds.org.br/). Veja mais na documentação:
+
+> [https://pyFBDS.readthedocs.io/](https://pyFBDS.readthedocs.io/)
+
+<br>
+
+---
+
+## Como Instalar?
+
+```shell
+pip3 install pyFBDS
+```
+
+<br>
+
+---
+
+## Como Usar?
+
+```shell
+# Importa pacote
+import pyFBDS
+
+# Instancia Objeto
+fbds = pyFBDS.Repo(output_path='.')
+
+# Chama o método
+fbds.get_municipio(id_ibge=353243)
+```

pyfbds-0.1.0/README.md

@@ -0,0 +1,37 @@
+# pyFBDS
+
+[![Repo](https://img.shields.io/badge/GitHub-repo-blue?logo=github&logoColor=f5f5f5)](https://github.com/open-geodata/pyFBDS)
+[![PyPI - Version](https://img.shields.io/pypi/v/pyfbds?logo=pypi&label=PyPI&color=blue)](https://pypi.org/project/pyfbds/)<br>
+[![Read the Docs](https://img.shields.io/readthedocs/pyFBDS?logo=ReadTheDocs&label=Read%20The%20Docs)](https://pyFBDS.readthedocs.io/)
+[![Publish Python to PyPI](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml/badge.svg)](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml)
+
+_Scripts_ para obter dados espaciais do [**repositório público de mapas e _shapefiles_ para _download_**](https://geo.fbds.org.br/). Veja mais na documentação:
+
+> [https://pyFBDS.readthedocs.io/](https://pyFBDS.readthedocs.io/)
+
+<br>
+
+---
+
+## Como Instalar?
+
+```shell
+pip3 install pyFBDS
+```
+
+<br>
+
+---
+
+## Como Usar?
+
+```shell
+# Importa pacote
+import pyFBDS
+
+# Instancia Objeto
+fbds = pyFBDS.Repo(output_path='.')
+
+# Chama o método
+fbds.get_municipio(id_ibge=353243)
+```

pyfbds-0.1.0/pyFBDS.egg-info/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.4
+Name: pyFBDS
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.32.5
+Requires-Dist: requests-cache>=1.2.1
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: lxml>=6.0.0
+Requires-Dist: ipywidgets>=8.1.8
+Requires-Dist: aiohttp>=3.13.2
+Requires-Dist: nest-asyncio>=1.6.0
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: selenium>=4.38.0
+Dynamic: license-file
+
+# pyFBDS
+
+[![Repo](https://img.shields.io/badge/GitHub-repo-blue?logo=github&logoColor=f5f5f5)](https://github.com/open-geodata/pyFBDS)
+[![PyPI - Version](https://img.shields.io/pypi/v/pyfbds?logo=pypi&label=PyPI&color=blue)](https://pypi.org/project/pyfbds/)<br>
+[![Read the Docs](https://img.shields.io/readthedocs/pyFBDS?logo=ReadTheDocs&label=Read%20The%20Docs)](https://pyFBDS.readthedocs.io/)
+[![Publish Python to PyPI](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml/badge.svg)](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml)
+
+_Scripts_ para obter dados espaciais do [**repositório público de mapas e _shapefiles_ para _download_**](https://geo.fbds.org.br/). Veja mais na documentação:
+
+> [https://pyFBDS.readthedocs.io/](https://pyFBDS.readthedocs.io/)
+
+<br>
+
+---
+
+## Como Instalar?
+
+```shell
+pip3 install pyFBDS
+```
+
+<br>
+
+---
+
+## Como Usar?
+
+```shell
+# Importa pacote
+import pyFBDS
+
+# Instancia Objeto
+fbds = pyFBDS.Repo(output_path='.')
+
+# Chama o método
+fbds.get_municipio(id_ibge=353243)
+```

pyfbds-0.1.0/pyFBDS.egg-info/SOURCES.txt

@@ -0,0 +1,23 @@
+LICENSE
+README.md
+pyproject.toml
+pyFBDS.egg-info/PKG-INFO
+pyFBDS.egg-info/SOURCES.txt
+pyFBDS.egg-info/dependency_links.txt
+pyFBDS.egg-info/requires.txt
+pyFBDS.egg-info/top_level.txt
+pyFDBS/__init__.py
+pyFDBS/requests/__init__.py
+pyFDBS/requests/cache.py
+pyFDBS/requests/download.py
+pyFDBS/requests/logger.py
+pyFDBS/requests/teste.py
+pyFDBS/requests/web.py
+pyFDBS/scraper/__init__.py
+pyFDBS/scraper/page/__init__.py
+pyFDBS/scraper/page/sss.py
+pyFDBS/scraper/webdriver/__init__.py
+pyFDBS/scraper/webdriver/chrome.py
+pyFDBS/scraper/webdriver/config.py
+pyFDBS/scraper/webdriver/firefox.py
+pyFDBS/scraper/webdriver/gecko.py

pyfbds-0.1.0/pyFBDS.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pyfbds-0.1.0/pyFBDS.egg-info/requires.txt

@@ -0,0 +1,9 @@
+requests>=2.32.5
+requests-cache>=1.2.1
+tqdm>=4.67.1
+lxml>=6.0.0
+ipywidgets>=8.1.8
+aiohttp>=3.13.2
+nest-asyncio>=1.6.0
+pandas>=2.3.3
+selenium>=4.38.0

pyfbds-0.1.0/pyFBDS.egg-info/top_level.txt

@@ -0,0 +1 @@
+pyFDBS

pyfbds-0.1.0/pyFDBS/__init__.py

@@ -0,0 +1,4 @@
+from .requests.download import download_files_parallel
+from .requests.logger import FBDSLogger
+from .requests.web import FBDS
+

pyfbds-0.1.0/pyFDBS/requests/cache.py

@@ -0,0 +1,42 @@
+"""
+Para cache das requisições
+"""
+
+from datetime import timedelta
+
+import requests
+import requests_cache
+
+# Configuração do cache
+requests_cache.install_cache(
+    cache_name='fbds_cache',  # Nome do arquivo de cache
+    backend='sqlite',  # Backend para armazenamento (SQLite)
+    expire_after=timedelta(days=3),  # Cache expira após 7 dias
+    allowable_methods=('GET', 'POST'),  # Métodos HTTP permitidos
+)
+
+
+def make_request(url):
+    """
+    Faz uma requisição HTTP com suporte a cache
+
+    Parameters:
+    -----------
+    url : str
+        URL para fazer a requisição
+
+    Returns:
+    --------
+    response : requests.Response
+        Resposta da requisição
+    is_cached : bool
+        Indica se a resposta veio do cache
+    """
+    response = requests.get(url)
+    is_cached = getattr(response, 'from_cache', False)
+
+    # Informação sobre o cache
+    # cache_status = 'CACHE' if is_cached else 'NOVA REQUISIÇÃO'
+    # print(f"{cache_status}: {url}")
+
+    return response, is_cached

pyfbds-0.1.0/pyFDBS/requests/download.py

@@ -0,0 +1,148 @@
+"""
+Módulo para download dos dados usando asyncio
+"""
+
+import asyncio
+from pathlib import Path
+
+import aiohttp
+from tqdm.asyncio import tqdm_asyncio
+from tqdm.notebook import tqdm
+
+from .cache import make_request
+from .logger import FBDSLogger
+
+
+async def download_file_async(session, url_info, output_dir):
+    """
+    Download assíncrono de um único arquivo
+
+    Parameters:
+    -----------
+    session : aiohttp.ClientSession
+        Sessão HTTP assíncrona
+    url_info : dict
+        Dicionário com informações do arquivo (url, name, etc)
+    output_dir : str or Path
+        Diretório onde salvar o arquivo
+    """
+    try:
+        url = url_info['url']
+        # Remove o base URL e usa o caminho relativo
+        relative_path = url.replace('https://geo.fbds.org.br/', '')
+        output_path = Path(output_dir) / relative_path
+
+        # Cria o diretório se não existir
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Faz o download
+        async with session.get(url) as response:
+            if response.status == 200:
+                content = await response.read()
+
+                # Salva o arquivo
+                with open(output_path, 'wb') as f:
+                    f.write(content)
+
+                result = {
+                    'nome': url_info['name'],
+                    'status': 'sucesso',
+                    'size': len(content),
+                }
+            else:
+                result = {
+                    'nome': url_info['name'],
+                    'status': 'erro',
+                    'erro': f'Status code: {response.status}',
+                }
+    except Exception as e:
+        result = {'nome': url_info['name'], 'status': 'erro', 'erro': str(e)}
+
+    return result
+
+
+async def download_files_async(url_list, output_dir, max_concurrent=5):
+    """
+    Download assíncrono de múltiplos arquivos
+
+    Parameters:
+    -----------
+    url_list : list
+        Lista de dicionários com informações dos arquivos
+    output_dir : str or Path
+        Diretório onde salvar os arquivos
+    max_concurrent : int
+        Número máximo de downloads simultâneos
+    """
+    # Configura conexão com limite de conexões simultâneas
+    conn = aiohttp.TCPConnector(limit=max_concurrent)
+
+    async with aiohttp.ClientSession(connector=conn) as session:
+        # Cria a lista de tarefas
+        tasks = []
+        for url_info in url_list:
+            task = download_file_async(session, url_info, output_dir)
+            tasks.append(task)
+
+        # Executa as tasks com barra de progresso
+        results = await tqdm_asyncio.gather(
+            *tasks,
+            desc="Downloading files",
+            total=len(tasks),
+            ascii=True,  # Melhor compatibilidade
+            mininterval=0.5,  # Atualiza a cada 0.5 segundos
+        )
+
+    return results
+
+
+def download_files_parallel(
+    url_list, output_dir, max_concurrent=5, logger=None
+):
+    """
+    Wrapper para executar o download assíncrono
+
+    Parameters:
+    -----------
+    url_list : list
+        Lista de dicionários com informações dos arquivos
+    output_dir : str or Path
+        Diretório onde salvar os arquivos
+    max_concurrent : int
+        Número máximo de downloads simultâneos
+    logger : FBDSLogger, optional
+        Logger existente para usar. Se None, cria um novo.
+    """
+    try:
+        # Usa o logger fornecido ou cria um novo
+        if logger is None:
+            logger = FBDSLogger()
+        logger.start_download_session()
+
+        # Pega o loop de eventos atual ou cria um novo se não existir
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        # Se estamos em um notebook IPython, use o nest_asyncio
+        try:
+            import nest_asyncio
+
+            nest_asyncio.apply()
+        except ImportError:
+            pass
+
+        # Executa o download assíncrono
+        results = loop.run_until_complete(
+            download_files_async(url_list, output_dir, max_concurrent)
+        )
+
+        # Analisa e registra os resultados
+        logger.analyze_results(results)
+        return results
+
+    except Exception as e:
+        logger.logger.error(f"Erro durante o download: {str(e)}")
+        return []

pyfbds-0.1.0/pyFDBS/requests/logger.py

@@ -0,0 +1,168 @@
+"""
+Sistema de logs para a aplicação FBDS
+"""
+
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+
+# Obtém o diretório raiz do projeto
+PROJECT_ROOT = Path(__file__).parent.parent
+DEFAULT_LOG_DIR = PROJECT_ROOT / 'logs'
+
+
+class FBDSLogger:
+    _instance = None
+    _initialized = False
+
+    def __new__(cls, log_dir=None, new_session=False):
+        # Verifica se já existe uma instância ou se foi pedida uma nova sessão
+        if cls._instance is None or new_session:
+            # Cria uma nova instância se não existir ou se new_session=True
+            cls._instance = super(FBDSLogger, cls).__new__(cls)
+            # Marca como não inicializado para forçar a execução do __init__
+            cls._initialized = False
+        # Retorna a instância (seja ela nova ou existente)
+        return cls._instance
+
+    def __init__(self, log_dir=None, new_session=False):
+        if not self._initialized or new_session:
+            # Usa o diretório fornecido ou o padrão
+            self.log_dir = Path(log_dir) if log_dir else DEFAULT_LOG_DIR
+            self.log_dir.mkdir(parents=True, exist_ok=True)
+
+            # Configura o logger principal
+            self.logger = logging.getLogger('FBDS')
+            self.logger.setLevel(logging.INFO)
+
+            # Remove handlers anteriores se existirem
+            for handler in self.logger.handlers[:]:
+                self.logger.removeHandler(handler)
+
+            # Cria handlers
+            self._setup_handlers()
+
+            # Dicionário para armazenar estatísticas
+            self.stats = {
+                'total': 0,
+                'success': 0,
+                'errors': 0,
+                'cached': 0,
+                'start_time': None,
+                'end_time': None,
+                'errors_list': [],
+            }
+
+            self._initialized = True
+
+    def _setup_handlers(self):
+        # Handler para arquivo
+        # Usa apenas a data, não o timestamp completo
+        date_str = datetime.now().strftime('%Y%m%d')
+        self.log_file = self.log_dir / f'fbds_{date_str}.log'
+        # self.stats_file = self.log_dir / f'stats_{date_str}.json'
+
+        file_handler = logging.FileHandler(
+            self.log_file, encoding='utf-8', mode='a'
+        )
+        file_handler.setLevel(logging.INFO)
+
+        # Handler para console
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.INFO)
+
+        # Formato do log
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S',
+        )
+
+        file_handler.setFormatter(formatter)
+        console_handler.setFormatter(formatter)
+
+        # Adiciona handlers ao logger
+        self.logger.addHandler(file_handler)
+        self.logger.addHandler(console_handler)
+
+    def start_download_session(self):
+        """Inicia uma nova sessão de download"""
+        self.stats = {
+            'total': 0,
+            'success': 0,
+            'errors': 0,
+            'cached': 0,
+            'start_time': datetime.now(),
+            'end_time': None,
+            'errors_list': [],
+        }
+        self.logger.info('Iniciando nova sessão de download')
+
+    def end_download_session(self):
+        """Finaliza a sessão de download e gera relatório"""
+        self.stats['end_time'] = datetime.now()
+        duration = self.stats['end_time'] - self.stats['start_time']
+
+        # Log do resumo
+        self.logger.info(f"=== Resumo da Sessão de Download ===")
+        self.logger.info(
+            f"Downloads com sucesso: {self.stats['success']} de {self.stats['total']}"
+        )
+        if self.stats['errors'] > 0:
+            self.logger.error(
+                f"Erros: {self.stats['errors']} de {self.stats['total']}"
+            )
+        # self.logger.info(
+        #     f"Arquivos do cache: {self.stats['cached']} de {self.stats['total']}"
+        # )
+        self.logger.info(f"Duração total: {duration}")
+
+        # Se houver erros, registra eles
+        if self.stats['errors_list']:
+            self.logger.error("Erros encontrados:")
+            for error in self.stats['errors_list']:
+                self.logger.error(f"- {error['nome']}: {error['erro']}")
+
+        # # Salva as estatísticas em JSON
+        # stats_file = (
+        #     self.log_dir
+        #     / f'stats_{datetime.now().strftime("%Y%m%d_%H%M%S")}.json'
+        # )
+        # with open(stats_file, 'w', encoding='utf-8') as f:
+        #     # Converte datetime para string
+        #     stats_dict = self.stats.copy()
+        #     stats_dict['start_time'] = self.stats['start_time'].isoformat()
+        #     stats_dict['end_time'] = self.stats['end_time'].isoformat()
+        #     json.dump(stats_dict, f, ensure_ascii=False, indent=4)
+
+    def log_result(self, result):
+        """Registra o resultado de um download"""
+        self.stats['total'] += 1
+
+        if result.get('cached', False):
+            self.stats['cached'] += 1
+            self.logger.info(f"Arquivo em cache: {result['nome']}")
+
+        if result['status'] == 'sucesso':
+            self.stats['success'] += 1
+            self.logger.info(
+                f"Download concluído: {result['nome']} ({result['size']} bytes)"
+            )
+        else:
+            self.stats['errors'] += 1
+            self.stats['errors_list'].append(result)
+            self.logger.error(
+                f"Erro no download de {result['nome']}: {result['erro']}"
+            )
+
+    def analyze_results(self, results):
+        """
+        Analisa os resultados dos downloads
+        """
+        self.start_download_session()
+
+        for result in results:
+            self.log_result(result)
+
+        self.end_download_session()
+        return self.stats

pyfbds-0.1.0/pyFDBS/requests/teste.py

@@ -0,0 +1,13 @@
+# """
+# Sistema de logs para a aplicação FBDS
+# """
+
+# import json
+# import logging
+# from datetime import datetime
+# from pathlib import Path
+
+# # Obtém o diretório raiz do projeto
+# PROJECT_ROOT = Path(__file__).parent.parent
+# DEFAULT_LOG_DIR = PROJECT_ROOT / 'logs'
+# print(DEFAULT_LOG_DIR)

pyfbds-0.1.0/pyFDBS/requests/web.py

@@ -0,0 +1,78 @@
+"""
+Summary
+"""
+
+from pathlib import Path
+from urllib.parse import urljoin
+
+from lxml import html
+
+from .cache import make_request
+
+
+class FBDS:
+    def __init__(self) -> None:
+        self.url_base = "https://geo.fbds.org.br/"
+
+    def get_links(self, url, ignore_first):
+        # Usa a função make_request com cache
+        response, is_cached = make_request(url)
+        response.raise_for_status()
+        tree = html.fromstring(response.content)
+
+        list_folders = []
+        folders = tree.xpath("//tr")
+
+        # Ignora o primeiro tr, que é o cabeçalho
+        folders = folders[ignore_first:]
+
+        for folder in folders:
+            # Get Data
+            link = folder.xpath('.//td[@class="fb-n"]/a/@href')[0]
+            tipo = folder.xpath('.//td[@class="fb-i"]/img/@src')[0]
+            name = folder.xpath('.//td[@class="fb-n"]/a')[0].text.strip()
+            data = folder.xpath('.//td[@class="fb-d"]')[0].text.strip()
+            size = folder.xpath('.//td[@class="fb-s"]')[0].text.strip()
+
+            # Append to list
+            list_folders.append(
+                {
+                    "url": urljoin(self.url_base, link),
+                    "type": Path(tipo).stem,
+                    "name": name,
+                    "date": data,
+                    "size": size,
+                }
+            )
+        return list_folders
+
+    def get_states(self) -> list[dict]:
+        """
+        
+
+        :return: _description_
+        :rtype: list[dict]
+        """
+        return self.get_links(url=self.url_base, ignore_first=1)
+
+    def get_municipalities(self, uf):
+        state = self.get_state(uf=uf)
+        url = state["url"]
+        return self.get_links(url=url, ignore_first=2)
+
+    def get_layers(self, municipality, uf):
+        municipalitie = self.get_municipalitie(municipality=municipality, uf=uf)
+        url = municipalitie["url"]
+        return self.get_links(url=url, ignore_first=2)
+
+    def get_state(self, uf=None):
+        states = self.get_states()
+        return [x for x in states if x["name"] == uf][0]
+
+    def get_municipalitie(self, municipality=None, uf=None):
+        municipalities = self.get_municipalities(uf=uf)
+        return [x for x in municipalities if x["name"] == municipality][0]
+
+    def get_layer(self, municipality=None, uf=None, layer=None):
+        layers = self.get_layers(municipality=municipality, uf=uf)
+        return [x for x in layers if x["name"] == layer][0]

pyfbds-0.1.0/pyFDBS/scraper/__init__.py

@@ -0,0 +1 @@
+#from . import page, webdriver, search, outros, params

pyfbds-0.1.0/pyFDBS/scraper/webdriver/__init__.py

@@ -0,0 +1,2 @@
+from .firefox import Firefox
+from .chrome import Chrome