pyFBDS 0.1.0__py3-none-any.whl

pyFDBS/__init__.py

@@ -0,0 +1,4 @@
+from .requests.download import download_files_parallel
+from .requests.logger import FBDSLogger
+from .requests.web import FBDS
+

pyFDBS/requests/cache.py

@@ -0,0 +1,42 @@
+"""
+Para cache das requisições
+"""
+
+from datetime import timedelta
+
+import requests
+import requests_cache
+
+# Configuração do cache
+requests_cache.install_cache(
+    cache_name='fbds_cache',  # Nome do arquivo de cache
+    backend='sqlite',  # Backend para armazenamento (SQLite)
+    expire_after=timedelta(days=3),  # Cache expira após 7 dias
+    allowable_methods=('GET', 'POST'),  # Métodos HTTP permitidos
+)
+
+
+def make_request(url):
+    """
+    Faz uma requisição HTTP com suporte a cache
+
+    Parameters:
+    -----------
+    url : str
+        URL para fazer a requisição
+
+    Returns:
+    --------
+    response : requests.Response
+        Resposta da requisição
+    is_cached : bool
+        Indica se a resposta veio do cache
+    """
+    response = requests.get(url)
+    is_cached = getattr(response, 'from_cache', False)
+
+    # Informação sobre o cache
+    # cache_status = 'CACHE' if is_cached else 'NOVA REQUISIÇÃO'
+    # print(f"{cache_status}: {url}")
+
+    return response, is_cached

pyFDBS/requests/download.py

@@ -0,0 +1,148 @@
+"""
+Módulo para download dos dados usando asyncio
+"""
+
+import asyncio
+from pathlib import Path
+
+import aiohttp
+from tqdm.asyncio import tqdm_asyncio
+from tqdm.notebook import tqdm
+
+from .cache import make_request
+from .logger import FBDSLogger
+
+
+async def download_file_async(session, url_info, output_dir):
+    """
+    Download assíncrono de um único arquivo
+
+    Parameters:
+    -----------
+    session : aiohttp.ClientSession
+        Sessão HTTP assíncrona
+    url_info : dict
+        Dicionário com informações do arquivo (url, name, etc)
+    output_dir : str or Path
+        Diretório onde salvar o arquivo
+    """
+    try:
+        url = url_info['url']
+        # Remove o base URL e usa o caminho relativo
+        relative_path = url.replace('https://geo.fbds.org.br/', '')
+        output_path = Path(output_dir) / relative_path
+
+        # Cria o diretório se não existir
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Faz o download
+        async with session.get(url) as response:
+            if response.status == 200:
+                content = await response.read()
+
+                # Salva o arquivo
+                with open(output_path, 'wb') as f:
+                    f.write(content)
+
+                result = {
+                    'nome': url_info['name'],
+                    'status': 'sucesso',
+                    'size': len(content),
+                }
+            else:
+                result = {
+                    'nome': url_info['name'],
+                    'status': 'erro',
+                    'erro': f'Status code: {response.status}',
+                }
+    except Exception as e:
+        result = {'nome': url_info['name'], 'status': 'erro', 'erro': str(e)}
+
+    return result
+
+
+async def download_files_async(url_list, output_dir, max_concurrent=5):
+    """
+    Download assíncrono de múltiplos arquivos
+
+    Parameters:
+    -----------
+    url_list : list
+        Lista de dicionários com informações dos arquivos
+    output_dir : str or Path
+        Diretório onde salvar os arquivos
+    max_concurrent : int
+        Número máximo de downloads simultâneos
+    """
+    # Configura conexão com limite de conexões simultâneas
+    conn = aiohttp.TCPConnector(limit=max_concurrent)
+
+    async with aiohttp.ClientSession(connector=conn) as session:
+        # Cria a lista de tarefas
+        tasks = []
+        for url_info in url_list:
+            task = download_file_async(session, url_info, output_dir)
+            tasks.append(task)
+
+        # Executa as tasks com barra de progresso
+        results = await tqdm_asyncio.gather(
+            *tasks,
+            desc="Downloading files",
+            total=len(tasks),
+            ascii=True,  # Melhor compatibilidade
+            mininterval=0.5,  # Atualiza a cada 0.5 segundos
+        )
+
+    return results
+
+
+def download_files_parallel(
+    url_list, output_dir, max_concurrent=5, logger=None
+):
+    """
+    Wrapper para executar o download assíncrono
+
+    Parameters:
+    -----------
+    url_list : list
+        Lista de dicionários com informações dos arquivos
+    output_dir : str or Path
+        Diretório onde salvar os arquivos
+    max_concurrent : int
+        Número máximo de downloads simultâneos
+    logger : FBDSLogger, optional
+        Logger existente para usar. Se None, cria um novo.
+    """
+    try:
+        # Usa o logger fornecido ou cria um novo
+        if logger is None:
+            logger = FBDSLogger()
+        logger.start_download_session()
+
+        # Pega o loop de eventos atual ou cria um novo se não existir
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        # Se estamos em um notebook IPython, use o nest_asyncio
+        try:
+            import nest_asyncio
+
+            nest_asyncio.apply()
+        except ImportError:
+            pass
+
+        # Executa o download assíncrono
+        results = loop.run_until_complete(
+            download_files_async(url_list, output_dir, max_concurrent)
+        )
+
+        # Analisa e registra os resultados
+        logger.analyze_results(results)
+        return results
+
+    except Exception as e:
+        logger.logger.error(f"Erro durante o download: {str(e)}")
+        return []

pyFDBS/requests/logger.py

@@ -0,0 +1,168 @@
+"""
+Sistema de logs para a aplicação FBDS
+"""
+
+import json
+import logging
+from datetime import datetime
+from pathlib import Path
+
+# Obtém o diretório raiz do projeto
+PROJECT_ROOT = Path(__file__).parent.parent
+DEFAULT_LOG_DIR = PROJECT_ROOT / 'logs'
+
+
+class FBDSLogger:
+    _instance = None
+    _initialized = False
+
+    def __new__(cls, log_dir=None, new_session=False):
+        # Verifica se já existe uma instância ou se foi pedida uma nova sessão
+        if cls._instance is None or new_session:
+            # Cria uma nova instância se não existir ou se new_session=True
+            cls._instance = super(FBDSLogger, cls).__new__(cls)
+            # Marca como não inicializado para forçar a execução do __init__
+            cls._initialized = False
+        # Retorna a instância (seja ela nova ou existente)
+        return cls._instance
+
+    def __init__(self, log_dir=None, new_session=False):
+        if not self._initialized or new_session:
+            # Usa o diretório fornecido ou o padrão
+            self.log_dir = Path(log_dir) if log_dir else DEFAULT_LOG_DIR
+            self.log_dir.mkdir(parents=True, exist_ok=True)
+
+            # Configura o logger principal
+            self.logger = logging.getLogger('FBDS')
+            self.logger.setLevel(logging.INFO)
+
+            # Remove handlers anteriores se existirem
+            for handler in self.logger.handlers[:]:
+                self.logger.removeHandler(handler)
+
+            # Cria handlers
+            self._setup_handlers()
+
+            # Dicionário para armazenar estatísticas
+            self.stats = {
+                'total': 0,
+                'success': 0,
+                'errors': 0,
+                'cached': 0,
+                'start_time': None,
+                'end_time': None,
+                'errors_list': [],
+            }
+
+            self._initialized = True
+
+    def _setup_handlers(self):
+        # Handler para arquivo
+        # Usa apenas a data, não o timestamp completo
+        date_str = datetime.now().strftime('%Y%m%d')
+        self.log_file = self.log_dir / f'fbds_{date_str}.log'
+        # self.stats_file = self.log_dir / f'stats_{date_str}.json'
+
+        file_handler = logging.FileHandler(
+            self.log_file, encoding='utf-8', mode='a'
+        )
+        file_handler.setLevel(logging.INFO)
+
+        # Handler para console
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.INFO)
+
+        # Formato do log
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S',
+        )
+
+        file_handler.setFormatter(formatter)
+        console_handler.setFormatter(formatter)
+
+        # Adiciona handlers ao logger
+        self.logger.addHandler(file_handler)
+        self.logger.addHandler(console_handler)
+
+    def start_download_session(self):
+        """Inicia uma nova sessão de download"""
+        self.stats = {
+            'total': 0,
+            'success': 0,
+            'errors': 0,
+            'cached': 0,
+            'start_time': datetime.now(),
+            'end_time': None,
+            'errors_list': [],
+        }
+        self.logger.info('Iniciando nova sessão de download')
+
+    def end_download_session(self):
+        """Finaliza a sessão de download e gera relatório"""
+        self.stats['end_time'] = datetime.now()
+        duration = self.stats['end_time'] - self.stats['start_time']
+
+        # Log do resumo
+        self.logger.info(f"=== Resumo da Sessão de Download ===")
+        self.logger.info(
+            f"Downloads com sucesso: {self.stats['success']} de {self.stats['total']}"
+        )
+        if self.stats['errors'] > 0:
+            self.logger.error(
+                f"Erros: {self.stats['errors']} de {self.stats['total']}"
+            )
+        # self.logger.info(
+        #     f"Arquivos do cache: {self.stats['cached']} de {self.stats['total']}"
+        # )
+        self.logger.info(f"Duração total: {duration}")
+
+        # Se houver erros, registra eles
+        if self.stats['errors_list']:
+            self.logger.error("Erros encontrados:")
+            for error in self.stats['errors_list']:
+                self.logger.error(f"- {error['nome']}: {error['erro']}")
+
+        # # Salva as estatísticas em JSON
+        # stats_file = (
+        #     self.log_dir
+        #     / f'stats_{datetime.now().strftime("%Y%m%d_%H%M%S")}.json'
+        # )
+        # with open(stats_file, 'w', encoding='utf-8') as f:
+        #     # Converte datetime para string
+        #     stats_dict = self.stats.copy()
+        #     stats_dict['start_time'] = self.stats['start_time'].isoformat()
+        #     stats_dict['end_time'] = self.stats['end_time'].isoformat()
+        #     json.dump(stats_dict, f, ensure_ascii=False, indent=4)
+
+    def log_result(self, result):
+        """Registra o resultado de um download"""
+        self.stats['total'] += 1
+
+        if result.get('cached', False):
+            self.stats['cached'] += 1
+            self.logger.info(f"Arquivo em cache: {result['nome']}")
+
+        if result['status'] == 'sucesso':
+            self.stats['success'] += 1
+            self.logger.info(
+                f"Download concluído: {result['nome']} ({result['size']} bytes)"
+            )
+        else:
+            self.stats['errors'] += 1
+            self.stats['errors_list'].append(result)
+            self.logger.error(
+                f"Erro no download de {result['nome']}: {result['erro']}"
+            )
+
+    def analyze_results(self, results):
+        """
+        Analisa os resultados dos downloads
+        """
+        self.start_download_session()
+
+        for result in results:
+            self.log_result(result)
+
+        self.end_download_session()
+        return self.stats

pyFDBS/requests/teste.py

@@ -0,0 +1,13 @@
+# """
+# Sistema de logs para a aplicação FBDS
+# """
+
+# import json
+# import logging
+# from datetime import datetime
+# from pathlib import Path
+
+# # Obtém o diretório raiz do projeto
+# PROJECT_ROOT = Path(__file__).parent.parent
+# DEFAULT_LOG_DIR = PROJECT_ROOT / 'logs'
+# print(DEFAULT_LOG_DIR)

pyFDBS/requests/web.py

@@ -0,0 +1,78 @@
+"""
+Summary
+"""
+
+from pathlib import Path
+from urllib.parse import urljoin
+
+from lxml import html
+
+from .cache import make_request
+
+
+class FBDS:
+    def __init__(self) -> None:
+        self.url_base = "https://geo.fbds.org.br/"
+
+    def get_links(self, url, ignore_first):
+        # Usa a função make_request com cache
+        response, is_cached = make_request(url)
+        response.raise_for_status()
+        tree = html.fromstring(response.content)
+
+        list_folders = []
+        folders = tree.xpath("//tr")
+
+        # Ignora o primeiro tr, que é o cabeçalho
+        folders = folders[ignore_first:]
+
+        for folder in folders:
+            # Get Data
+            link = folder.xpath('.//td[@class="fb-n"]/a/@href')[0]
+            tipo = folder.xpath('.//td[@class="fb-i"]/img/@src')[0]
+            name = folder.xpath('.//td[@class="fb-n"]/a')[0].text.strip()
+            data = folder.xpath('.//td[@class="fb-d"]')[0].text.strip()
+            size = folder.xpath('.//td[@class="fb-s"]')[0].text.strip()
+
+            # Append to list
+            list_folders.append(
+                {
+                    "url": urljoin(self.url_base, link),
+                    "type": Path(tipo).stem,
+                    "name": name,
+                    "date": data,
+                    "size": size,
+                }
+            )
+        return list_folders
+
+    def get_states(self) -> list[dict]:
+        """
+        
+
+        :return: _description_
+        :rtype: list[dict]
+        """
+        return self.get_links(url=self.url_base, ignore_first=1)
+
+    def get_municipalities(self, uf):
+        state = self.get_state(uf=uf)
+        url = state["url"]
+        return self.get_links(url=url, ignore_first=2)
+
+    def get_layers(self, municipality, uf):
+        municipalitie = self.get_municipalitie(municipality=municipality, uf=uf)
+        url = municipalitie["url"]
+        return self.get_links(url=url, ignore_first=2)
+
+    def get_state(self, uf=None):
+        states = self.get_states()
+        return [x for x in states if x["name"] == uf][0]
+
+    def get_municipalitie(self, municipality=None, uf=None):
+        municipalities = self.get_municipalities(uf=uf)
+        return [x for x in municipalities if x["name"] == municipality][0]
+
+    def get_layer(self, municipality=None, uf=None, layer=None):
+        layers = self.get_layers(municipality=municipality, uf=uf)
+        return [x for x in layers if x["name"] == layer][0]

pyFDBS/scraper/__init__.py

@@ -0,0 +1 @@
+#from . import page, webdriver, search, outros, params

pyFDBS/scraper/webdriver/__init__.py

@@ -0,0 +1,2 @@
+from .firefox import Firefox
+from .chrome import Chrome

pyFDBS/scraper/webdriver/chrome.py

@@ -0,0 +1,295 @@
+"""
+Módulo para usar driver do Chrome
+
+Michel Metran
+Data: 24.10.2024
+Atualizado em: 24.10.2024
+"""
+
+import tempfile
+from pathlib import Path
+
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options as ChromeOptions
+
+from . import config
+
+
+class Chrome(webdriver.Chrome):
+    """
+    Cria driver customizado do Selenium
+
+    :param webdriver: _description_
+    :type webdriver: _type_
+    """
+
+    def __init__(
+        self,
+        # driver_path: Path,
+        # logs_path: Path,
+        # down_path: Path,
+        *args,
+        **kwargs,
+    ):
+        """
+        - verify_ssl
+        - headless
+        - download_path
+        """
+        # Parameters
+        headless = kwargs.get('headless', False)
+        self.download_path = kwargs.get('download_path', False)
+        modo_colab = kwargs.get('modo_colab', False)
+
+        # Temp Path
+        temp_path = tempfile.gettempdir()
+        project_temp_path = Path(temp_path) / config.TEMP_PATH_NAME
+
+        # Scrapy Path
+        scrapy_path = project_temp_path / 'scrapy'
+        scrapy_path.mkdir(exist_ok=True, parents=True)
+
+        # Download Path
+        if self.download_path is False:
+            # Cria Pasta
+            self.download_path = scrapy_path / 'download'
+            self.download_path.mkdir(exist_ok=True, parents=True)
+
+        # my_service = ChromeService()
+        # print(str(self.download_path))
+        # print(self.download_path)
+        # print(self.download_path.is_dir())
+        # print(str(self.download_path) + os.path.sep)
+
+        # Options
+        options = ChromeOptions()
+        options.add_argument('--start-maximized')
+        options.add_argument('--disable-gpu')
+
+        # Se tem Modo Anônimo, o download não funciona adequadamente
+        # options.add_argument('--incognito')
+
+        # Certificados
+        options.add_argument('--ignore-certificate-errors-spki-list')
+        options.add_argument('--ignore-certificate-errors')
+        options.add_argument('--ignore-ssl-errors')
+
+        options.add_argument('--disable-infobars')
+        options.add_argument('--disable-extensions')
+        # options.add_argument('--disable-logging')
+        # Remove a mensagem "Chrome is being controlled by automated test software"
+        # que aparece quando o Chrome é iniciado pelo Selenium.
+        options.add_experimental_option(
+            'excludeSwitches', ['enable-automation']
+        )
+        # Desativa a extensão de automação do Chrome que é carregada por
+        # padrão quando o Chrome é iniciado pelo Selenium.
+        # Isso ajuda a evitar que sites detectem que o navegador está
+        # sendo controlado por um script de automação.
+        options.add_experimental_option('useAutomationExtension', False)
+
+        options.add_experimental_option(
+            'prefs',
+            {
+                'credentials_enable_service': False,
+                'profile.password_manager_enabled': False,
+                'download.default_directory': str(self.download_path),
+                # + os.path.sep,
+                'download.prompt_for_download': False,
+                'download.directory_upgrade': True,
+                'safebrowsing.enabled': True,
+            },
+        )
+
+        if headless is True:
+            options.add_argument('--headless')
+
+        if modo_colab is True:
+            options.add_argument('--headless')
+            options.add_argument('--no-sandbox')
+            options.add_argument('--disable-dev-shm-usage')
+
+        super().__init__(
+            # service=my_service,
+            options=options
+        )
+        self.maximize_window()
+
+
+if __name__ == '__main__':
+    from pathlib import Path
+
+    import pyesaj.scraper as esaj
+    from selenium.webdriver.common.by import By
+
+    # # Credenciais
+    # load_dotenv()
+    # USERNAME = os.getenv('USERNAME_TJSP')
+    # PASSWORD = os.getenv('PASSWORD_TJSP')
+    # Instancia Driver
+    driver = esaj.webdriver.Chrome(headless=False)
+
+    driver.get(url='https://filesamples.com/formats/csv')
+
+    print('vai pra botão')
+    btn = driver.find_element(
+        By.XPATH, '//a[@href="/samples/document/csv/sample4.csv"]'
+    )
+    print('clica!')
+    btn.click()
+
+    # # Login
+    # log = esaj.page.Login(driver=driver)
+    # log.login(username=USERNAME, password=PASSWORD)
+
+    # # Intimações
+    # processo = '2336412-80.2024.8.26.0000'
+
+    # # Define
+    # intim_search = esaj.params.intim.input.ConsultaIntimacoes(
+    #     em_nome_de='Ministério Público do Estado de São Paulo',
+    #     instancia='Segundo Grau',
+    #     # secao='Direito Criminal',
+    #     # orgao_julgador='17ª Câmara de Direito Privado B',
+    #     # especializacao='Criminal',
+    #     # especializacao_nao_definida=True,
+    #     # cargo='Secretaria ProcCriminal',
+    #     # cargo_nao_definido=True,
+    #     # assunto_principal='10527 - Livros / Jornais / Periódicos',
+    #     area='Ambas',
+    #     ciencia_ato='Todos',
+    #     natureza_comunicacao='Ambas',
+    #     situacao='Ambas',
+    #     processo=processo,
+    # )
+
+    # dados = intim_search
+    # print(intim_search)
+
+    # # Vai para página
+    # esaj.page.intim.Consulta(driver=driver)
+
+    # # Atributo: Em Nome De
+    # if dados.em_nome_de is not None:
+    #     intim_em_nome = esaj.page.intim.EmNomeDe(driver=driver)
+    #     intim_em_nome.set_option(option=dados.em_nome_de)
+
+    # # Atributo: Tipo de Participação
+    # if dados.tipo_participacao is not None:
+    #     intim_tipo = esaj.page.intim.TipoParticipacao(driver=driver)
+    #     intim_tipo.set_option(option=dados.tipo_participacao)
+
+    # # Atributo: Instância
+    # if dados.instancia is not None:
+    #     intim_inst = esaj.page.intim.Instancia(driver=driver)
+    #     intim_inst.set_option(instancia=dados.instancia)
+
+    # # Atributo: Foro
+    # if dados.foro is not None:
+    #     intim_foro = esaj.page.intim.Foro(driver=driver)
+    #     intim_foro.set_option(option=dados.foro)
+
+    # # Atributo: Vara
+    # if dados.vara is not None:
+    #     intim_vara = esaj.page.intim.Vara(driver=driver)
+    #     intim_vara.set_option(option=dados.vara)
+
+    # # Atributo: Seção
+    # if dados.secao is not None:
+    #     intim_secao = esaj.page.intim.Secao(driver=driver)
+    #     intim_secao.set_option(option=dados.secao)
+
+    # # Atributo: Órgão Julgador
+    # if dados.orgao_julgador is not None:
+    #     intim_org = esaj.page.intim.OrgaoJulgador(driver=driver)
+    #     intim_org.set_option(option=dados.orgao_julgador)
+
+    # # Atributo: Especialização
+    # if dados.especializacao is not None:
+    #     intim_esp = esaj.page.intim.Especializacao(driver=driver)
+    #     intim_esp.set_option(option=dados.especializacao)
+
+    # # Atributo: Especialização Checkbox
+    # if dados.especializacao_nao_definida is True:
+    #     intim_esp = esaj.page.intim.Especializacao(driver=driver)
+    #     intim_esp.apenas_processos_sem_especializacao()
+
+    # # Atributo: Cargo
+    # if dados.cargo is not None:
+    #     intim_cargo = esaj.page.intim.Cargo(driver=driver)
+    #     intim_cargo.set_option(option=dados.cargo)
+
+    # # Atributo: Cargo Checkbox
+    # if dados.especializacao_nao_definida is True:
+    #     intim_cargo = esaj.page.intim.Cargo(driver=driver)
+    #     intim_cargo.apenas_processos_sem_cargo()
+
+    # # Atributo: Classe
+    # if dados.classe is not None:
+    #     intim_classe = esaj.page.intim.Classe(driver=driver)
+    #     intim_classe.set_option(option=dados.classe)
+
+    # # Atributo: Assunto Principal
+    # if dados.assunto_principal is not None:
+    #     intim_assunto = esaj.page.intim.AssuntoPrincipal(driver=driver)
+    #     intim_assunto.set_option(option=dados.assunto_principal)
+
+    # # Atributo: Área
+    # if dados.area is not None:
+    #     intim_area = esaj.page.intim.Area(driver=driver)
+    #     intim_area.set_option(option=dados.area)
+
+    # # Período: De
+    # if dados.periodo_de is not None:
+    #     intim_per = esaj.page.intim.consulta.Periodo(driver=driver)
+    #     intim_per.de(data=dados.periodo_de)
+
+    # # Período: Até
+    # if dados.periodo_ate is not None:
+    #     intim_per = esaj.page.intim.consulta.Periodo(driver=driver)
+    #     intim_per.ate(data=dados.periodo_ate)
+
+    # # Período: Intervalo
+    # if dados.periodo_de is not None and dados.periodo_ate is not None:
+    #     intim_per = esaj.page.intim.consulta.Periodo(driver=driver)
+    #     intim_per.define_intervalo(de=dados.periodo_de, ate=dados.periodo_ate)
+
+    # # Atributo: Processo
+    # if dados.processo is not None:
+    #     intim_proc = esaj.page.intim.Processo(driver=driver)
+    #     intim_proc.write(texto=dados.processo)
+
+    # # Atributo: Ciência do Ato
+    # if dados.ciencia_ato is not None:
+    #     intim_cien = esaj.page.intim.consulta.CienciaAto(driver=driver)
+    #     intim_cien.set_option(option=dados.ciencia_ato)
+
+    # # Atributo: Natureza de Comunicação
+    # if dados.natureza_comunicacao is not None:
+    #     intim_nat = esaj.page.intim.NaturezaComunicacao(driver=driver)
+    #     intim_nat.set_option(natureza=dados.natureza_comunicacao)
+
+    # # Atributo: Situação
+    # if dados.situacao is not None:
+    #     intim_sit = esaj.page.intim.consulta.Situacao(driver=driver)
+    #     intim_sit.set_option(situacao=dados.situacao)
+
+    # # Aguarda
+    # time.sleep(2)
+
+    # # Consulta
+    # consulta = esaj.page.intim.ConsultarIntimacoes(driver=driver)
+    # consulta.consultar()
+
+    # # Tabela
+    # tab = esaj.page.intim.Tabela(driver=driver)
+    # tab.get_table(
+    #     # instancia=instancia,
+    #     seleciona_processos=[processo],
+    # )
+
+    # acao = esaj.page.intim.Acoes(driver=driver)
+    # csv_filepath = acao.export_csv()
+    # print(csv_filepath)
+
+    # driver.quit()

pyFDBS/scraper/webdriver/config.py

@@ -0,0 +1,15 @@
+"""
+Módulo com variáveis sobre a definição do driver
+E extensões do driver.
+"""
+
+# Nome da pasta que ficará dentro da pasta temp.
+# Nessa pasta serão armazenados logs e drivers do Selenium
+TEMP_PATH_NAME = 'fbds'
+
+# Url do GeckoDriver
+URL_GECKODRIVER_WINDOWS = 'https://github.com/mozilla/geckodriver/releases/download/v0.32.0/geckodriver-v0.32.0-win64.zip'
+URL_GECKODRIVER_LINUX = 'https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz'
+
+# URLs da Extensão xPath
+URL_ADDONS_XPATH = 'https://addons.mozilla.org/firefox/downloads/file/3588871/xpath_finder-1.0.2-fx.xpi'

pyFDBS/scraper/webdriver/firefox.py

@@ -0,0 +1,174 @@
+"""
+Módulo para usar driver do FireFox
+
+Michel Metran
+Data: mar.2023
+Atualizado em: mar.2023
+"""
+
+import tempfile
+from pathlib import Path
+
+import requests
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options as FirefoxOptions
+from selenium.webdriver.firefox.service import Service as FirefoxService
+
+from . import config, gecko
+
+
+class Firefox(webdriver.Firefox):
+    """
+    Cria driver customizado do Selenium
+
+    :param webdriver: _description_
+    :type webdriver: _type_
+    """
+
+    def __init__(
+            self,
+            # driver_path: Path,
+            # logs_path: Path,
+            # down_path: Path,
+            *args,
+            **kwargs,
+    ):
+        """
+        - verify_ssl
+        - headless
+        - download_path
+
+        :param my_driver_path: _description_
+        :type my_driver_path: pathlib
+        :param my_logs_path: _description_
+        :type my_logs_path: pathlib
+        """
+
+        # Parameters
+        verify_ssl = kwargs.get('verify_ssl', True)
+        headless = kwargs.get('headless', False)
+        self.download_path = kwargs.get('download_path', False)
+
+        # Temp Path
+        temp_path = tempfile.gettempdir()
+        project_temp_path = Path(temp_path) / config.TEMP_PATH_NAME
+
+        # Scrapy Path
+        scrapy_path = project_temp_path / 'scrapy'
+        scrapy_path.mkdir(exist_ok=True, parents=True)
+
+        # Driver Path
+        self.driver_path = scrapy_path / 'driver'
+        self.driver_path.mkdir(exist_ok=True, parents=True)
+
+        # Logs Path
+        self.logs_path = scrapy_path / 'logs'
+        self.logs_path.mkdir(exist_ok=True, parents=True)
+
+        # Services
+        geckodriver = gecko.Gecko()
+        gecko_path = geckodriver.get_path_geckodriver(verify_ssl=verify_ssl)
+
+        # Logs
+        logs_filepath = self.logs_path / 'geckodriver.log'
+
+        # Services
+        my_service = FirefoxService(
+            executable_path=gecko_path, log_path=logs_filepath
+        )
+
+        # Options
+        my_options = FirefoxOptions()
+        if headless:
+            my_options.add_argument('--headless')
+
+        # Download Path
+        if self.download_path is False:
+            # Cria Pasta
+            self.download_path = scrapy_path / 'download'
+            self.download_path.mkdir(exist_ok=True, parents=True)
+
+        # Define pasta de Download
+        my_options.set_preference(
+            'browser.download.dir', str(self.download_path)
+        )
+
+        my_options.set_preference('intl.accept_languages', 'pt-BR, pt')
+        my_options.set_preference('browser.download.folderList', 2)
+        my_options.set_preference(
+            'browser.download.manager.showWhenStarting', False
+        )
+        my_options.set_preference('pdfjs.disabled', True)
+        my_options.set_preference('plugin.scan.Acrobat', '99.0')
+        my_options.set_preference('plugin.scan.plid.all', False)
+        my_options.set_preference(
+            'browser.helperApps.showOpenOptionForPdfJS', False
+        )
+        my_options.set_preference('browser.download.forbid_open_with', True)
+        my_options.set_preference(
+            'browser.helperApps.neverAsk.saveToDisk',
+            'application/octet-stream, application/pdf',
+        )
+        # Quando é necessário fazer repost about: config
+        # https://superuser.com/questions/1410598/disable-firefox-must-send-information-that-will-repeat-any-action-dialog-box
+        my_options.set_preference(
+            'dom.confirm_repost.testing.always_accept', True
+        )
+
+        # Driver
+        # my_driver = super(Driver, self)
+        # my_driver.__init__(service=my_service, options=my_options)
+        super().__init__(service=my_service, options=my_options)
+        self.maximize_window()
+
+    def add_extension_xpath(self) -> None:
+        """
+        Adiciona xPath extension
+        """
+
+        # Temp Path
+        temp_path = tempfile.gettempdir()
+        project_temp_path = Path(temp_path) / config.TEMP_PATH_NAME
+
+        # Scrapy Path
+        scrapy_path = project_temp_path / 'scrapy'
+        scrapy_path.mkdir(exist_ok=True, parents=True)
+
+        # Adds Path
+        adds_path = scrapy_path / 'adds'
+        adds_path.mkdir(exist_ok=True, parents=True)
+
+        # Add-ons Xpath
+        xpath_path = adds_path / 'xpath.xpi'
+        xpath_path = xpath_path.absolute().resolve()
+
+        # Download
+        if not xpath_path.is_file():
+            r = requests.get(
+                url=config.URL_ADDONS_XPATH, timeout=60, verify=False
+            )
+
+            with open(xpath_path, 'wb') as f:
+                f.write(r.content)
+
+        # Add
+        self.install_addon(str(xpath_path), temporary=True)
+
+
+if __name__ == '__main__':
+    import time
+
+    # from pyesaj import paths
+    # Instancia Driver
+    driver = Firefox(headless=False, verify_ssl=False, )
+
+    # Add xPath Extension
+    driver.add_extension_xpath()
+
+    driver.get(url='https://www.uol.com.br/')
+    time.sleep(3)
+    driver.quit()
+
+    gecko = gecko.Gecko()
+    gecko_p = gecko.get_path_geckodriver()
+    print(gecko_p)

pyFDBS/scraper/webdriver/gecko.py

@@ -0,0 +1,151 @@
+"""
+_summary_
+
+:raises Exception: _description_
+:return: _description_
+:rtype: _type_
+"""
+
+import platform
+import tarfile
+import tempfile
+from pathlib import Path
+from zipfile import ZipFile
+
+import requests
+
+from . import config
+
+
+class Gecko:
+    """
+    _summary_
+    """
+
+    def __init__(self) -> None:
+        """
+        _summary_
+
+        :return: _description_
+        :rtype: _type_
+        """
+
+        # Temp Path
+        temp_path = tempfile.gettempdir()
+        project_temp_path = Path(temp_path) / config.TEMP_PATH_NAME
+
+        # Scrapy Path
+        scrapy_path = project_temp_path / 'scrapy'
+        scrapy_path.mkdir(exist_ok=True, parents=True)
+
+        # Driver Path
+        self.driver_path = scrapy_path / 'driver'
+        self.driver_path.mkdir(exist_ok=True, parents=True)
+
+        # Drivers por Sistema Operacional
+        if platform.system() == 'Windows':
+            gecko_win_filepath = self.driver_path / 'geckodriver.exe'
+            if gecko_win_filepath.is_file():
+                self.has_geckodriver = True
+
+            elif not gecko_win_filepath.is_file():
+                self.has_geckodriver = False
+
+        elif platform.system() == 'Linux':
+            gecko_linux_filepath = self.driver_path / 'geckodriver'
+            if gecko_linux_filepath.is_file():
+                self.has_geckodriver = True
+
+            elif not gecko_linux_filepath.is_file():
+                self.has_geckodriver = False
+
+    def _get_geckodriver(self, verify_ssl: bool):
+        """
+        Faz o download do geckodriver!
+
+        :return:
+        """
+
+        # print(gecko_zip_filepath)
+        if not self.has_geckodriver:
+            if platform.system() == 'Windows':
+                # Download do geckodriver
+                r = requests.get(
+                    config.URL_GECKODRIVER_WINDOWS,
+                    timeout=60,
+                    verify=verify_ssl,
+                )
+
+                # Save
+                name_gecko = Path(config.URL_GECKODRIVER_WINDOWS).name
+                gecko_zip_filepath = self.driver_path / name_gecko
+                with open(gecko_zip_filepath, 'wb') as f:
+                    f.write(r.content)
+
+                # Extract
+                with ZipFile(gecko_zip_filepath, 'r') as zip_ref:
+                    zip_ref.extractall(self.driver_path)
+
+            elif platform.system() == 'Linux':
+                # Download do geckodriver
+                r = requests.get(
+                    config.URL_GECKODRIVER_LINUX, timeout=60, verify=verify_ssl
+                )
+
+                # Save
+                name_gecko = Path(config.URL_GECKODRIVER_LINUX).name
+                gecko_zip_filepath = self.driver_path / name_gecko
+                with open(gecko_zip_filepath, 'wb') as f:
+                    f.write(r.content)
+
+                # Extract
+                with tarfile.open(gecko_zip_filepath, 'r') as tar_ref:
+                    tar_ref.extractall(self.driver_path)
+
+        elif self.has_geckodriver:
+            print(f'Geckodriver already in {self.driver_path}')
+
+    def get_path_geckodriver(self, verify_ssl: bool = False) -> Path | str:
+        """
+        dddd
+        """
+
+        # Faz download se for necessário
+        self._get_geckodriver(verify_ssl=verify_ssl)
+
+        # Path
+        if platform.system() == 'Windows':
+            _gecko_path = self.driver_path / 'geckodriver.exe'
+            _gecko_path = _gecko_path.resolve().as_posix().replace('/', '\\')
+            return _gecko_path
+
+        elif platform.system() == 'Linux':
+            _gecko_path = self.driver_path / 'geckodriver'
+            return _gecko_path
+
+        else:
+            raise Exception(f'Ajustar para plataforma {platform.system()}')
+
+
+if __name__ == '__main__':
+    import time
+
+    from pyesaj import webdriver
+
+    # Instancia Driver
+    driver = webdriver.Firefox(
+        # driver_path=paths.driver_path,
+        # logs_path=paths.app_logs_path,
+        # down_path=paths.driver_path,
+        headless=False,
+        verify_ssl=False,
+    )
+    # Add xPath Extension
+    driver.add_extension_xpath()
+    driver.get(url='https://www.uol.com.br/')
+    time.sleep(3)
+    driver.quit()
+
+    gecko = Gecko()
+    gecko_path = gecko.get_path_geckodriver()
+    print(gecko_path)

pyfbds-0.1.0.dist-info/METADATA

@@ -0,0 +1,55 @@
+Metadata-Version: 2.4
+Name: pyFBDS
+Version: 0.1.0
+Summary: Add your description here
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.32.5
+Requires-Dist: requests-cache>=1.2.1
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: lxml>=6.0.0
+Requires-Dist: ipywidgets>=8.1.8
+Requires-Dist: aiohttp>=3.13.2
+Requires-Dist: nest-asyncio>=1.6.0
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: selenium>=4.38.0
+Dynamic: license-file
+
+# pyFBDS
+
+[![Repo](https://img.shields.io/badge/GitHub-repo-blue?logo=github&logoColor=f5f5f5)](https://github.com/open-geodata/pyFBDS)
+[![PyPI - Version](https://img.shields.io/pypi/v/pyfbds?logo=pypi&label=PyPI&color=blue)](https://pypi.org/project/pyfbds/)<br>
+[![Read the Docs](https://img.shields.io/readthedocs/pyFBDS?logo=ReadTheDocs&label=Read%20The%20Docs)](https://pyFBDS.readthedocs.io/)
+[![Publish Python to PyPI](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml/badge.svg)](https://github.com/michelmetran/pyFBDS/actions/workflows/publish-to-pypipoetry.yml)
+
+_Scripts_ para obter dados espaciais do [**repositório público de mapas e _shapefiles_ para _download_**](https://geo.fbds.org.br/). Veja mais na documentação:
+
+> [https://pyFBDS.readthedocs.io/](https://pyFBDS.readthedocs.io/)
+
+<br>
+
+---
+
+## Como Instalar?
+
+```shell
+pip3 install pyFBDS
+```
+
+<br>
+
+---
+
+## Como Usar?
+
+```shell
+# Importa pacote
+import pyFBDS
+
+# Instancia Objeto
+fbds = pyFBDS.Repo(output_path='.')
+
+# Chama o método
+fbds.get_municipio(id_ibge=353243)
+```

pyfbds-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+pyFDBS/__init__.py,sha256=zUuCifHMhjcfy-5UfEK8FqJBqyRVpObhZzZFfUn7080,127
+pyFDBS/requests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyFDBS/requests/cache.py,sha256=9fxhr62qOzqH-3yfMWgxWFfwJRRTLD09YzWhHDAmLhQ,1009
+pyFDBS/requests/download.py,sha256=mps8mzvG8Dc0Rmlr0YLVv9bdtTLGbKTH3upRKDknWZ4,4324
+pyFDBS/requests/logger.py,sha256=jtR2tzII8WXTJSliKsMOA005YVqQX--fkNI5hmRvYMs,5807
+pyFDBS/requests/teste.py,sha256=H9U5Crhg_bHJfLrU2gHSNGhDi6xgdGD21t-GVPcAznA,298
+pyFDBS/requests/web.py,sha256=PqRRd2ty6zMAXfyV_g6m2V2GaLBNmhYYPY41Zug-RuY,2451
+pyFDBS/scraper/__init__.py,sha256=hY_8LuHGs7u4O4sEdLJ7l4jaBmgh2bwrnUxx85g_7W4,55
+pyFDBS/scraper/page/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyFDBS/scraper/page/sss.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyFDBS/scraper/webdriver/__init__.py,sha256=n0Hqj0eP5RBQicZ2sBt7A8JSNNPBWR7vAq9sWoMbsfY,56
+pyFDBS/scraper/webdriver/chrome.py,sha256=ZchR5qgWNZDb0tcVJifdnxGhQtXp3CyOE_YZ_hULegk,9775
+pyFDBS/scraper/webdriver/config.py,sha256=ebHxrHMNjQRQtXRqqevRlU0yxg37i5zj3d7_nj3gCTw,623
+pyFDBS/scraper/webdriver/firefox.py,sha256=3KJM6Ud2iGRNav5Xv0UdDn-6mhKJKqL6jD3Lqzl2v_4,5088
+pyFDBS/scraper/webdriver/gecko.py,sha256=75aOODWkfV8WWaUGhdyYrSOTzMmRDbtf34so82AU7tU,4263
+pyfbds-0.1.0.dist-info/licenses/LICENSE,sha256=_iz4azhfrEKs8_YrVR8otmznlfBagqvUygzZuV7FFjM,1090
+pyfbds-0.1.0.dist-info/METADATA,sha256=mH5GYdbCMxvZ5C2SqRibi2-2fKVqojSc6oWe9snlZ10,1561
+pyfbds-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+pyfbds-0.1.0.dist-info/top_level.txt,sha256=OlAFr-ScJuI1s1bYukiWsBvf2_SUErG5GUFYNXxsrlM,7
+pyfbds-0.1.0.dist-info/RECORD,,

pyfbds-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pyfbds-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Open Geodata
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyfbds-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pyFDBS