luxorasap 0.0.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

luxorasap/__init__.py

@@ -13,7 +13,7 @@ from types import ModuleType
 try:
     __version__: str = metadata.version(__name__)
 except metadata.PackageNotFoundError:  # editable install
-    __version__ = "0.0.2"
+    __version__ = "0.1.0"
 
 # ─── Lazy loader ─────────────────────────────────────────────────
 def __getattr__(name: str) -> ModuleType:

luxorasap/datareader/core.py

@@ -1,6 +1,4 @@
 # Imports
-__version__ = "0.0.1"
-
 import pandas as pd
 import datetime as dt
 from datetime import timezone
@@ -9,20 +7,16 @@ import os, sys
 import time
 import numpy as np
 from scipy.optimize import newton
-from pathlib import Path
 import io
-try:
-    from azure.storage.blob import BlobServiceClient
-    import pyarrow as pa
-    import pyarrow.parquet as pq
-    from dotenv import load_dotenv
-    load_dotenv()
-except ImportError:
-    print("Favor instalar as dependencias abaixo:")
-    print('pip install azure-storage-blob')
-    print('pip install pyarrow')
-    print('pip install python-dotenv')
-    sys.exit()
+from dotenv import load_dotenv
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from luxorasap.utils.storage import BlobParquetClient
+load_dotenv()
+
+# Nao fazer import do ingest, risco de impor circular.
 
 
 #ADLS_CONNECTION_STRING = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
@@ -32,22 +26,16 @@ class LuxorQuery:
 
     def __init__(self, update_mode="optimized", is_develop_mode=False, tables_path=None, 
                  blob_directory='enriched/parquet', adls_connection_string:str=None, container_name="luxorasap"):
-
         """
             update_mode: 
                 'standard' - Carrega todas as tabelas disponiveis
                 'optimized' - Carrega apenas as tabelas utilizadas sob demanda
         """
-
-        if adls_connection_string is None:
-            adls_connection_string = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
             
+        self.blob_client = BlobParquetClient(adls_connection_string=adls_connection_string,
+                                container=container_name)
         self.blob_directory = blob_directory
-        self.container_name = container_name
 
-        self.CONNECTION_STRING = adls_connection_string
-        self.blob_service_client = BlobServiceClient.from_connection_string(self.CONNECTION_STRING)
-        self.container_client = self.blob_service_client.get_container_client(self.container_name)
         
         self.modified_tables = []
         self.is_develop_mode = is_develop_mode
@@ -71,40 +59,12 @@ class LuxorQuery:
         self.update() # Nessa 1° exec. vai inicializar os dicionarios acima
 
 
-
-    #def __set_tables_path(self):
-    #    
-    #    cur_dir = Path().absolute()
-    #    cur_dir_items = cur_dir.parts
-    #    home = cur_dir.home()
-    #    onedrive_name = cur_dir_items[3]
-    #    if "OneDrive".lower() not in onedrive_name.lower():
-    #        logger.critical("No formato atual, para utilizar esse modulo a main deve ser executada dentro do diretorio do OneDrive.")
-    #    #formato considerado eh: "C:/Users/{user}/{onedrive_name}"
-    #    
-    #    tables_path = Path(home)/onedrive_name/"projetos"/"LuxorASAP"/"luxorDB"/"tables"
-    #    if self.is_develop_mode:
-    #        tables_path = Path(home)/onedrive_name/"projetos"/"LuxorASAP_Develop"/"luxorDB"/"tables"
-    #
-    #    return tables_path
-
-
-    def __get_blob_update_time(self, table_name):
-            
-            blob_name = f"{self.blob_directory}/{table_name}.parquet"
-            blob_client = self.blob_service_client.get_blob_client(container=self.container_name, blob=blob_name)
-
-            # Obter as propriedades do blob
-            properties = blob_client.get_blob_properties()
-            return properties['last_modified'].replace(tzinfo=timezone.utc).timestamp()
-
-
     def __is_table_modified(self, table_name):
         """ Retorna 'True' ou 'False' informando se a tabela informada em 'table_name' foi criada ou modificada.
 
         Args:
             table_name (str): nome tabela
-
+            table_path (str): caminho ate a tabela no blob
         Returns:
             bool: True se foi criada ou modificada
         """
@@ -114,32 +74,13 @@ class LuxorQuery:
         
         try:
             file_path = self.tables_in_use[table_name]["table_path"]
-            file_last_update = self.__get_blob_update_time(table_name)
+            file_last_update = self.blob_client.get_df_update_time(file_path)#self.__get_blob_update_time(table_name)
             return file_last_update > self.tables_in_use[table_name]["update_time"]
 
         except:
             logger.critical(f"Arquivo <{file_path}> não encontrado.")
 
         return False
-
-
-    def __persist_column_formatting(self, t):
-
-        columns_to_persist = {"Name", "Class", "Vehicles", "Segment"}
-
-        if len(set(t.columns).intersection(columns_to_persist)) > 0:
-            # Vamos persistir a formatacao de algumas colunas
-            columns_order = list(t.columns)
-            columns_to_persist = list(set(t.columns).intersection(columns_to_persist))
-            persistent_data = t[columns_to_persist].copy()
-
-            columns_to_normalize = list(set(columns_order) - set(columns_to_persist))
-            t = self.text_to_lowercase(t[columns_to_normalize])
-            t.loc[:,columns_to_persist] = persistent_data
-            return t[columns_order]
-        
-        # Nos outros casos, transformaremos tudo em lowercase
-        return self.text_to_lowercase(t)
          
     
     def __get_tickers_bbg(self):
@@ -152,15 +93,13 @@ class LuxorQuery:
     def table_exists(self, table_name, blob_directory=None):
         # Checa no ADLS se existe alguma tabela com o nome informado
         
-        table_path = f"{blob_directory}/{table_name}.parquet"
         if blob_directory is None:
             blob_directory = self.blob_directory
-            table_path = f"{blob_directory}/{table_name}.parquet"
+        
+        table_path = f"{blob_directory}/{table_name}.parquet"
             
-        blob_client = self.container_client.get_blob_client(table_path)
+        return self.blob_client.table_exists(table_path)
 
-        return blob_client.exists()
-        
 
     def get_table(self, table_name, index=False, index_name="index", dtypes_override={}, force_reload=False):
         """
@@ -189,6 +128,7 @@ class LuxorQuery:
             dtypes_override: dict : set - Dicionario com os tipos de dados das colunas devem ser sobrescritos.
                 Deve possuir as chaves 'float', 'date', 'bool' e 'str_nan_format'(troca 'nan' por pd.NA)
                     Para cada chave, colocar um Set com os nomes das colunas que receberao o cast.
+            
         """
         table_name = table_name.lower().replace(" ", "_")
         if table_name == 'bbg_tickers': return self.__get_tickers_bbg() # DEPRECATED TODO: remover apos testes
@@ -201,47 +141,24 @@ class LuxorQuery:
 
 
     def __load_table(self, table_name, index=False, index_name="index", dtypes_override={}):
-        
-
-        def __read_blob_parquet(table_name):
-
-            container_name = "luxorasap"
-            blob_name = f"{self.blob_directory}/{table_name}.parquet"
-
-            blob_client = self.blob_service_client.get_blob_client(container=container_name, blob=blob_name)
-
-            # Download do blob em memória
-            download_stream = None
-            try:
-                download_stream = blob_client.download_blob()
-            except Exception:
-                print(f"Tabela '{table_name}' não encontrada no blob.")
-                return None, False
-            parquet_data = download_stream.readall()
-
-            # Ler o parquet do stream em memória
-            parquet_buffer = io.BytesIO(parquet_data)
-            table = pq.read_table(parquet_buffer)
-            df = table.to_pandas()
-
-            return df, True
     
-
         def __load_parquet(table_name):
             table_path = f"{self.blob_directory}/{table_name}.parquet"#self.tables_path/"parquet"/f"{table_name}.parquet"
             
             try:
                 #update_time = os.path.getmtime(table_path)
-                update_time = self.__get_blob_update_time(table_name)
                 table_data = None
                 # Primeiro, vamos tentar ler do blob
-                table_data, blob_read_success = __read_blob_parquet(table_name)
-
+                table_data, blob_read_success = self.blob_client.read_df(table_path)#__read_blob_parquet(table_name)
+        
                 if not blob_read_success:
                     logger.critical(f"Não foi possível carregar a tabela '{table_name}' do blob.")
                     #print("--> Onedrive fallback.")
                     #table_data = pd.read_parquet(table_path,engine="fastparquet")
-
+                update_time = self.blob_client.get_df_update_time(table_path)
+                
+                assert(table_data is not None)
+                
                 table_columns = set(table_data.columns)
                 
                 float_dtypes = {"Last_Price", "Price", "px_last", "Quota", "#", "Avg_price", "Variation", "Variation_tot",
@@ -339,7 +256,7 @@ class LuxorQuery:
             except Exception:
                 logger.error(f"Nao foi possível setar a coluna {index_name} como index para a tabela {table_name}.")
 
-        table_data = self.__persist_column_formatting(table_data)
+        #table_data = self.__persist_column_formatting(table_data)
 
         self.tables_in_use[table_name] = {"table_data" : table_data,
                                           "table_path" : table_path,
@@ -2909,19 +2826,6 @@ class LuxorQuery:
         assets = self.get_table("assets")
         name = assets.query("Ticker == @ticker")["Name"].squeeze()
         return name
-        
-    
-    def list_blob_files(self, sub_dir, ends_with=None):
-        """
-            Lista todos os arquivos dentro de um diretorio no blob storage.
-        """
-        
-        # Vamos listar os arquivos do diretorio
-        blob_files = self.container_client.list_blobs(name_starts_with=sub_dir)
-        
-        if ends_with is not None:
-            return [blob_file.name for blob_file in blob_files if blob_file.name.endswith(ends_with)]
-        return [blob_file.name for blob_file in blob_files]
                 
                 
     def simulate_portfolio_performance(self, portfolio: dict, portfolio_date: dt.date, adm_fee: float, performance_fee: float = 0):

luxorasap/ingest/__init__.py

@@ -0,0 +1,23 @@
+"""Exporta a API “cloud” por padrão e mantém DataLoader legado disponível."""
+
+from importlib import import_module
+from warnings import warn
+
+# API moderna (recomendada)
+from .cloud import save_table, incremental_load  # noqa: F401
+
+__all__ = ["save_table", "incremental_load"]
+
+# Ponte para o loader antigo -------------------------------------------------
+try:
+    legacy_mod = import_module(".legacy_local.dataloader", __name__)
+    DataLoader = legacy_mod.DataLoader  # noqa: F401
+    warn(
+        "luxorasap.ingest.DataLoader está legado e será descontinuado; "
+        "migre para luxorasap.ingest.save_table / incremental_load.",
+        DeprecationWarning,
+        stacklevel=1,
+    )
+except Exception:
+    # se o arquivo legado não existir, simplesmente não exporta
+    pass

luxorasap/ingest/cloud/__init__.py

@@ -0,0 +1,54 @@
+"""Camada moderna de ingestão: grava / incrementa tabelas em ADLS (Parquet)."""
+
+import pandas as pd
+
+from luxorasap.utils.storage import BlobParquetClient
+from luxorasap.utils.dataframe import prep_for_save
+from luxorasap.datareader import LuxorQuery
+
+__all__ = ["save_table", "incremental_load"]
+
+_client = BlobParquetClient()   # instância única para o módulo
+
+
+# ────────────────────────────────────────────────────────────────
+def save_table(
+    table_name: str,
+    df,
+    *,
+    index: bool = False,
+    index_name: str = "index",
+    normalize_columns: bool = False,
+    directory: str = "enriched/parquet",
+):
+    """Salva DataFrame como Parquet em ADLS (sobrescrevendo)."""
+    df = prep_for_save(df, index=index, index_name=index_name, normalize=normalize_columns)
+    _client.write_df(df.astype(str), f"{directory}/{table_name}.parquet")
+
+
+def incremental_load(
+    lq: LuxorQuery,
+    table_name: str,
+    df,
+    *,
+    increment_column: str = "Date",
+    index: bool = False,
+    index_name: str = "index",
+    normalize_columns: bool = False,
+    directory: str = "enriched/parquet",
+):
+    """Concatena novos dados aos existentes, cortando duplicados pela data."""
+    if lq.table_exists(table_name):
+        prev = lq.get_table(table_name)
+        cutoff = df[increment_column].max()
+        prev = prev.query(f"{increment_column} < @cutoff")
+        df = pd.concat([prev, df], ignore_index=True)
+
+    save_table(
+        table_name,
+        df,
+        index=index,
+        index_name=index_name,
+        normalize_columns=normalize_columns,
+        directory=directory,
+    )

luxorasap/ingest/legacy_local/dataloader.py

@@ -0,0 +1,280 @@
+import pandas as pd
+import datetime as dt
+import time
+import io
+import sys, os
+
+from loguru import logger
+from pathlib import Path
+
+from azure.storage.blob import BlobServiceClient
+import pyarrow as pa
+import pyarrow.parquet as pq
+from dotenv import load_dotenv
+load_dotenv()
+
+from luxorasap.datareader import LuxorQuery
+from luxorasap.utils.dataframe import transforms
+
+import warnings
+warnings.warn(
+    "luxorasap.ingest.legacy_local.dataloader é legado; "
+    "utilize luxorasap.ingest.cloud em novas rotinas.",
+    DeprecationWarning, stacklevel=2
+)
+
+
+
+class DataLoader:
+
+    def __init__(self, luxorDB_directory = None):
+        """Fornece uma forma padronizada de carregar tabelas para a luxorDB.
+            1. Possui metodos para carregar tabelas que já estao carregadas na memória
+                - Sao os metodos que possuem 'table' no nome
+            2. Possui metodos para carregar arquivos de excel, com todas as suas abas
+                Inclui metodo para checagem de alteracao de versao do arquivo
+                - Sao os metodos que possuem 'file' no nome
+        Args:
+            luxorDB_directory (pathlib.Path, optional): Caminho completo ate o diretorio de destino dos dados.
+        """
+        self.luxorDB_directory = luxorDB_directory
+
+        if self.luxorDB_directory is None:
+            self.luxorDB_directory = Path(__file__).absolute().parent/"LuxorDB"/"tables"
+
+        self.tracked_files = {}
+        self.tracked_tables = {}
+    
+
+    def add_file_tracker(self, tracked_file_path, filetype="excel", sheet_names={}, 
+            excel_size_limit = None,index=False, index_name="index",normalize_columns=False):
+        """ Adiciona arquivo na lista para checar por alteracao
+        Args:
+            tracked_file_path (pathlib.Path): caminho completo ate o arquivo,
+                    incluindo nome do arquivo e extensão.
+            sheet_names (dict, optional): Caso seja uma planilha com varias abas, mapear
+                    aqui o nome da aba para o nome do arquivo de saida desejado.
+        """
+        if tracked_file_path not in self.tracked_files:
+            self.tracked_files[tracked_file_path] = {
+                    "last_mtime" : dt.datetime.timestamp(dt.datetime(2000,1,1)),
+                    "filetype" : filetype, "sheet_names": sheet_names,
+                    "excel_size_limit" : excel_size_limit,
+                    "index" : index,
+                    "index_name" : index_name,
+                    "normalize_columns" : normalize_columns,
+                }
+
+
+    def add_table_tracker(self, table_name:str):
+        """ Adiciona tabela na lista para controle de alteracao."""
+
+        if table_name not in self.tracked_tables:
+            self.tracked_tables[table_name] = dt.datetime.timestamp(dt.datetime(2000,1,1))
+
+
+    def remove_file_tracker(self, tracked_file_path):
+
+        if tracked_file_path in self.tracked_files:
+            del self.tracked_files[tracked_file_path]
+    
+
+    def remove_table_tracker(self, table_name:str):
+
+        if table_name in self.tracked_tables:
+            del self.tracked_tables[table_name]
+
+
+    def is_file_modified(self, tracked_file_path: Path) -> {bool, float}:
+        """ Checa se o arquivo foi modificado desde a ultima leitura.
+        Returns:
+            tuple(bool, float): (foi modificado?, timestamp da ultima modificacao)
+        """
+
+        file_data = self.tracked_files[tracked_file_path]
+        
+        last_saved_time = file_data["last_mtime"]
+        file_last_update = tracked_file_path.stat().st_mtime
+        return file_last_update > last_saved_time, file_last_update
+    
+
+    def set_file_modified_time(self, tracked_file_path, file_mtime):
+
+        self.tracked_files[tracked_file_path]["last_mtime"] = file_mtime
+        
+
+    def load_file_if_modified(self, tracked_file_path, export_to_blob=False, blob_directory='enriched/parquet'):
+        """Carrega arquivo no caminho indicado, carregando na base de dados caso modificado.
+        Args:
+            tracked_file_path (pathlib.Path): caminho ate o arquivo(cadastro previamente por add_file_tracker)
+            type_map (_type_, optional): _description_. Defaults to None.
+            filetype (str, optional): _description_. Defaults to "excel".
+        """
+        file_data = self.tracked_files[tracked_file_path]
+        
+        last_saved_time = file_data["last_mtime"]
+        filetype = file_data["filetype"]
+        file_sheets = file_data["sheet_names"]
+
+        file_last_update = tracked_file_path.stat().st_mtime
+
+        if file_last_update > last_saved_time: # Houve alteracao no arquivo
+            if filetype == "excel":
+                file_sheets = None if len(file_sheets) == 0 else list(file_sheets.keys())
+
+                # tables sera sempre um dicionario de tabelas
+                tables = None
+                trials = 25
+                t_counter = 1
+                while trials - t_counter > 0:
+                    try:
+                        tables = pd.read_excel(tracked_file_path, sheet_name=file_sheets)
+                        t_counter = trials # leitura concluida
+                    except PermissionError:
+                        
+                        logger.error(f"Erro ao tentar ler arquivo '{tracked_file_path}.\nTentativa {t_counter} de {trials};'.\nSe estiver aberto feche.")
+                        time.sleep(10)
+                        t_counter += 1
+
+                for sheet_name, table_data in tables.items():
+
+                    table_name = sheet_name if file_sheets is None else file_data["sheet_names"][sheet_name]
+
+                    if table_name == "trades":
+                        table_data["ID"] = table_data.index
+
+                    self.__export_table(table_name, table_data, index=file_data["index"], index_name=file_data["index_name"],
+                                            normalize_columns=file_data["normalize_columns"], export_to_blob=export_to_blob,
+                                            blob_directory=blob_directory)
+                self.tracked_files[tracked_file_path]["last_mtime"] = file_last_update
+            
+        
+    def load_table_if_modified(self, table_name, table_data, last_update, index=False, index_name="index", normalize_columns=False,
+                               do_not_load_excel=False, export_to_blob=False, blob_directory='enriched/parquet',
+                               is_data_in_bytes=False, bytes_extension=".xlsx"):
+        """
+        Args:
+            table_name (str): nome da tabela (sera o mesmo do arquivo a ser salvo)
+            table_data (pd.DataFrame): tabela de dados
+            last_update (timestamp): timestamp da ultima edicao feita na tabela
+        """
+
+        if table_name not in self.tracked_tables:
+            self.add_table_tracker(table_name)
+
+
+        last_update_time = self.tracked_tables[table_name]
+        if last_update > last_update_time:
+            
+            self.tracked_tables[table_name] = last_update
+            self.__export_table(table_name, table_data, index=index, index_name=index_name, normalize_columns=normalize_columns,
+                                do_not_load_excel=do_not_load_excel, export_to_blob=export_to_blob, blob_directory=blob_directory,
+                                is_data_in_bytes=is_data_in_bytes, bytes_extension=bytes_extension)
+
+
+    def scan_files(self, export_to_blob=False, blob_directory='enriched/parquet'):
+        """
+            Para todos os arquivos cadastrados, vai buscar e carregar quando houver
+            arquivo mais recente.
+        """
+
+        for file in self.tracked_files:
+            
+            self.load_file_if_modified(file, export_to_blob=export_to_blob, blob_directory=blob_directory)
+
+
+    #def __load_bytes(self, content: bytes, extension=".xlsx") -> pd.DataFrame:
+    #    if extension == ".xlsx" or extension == "xlsx" or extension == "xls":
+    #        df = pd.read_excel(io.BytesIO(content), engine="openpyxl")
+    #    
+    #        return df
+#
+    #    raise ValueError(f'Extension {extension} not supported')
+    
+    def __load_bytes(self, content: bytes, extension: str) -> pd.DataFrame:
+        extension = extension.lower()
+
+        if extension in [".xlsx", ".xls", "xlsx", "xls"]:
+            df = pd.read_excel(io.BytesIO(content), engine="openpyxl")
+            return df
+        
+        if extension == ".csv":
+            try:
+                return pd.read_csv(io.BytesIO(content), encoding="utf-8")
+            except UnicodeDecodeError:
+                return pd.read_csv(io.BytesIO(content), encoding="latin1")
+
+        if extension == ".parquet":
+            df = pd.read_parquet(io.BytesIO(content))
+            return df
+
+        raise ValueError(f'Extension {extension} not supported')
+
+
+    def __export_table(self, table_name, table_data, index=False, index_name="index", normalize_columns=False,
+                       do_not_load_excel=False, export_to_blob=False, blob_directory='enriched/parquet',
+                       is_data_in_bytes=False, bytes_extension=".xlsx"):
+        
+        dest_directory = self.luxorDB_directory
+        #TODO -> formatar para index=False
+        # Salvando em formato excel
+        attempts = 10
+        count_attempt = 0
+
+        if is_data_in_bytes:
+            table_data = self.__load_bytes(table_data, extension=bytes_extension)
+
+        # Se o index tiver dados, vamos trata-los para virar uma coluna
+        if index:
+            # Tratando o nome do index, caso seja necessario transformar em coluna
+            prev_index = table_data.index.name
+            if prev_index is not None and index_name == "index":
+                index_name = prev_index
+            table_data.index.name = index_name
+
+            table_data = table_data.reset_index()
+
+
+        if normalize_columns:
+            table_data = transforms.persist_column_formatting(table_data)
+
+        if not do_not_load_excel:
+            while count_attempt < attempts:
+                count_attempt += 1
+                try:
+                    if len(table_data) > 1_000_000:
+                        table_data = table_data.tail(1_000_000)
+                    table_data.to_excel(dest_directory/f"{table_name}.xlsx", index=False)
+                    count_attempt = attempts # sair do loop
+                    
+                except PermissionError:
+                    logger.error(f"Erro ao tentar salvar arquivo {table_name}. Feche o arquivo. Tentativa {count_attempt} de {attempts}")
+                    time.sleep(10 + count_attempt * 5)
+
+        # Salvando em csv 
+        # -> Salvar em csv foi descontinuado por falta de uso.
+        #table_data.to_csv(dest_directory/"csv"/f"{table_name}.csv", sep=";", index=False)
+
+        # Salvando em parquet (tudo como string... dtypes deverao ser atribuidos na leitura)
+        table_data = table_data.astype(str)
+        table_data.to_parquet(dest_directory/"parquet"/f"{table_name}.parquet", engine="pyarrow", index=False)
+
+        if export_to_blob:
+            # Definindo o Container e o Blob Name
+            container_name = "luxorasap"
+            blob_name = f"{blob_directory}/{table_name}.parquet"  #
+            
+            # Conversão para parquet em memória (sem precisar salvar local)
+            table = pa.Table.from_pandas(table_data)
+            parquet_buffer = io.BytesIO()
+            pq.write_table(table, parquet_buffer)
+            parquet_buffer.seek(0)  # Reseta o ponteiro para o início do buffer
+
+            # Conectando ao Blob Storage
+            connection_string = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
+            blob_service_client = BlobServiceClient.from_connection_string(conn_str=connection_string)
+
+            # Criando um Blob Client
+            blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
+            blob_client.upload_blob(parquet_buffer, overwrite=True)
+        

luxorasap/utils/dataframe/__init__.py

@@ -0,0 +1,2 @@
+from .transforms import prep_for_save, persist_column_formatting, text_to_lowercase
+__all__ = ["prep_for_save", "persist_column_formatting", "text_to_lowercase"]

luxorasap/utils/dataframe/transforms.py

@@ -0,0 +1,54 @@
+import pandas as pd
+from ..storage.blob import BlobParquetClient   # ok – utils → utils
+
+
+def text_to_lowercase(t: pd.DataFrame) -> pd.DataFrame:
+    """
+    Converte todas as colunas de texto para lowercase
+    Args:
+        t (pd.DataFrame): pandas DataFrame
+    Returns:
+        pd.DataFrame
+    """
+
+    return t.map(lambda x: x.lower().strip() if isinstance(x, str) else x)
+
+
+def persist_column_formatting(t: pd.DataFrame, columns_to_persist_override : set = {}) -> pd.DataFrame:
+    """
+    Persiste a formatacao de algumas colunas, e transforma o resto em lowercase
+    Args:
+        t (pd.DataFrame): pandas DataFrame
+    Returns:
+        pd.DataFrame
+    """
+
+    columns_to_persist = {"Name", "Class", "Vehicles", "Segment"}
+    columns_to_persist = columns_to_persist.union(columns_to_persist_override)
+
+    if len(set(t.columns).intersection(columns_to_persist)) > 0:
+        # Vamos persistir a formatacao de algumas colunas
+        columns_order = list(t.columns)
+        columns_to_persist = list(set(t.columns).intersection(columns_to_persist))
+        persistent_data = t[columns_to_persist].copy()
+
+        columns_to_normalize = list(set(columns_order) - set(columns_to_persist))
+        t = text_to_lowercase(t[columns_to_normalize])
+        t.loc[:,columns_to_persist] = persistent_data
+        return t[columns_order]
+    
+    # Nos outros casos, transformaremos tudo em lowercase
+    return text_to_lowercase(t)
+
+
+def prep_for_save(
+    df: pd.DataFrame,
+    *,
+    index: bool = False,
+    index_name: str = "index",
+    normalize: bool = False,
+):
+    if index:
+        name = df.index.name or index_name
+        df = df.reset_index().rename(columns={"index": name})
+    return persist_column_formatting(df) if normalize else df

luxorasap/utils/storage/__init__.py

@@ -0,0 +1,2 @@
+from .blob import BlobParquetClient
+__all__ = ["BlobParquetClient"]

luxorasap/utils/storage/blob.py

@@ -0,0 +1,91 @@
+import io, os
+from pathlib import PurePosixPath
+from datetime import timezone
+import pandas as pd
+import pyarrow as pa, pyarrow.parquet as pq
+from azure.storage.blob import BlobServiceClient
+
+class BlobParquetClient:
+    """Leitura/gravacao de Parquet em Azure Blob – stateless & reutilizável."""
+
+    def __init__(self, container: str = "luxorasap", adls_connection_string: str = None):
+        if adls_connection_string is None:
+            adls_connection_string = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
+
+        if adls_connection_string is None:
+            raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
+        self._svc = BlobServiceClient.from_connection_string(adls_connection_string)
+        self._container = container
+
+    # ---------- API pública ----------
+    def read_df(self, blob_path: str) -> (pd.DataFrame, bool):
+        buf = io.BytesIO()
+        try:
+            self._blob(blob_path).download_blob().readinto(buf)
+            
+            buf.seek(0)
+            return pq.read_table(buf).to_pandas(), True
+        except Exception:
+            return None, False
+            
+
+    def write_df(self, df, blob_path: str):
+        table = pa.Table.from_pandas(df)
+        buf = io.BytesIO()
+        pq.write_table(table, buf)
+        buf.seek(0)
+        self._blob(blob_path).upload_blob(buf, overwrite=True)
+        
+        
+    def get_df_update_time(self, blob_path: str) -> float:
+        try:
+            properties = self._blob(blob_path).get_blob_properties()
+            return properties['last_modified'].replace(tzinfo=timezone.utc).timestamp()
+        except Exception:
+            return 0.0
+    
+    
+    def exists_df(self, blob_path: str) -> bool:
+        try:
+            self._blob(blob_path).get_blob_properties()
+            return True
+        except Exception:
+            return False
+
+
+    def list_blob_files(self, blob_path: str, ends_with: str = None) -> list:
+        """
+        Lista os arquivos em um diretório do blob storage.
+
+        Args:
+            blob_path (str): O caminho do diretório no blob storage.
+            ends_with (str, optional): Filtra os arquivos que terminam com esta string.(Ex.: '.parquet')
+
+        Returns:
+            list: Uma lista de nomes de blob.
+            
+        """
+        try:
+            container_client = self._svc.get_container_client(self._container)
+            blob_list = container_client.list_blobs(name_starts_with=blob_path)
+            if ends_with:
+                return [blob.name for blob in blob_list if blob.name.endswith(ends_with)]
+            return [blob.name for blob in blob_list]
+        except Exception:
+            return []
+            
+
+    def table_exists(self, table_path: str) -> bool:
+        """
+            Checa se uma tabela existe no blob storage.
+        """
+        return self.exists_df(table_path)
+        
+            
+            
+    
+    
+    # ---------- interno --------------
+    def _blob(self, path: str):
+        path = str(PurePosixPath(path))
+        return self._svc.get_blob_client(self._container, path)

{luxorasap-0.0.2.dist-info → luxorasap-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.0.2
+Version: 0.1.0
 Summary: Luxor’s unified toolbox for data ingestion, querying and analytics.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved
@@ -18,10 +18,18 @@ Requires-Dist: pyarrow>=15.0
 Requires-Dist: requests>=2.32
 Requires-Dist: pydantic>=2.7
 Requires-Dist: scipy>=1.13
+Provides-Extra: storage
+Requires-Dist: azure-storage-blob>=12.19; extra == "storage"
+Requires-Dist: pyarrow>=15.0; extra == "storage"
+Provides-Extra: dataframe
+Requires-Dist: pandas>=2.2; extra == "dataframe"
 Provides-Extra: datareader
-Requires-Dist: pyarrow>=15.0; extra == "datareader"
+Requires-Dist: luxorasap[dataframe,storage]; extra == "datareader"
+Requires-Dist: numpy>=1.25; extra == "datareader"
+Requires-Dist: scipy>=1.13; extra == "datareader"
 Provides-Extra: ingest
-Requires-Dist: pandera>=0.18; extra == "ingest"
+Requires-Dist: luxorasap[dataframe,storage]; extra == "ingest"
+Requires-Dist: pandas>=2.2; extra == "ingest"
 Provides-Extra: btgapi
 Requires-Dist: requests>=2.32; extra == "btgapi"
 Requires-Dist: pydantic>=2.7; extra == "btgapi"

luxorasap-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+luxorasap/__init__.py,sha256=XLioUOz-0RizsKh06R0U3uYMG7EwTQ08p_GLVSQBXNY,1355
+luxorasap/datareader/__init__.py,sha256=41RAvbrQ4R6oj67S32CrKqolx0CJ2W8cbOF6g5Cqm2g,120
+luxorasap/datareader/core.py,sha256=LpXe5g4lZpfEqaz_gjjHizVA-vPEjBi5yJKg_7K0Nkw,153205
+luxorasap/ingest/__init__.py,sha256=XhxDTN2ar-u6UCPhnxNU_to-nWiit-SpQ6cA_N9eMSs,795
+luxorasap/ingest/cloud/__init__.py,sha256=V8cCNloP1RgPTEPsepHvWVL4m_t5geQuBORLm7x-OKQ,1729
+luxorasap/ingest/legacy_local/dataloader.py,sha256=zKPhuiBSFwkuWN6d8g2s60KkbVk1R_1cGMCtQM9j-0c,11908
+luxorasap/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+luxorasap/utils/dataframe/__init__.py,sha256=LXcMsv8kPFfIVzcwfA9lI-TExH6ty6a3NHBVBWyxfTA,161
+luxorasap/utils/dataframe/transforms.py,sha256=O5VxJHMV2g6zKLJc2O7F84wyF9c_hqo6kJOVjixLeI4,1757
+luxorasap/utils/storage/__init__.py,sha256=U3XRq94yzRp3kgBSUcRzs2tQgJ4o8h8a1ZzwiscA5XM,67
+luxorasap/utils/storage/blob.py,sha256=lJPN5VoTVheijUVp1zhSY51GnqapiYYDkypFTVtAb10,3086
+luxorasap-0.1.0.dist-info/METADATA,sha256=UD40ZCS8E2-jhopS3gTyy6jOaEOykQ71PnncCrVCjew,3018
+luxorasap-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+luxorasap-0.1.0.dist-info/entry_points.txt,sha256=XFh-dOwUhlya9DmGvgookMI0ezyUJjcOvTIHDEYS44g,52
+luxorasap-0.1.0.dist-info/top_level.txt,sha256=9YOL6bUIpzY06XFBRkUW1e4rgB32Ds91fQPGwUEjxzU,10
+luxorasap-0.1.0.dist-info/RECORD,,

luxorasap-0.0.2.dist-info/RECORD

@@ -1,10 +0,0 @@
-luxorasap/__init__.py,sha256=UTn522ZJQ17wVxA8touIplzT0y21tzJL3XmsqAbq7XY,1355
-luxorasap/datareader/__init__.py,sha256=41RAvbrQ4R6oj67S32CrKqolx0CJ2W8cbOF6g5Cqm2g,120
-luxorasap/datareader/core.py,sha256=VURibG9qly5Q1Fu6rrgvXypERb39S535wKd-xc8g0uU,156887
-luxorasap/ingest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-luxorasap/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-luxorasap-0.0.2.dist-info/METADATA,sha256=SrdjdL1Ce62fym3jfpsTYxI66NCuaRx_21GdwJwnQb0,2633
-luxorasap-0.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-luxorasap-0.0.2.dist-info/entry_points.txt,sha256=XFh-dOwUhlya9DmGvgookMI0ezyUJjcOvTIHDEYS44g,52
-luxorasap-0.0.2.dist-info/top_level.txt,sha256=9YOL6bUIpzY06XFBRkUW1e4rgB32Ds91fQPGwUEjxzU,10
-luxorasap-0.0.2.dist-info/RECORD,,