luxorasap 0.1.38__tar.gz → 0.2.0__tar.gz

{luxorasap-0.1.38 → luxorasap-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.1.38
+Version: 0.2.0
 Summary: Toolbox da Luxor para ingestão, análise e automação de dados financeiros.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved

{luxorasap-0.1.38 → luxorasap-0.2.0}/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 #############################
 [project]
 name            = "luxorasap"
-version = "0.1.38"
+version = "0.2.0"
 description     = "Toolbox da Luxor para ingestão, análise e automação de dados financeiros."
 readme          = "README.md"
 requires-python = ">=3.9"
@@ -78,7 +78,7 @@ exclude = ["tests*"]
 # bumpver (sem-ver)
 #############################
 [tool.bumpver]
-current_version = "0.1.38"
+current_version = "0.2.0"
 version_pattern = "MAJOR.MINOR.PATCH"
 
 # regex explícito – obrigatório no bumpver 2024+

{luxorasap-0.1.38 → luxorasap-0.2.0}/src/luxorasap/__init__.py

@@ -13,7 +13,7 @@ from types import ModuleType
 try:
     __version__: str = metadata.version(__name__)
 except metadata.PackageNotFoundError:  # editable install
-    __version__ = "0.1.38"
+    __version__ = "0.2.0"
 
 # ─── Lazy loader ─────────────────────────────────────────────────
 def __getattr__(name: str) -> ModuleType:

{luxorasap-0.1.38 → luxorasap-0.2.0}/src/luxorasap/datareader/core.py

@@ -24,12 +24,15 @@ load_dotenv()
 #@logger.catch
 class LuxorQuery:
 
-    def __init__(self, update_mode="optimized", is_develop_mode=False, tables_path=None, 
-                 blob_directory='enriched/parquet', adls_connection_string:str=None, container_name="luxorasap"):
+    # Criando construtor com docstring detalhada
+    def __init__(self, blob_directory='enriched/parquet', adls_connection_string:str=None, 
+                 container_name="luxorasap"):
         """
-            update_mode: 
-                'standard' - Carrega todas as tabelas disponiveis
-                'optimized' - Carrega apenas as tabelas utilizadas sob demanda
+        Classe para consulta de dados da Luxor.
+        Args:
+            blob_directory (str, optional): Diretório no blob onde estão as tabelas. Defaults to 'enriched/parquet'.
+            adls_connection_string (str, optional): String de conexão com o ADLS. Se None, usa variável de ambiente. Defaults to None.
+            container_name (str, optional): Nome do container no blob. Defaults to "luxorasap".
         """
             
         self.blob_client = BlobParquetClient(adls_connection_string=adls_connection_string,
@@ -38,10 +41,8 @@ class LuxorQuery:
 
         
         self.modified_tables = []
-        self.is_develop_mode = is_develop_mode
         
-
-        self.tables_path = tables_path
+        
         #if tables_path is None:
         #    self.tables_path = self.__set_tables_path()
 
@@ -54,8 +55,6 @@ class LuxorQuery:
         self.lipi_manga_incorp_date = dt.date(2022,12,9)
         
 
-        self.update_modes_name = {"standard" : 0, "optimized" : 1}
-        self.update_mode = self.update_modes_name[update_mode]
         self.update() # Nessa 1° exec. vai inicializar os dicionarios acima
 
 

{luxorasap-0.1.38 → luxorasap-0.2.0}/src/luxorasap/ingest/cloud/__init__.py

@@ -4,7 +4,7 @@ import pandas as pd
 import datetime as dt
 import numpy as np
 
-from luxorasap.utils.storage import BlobParquetClient
+from luxorasap.utils.storage import BlobParquetClient, BlobExcelClient, BlobPickleClient
 from luxorasap.utils.dataframe import prep_for_save, astype_str_inplace
 from luxorasap.datareader import LuxorQuery
 
@@ -12,6 +12,8 @@ from luxorasap.datareader import LuxorQuery
 __all__ = ["save_table", "incremental_load"]
 
 _client = BlobParquetClient()   # instância única para o módulo
+_client_excel = None
+_client_pickle = None
 
 
 # ────────────────────────────────────────────────────────────────
@@ -23,7 +25,8 @@ def save_table(
     index_name: str = "index",
     normalize_columns: bool = True,
     directory: str = "enriched/parquet",
-    override=False
+    override=False,
+    format='parquet'
 ):
     """Salva DataFrame como Parquet em ADLS (sobrescrevendo)."""
     
@@ -43,9 +46,27 @@ def save_table(
     
     df = prep_for_save(df, index=index, index_name=index_name, normalize=normalize_columns)
     
-    #_client.write_df(df.astype(str), f"{directory}/{table_name}.parquet")
-    astype_str_inplace(df)
-    _client.write_df(df, f"{directory}/{table_name}.parquet")
+    if format == 'parquet':
+        #_client.write_df(df.astype(str), f"{directory}/{table_name}.parquet")
+        astype_str_inplace(df)
+        _client.write_df(df, f"{directory}/{table_name}.parquet")
+    
+    elif format == 'excel':
+        global _client_excel
+        if _client_excel is None:
+            _client_excel = BlobExcelClient()
+        if index:
+            df = df.reset_index().rename(columns={"index": index_name})
+        _client_excel.write_excel(df, f"{directory}/{table_name}.xlsx")
+    
+    elif format == 'pickle':
+        global _client_pickle
+        if _client_pickle is None:
+            _client_pickle = BlobPickleClient()
+        _client_pickle.write_pickle(df, f"{directory}/{table_name}.pkl")
+    
+    else:
+        raise ValueError(f"Formato '{format}' não suportado. Use 'parquet', 'excel' ou 'pickle'.")
 
 
 

luxorasap-0.2.0/src/luxorasap/utils/storage/__init__.py

@@ -0,0 +1,12 @@
+from .blob import BlobParquetClient, BlobPickleClient, BlobExcelClient, delete_blob, list_blob_files
+from .change_tracker import BlobChangeWatcher, BlobMetadata
+
+__all__ = [
+    "BlobParquetClient",
+    "BlobPickleClient",
+    "BlobExcelClient",
+    "delete_blob",
+    "list_blob_files",
+    "BlobChangeWatcher",
+    "BlobMetadata",
+]

luxorasap-0.2.0/src/luxorasap/utils/storage/blob.py

@@ -0,0 +1,263 @@
+import io, os
+from pathlib import PurePosixPath
+from datetime import timezone
+import pandas as pd
+import pyarrow as pa, pyarrow.parquet as pq
+import pickle
+import re
+
+from azure.storage.blob import BlobServiceClient
+from azure.core.exceptions import ResourceNotFoundError
+
+from ..dataframe import read_bytes
+
+
+class BlobParquetClient:
+    """Leitura/gravacao de Parquet em Azure Blob – stateless & reutilizável."""
+
+    def __init__(self, container: str = "luxorasap", adls_connection_string: str = None):
+        if adls_connection_string is None:
+            adls_connection_string = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
+
+        if adls_connection_string is None:
+            raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
+        self._svc = BlobServiceClient.from_connection_string(adls_connection_string)
+        self._container = container
+
+    # ---------- API pública ----------
+    def read_df(self, blob_path: str) -> (pd.DataFrame, bool):
+        buf = io.BytesIO()
+        try:
+            self._blob(blob_path).download_blob().readinto(buf)
+            return (
+                read_bytes(buf.getvalue(), filename=PurePosixPath(blob_path).name),
+                True,
+            )
+        except Exception:
+            return None, False
+            
+
+    def write_df(self, df, blob_path: str):
+        
+        blob = self._blob(blob_path)
+        table = pa.Table.from_pandas(df)
+        buf = io.BytesIO()
+        pq.write_table(table, buf)
+        buf.seek(0)
+        blob.upload_blob(buf, overwrite=True)
+            
+        
+    def get_df_update_time(self, blob_path: str) -> float:
+        try:
+            properties = self._blob(blob_path).get_blob_properties()
+            return properties['last_modified'].replace(tzinfo=timezone.utc).timestamp()
+        except Exception:
+            return 0.0
+    
+    
+    def exists_df(self, blob_path: str) -> bool:
+        try:
+            self._blob(blob_path).get_blob_properties()
+            return True
+        except Exception:
+            return False
+
+
+    def list_blob_files(self, blob_path: str, ends_with: str = None) -> list:
+        """
+        Lista os arquivos em um diretório do blob storage.
+
+        Args:
+            blob_path (str): O caminho do diretório no blob storage.
+            ends_with (str, optional): Filtra os arquivos que terminam com esta string.(Ex.: '.parquet')
+
+        Returns:
+            list: Uma lista de nomes de blob.
+            
+        """
+        try:
+            container_client = self._svc.get_container_client(self._container)
+            blob_list = container_client.list_blobs(name_starts_with=blob_path)
+            if ends_with:
+                return [blob.name for blob in blob_list if blob.name.endswith(ends_with)]
+            return [blob.name for blob in blob_list]
+        except Exception:
+            return []
+            
+
+    def table_exists(self, table_path: str) -> bool:
+        """
+            Checa se uma tabela existe no blob storage.
+        """
+        return self.exists_df(table_path)
+    
+    
+    # ---------- interno --------------
+    def _blob(self, path: str):
+        path = str(PurePosixPath(path))
+        return self._svc.get_blob_client(self._container, path)
+    
+
+class BlobPickleClient:
+    def __init__(self, *, adls_connection_string: str = None, container: str = "luxorasap"):
+        if adls_connection_string is None:
+            adls_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+            
+        if adls_connection_string is None:
+            raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
+        
+        self._svc = BlobServiceClient.from_connection_string(adls_connection_string)
+        self._container = self._svc.get_container_client(container)
+
+
+    def write_pickle(self, obj, blob_name: str):
+        """Salva objeto Python (ex: DataFrame) como pickle no blob"""
+        buf = io.BytesIO()
+        pickle.dump(obj, buf)
+        buf.seek(0)
+        self._container.upload_blob(name=blob_name, data=buf, overwrite=True)
+
+
+    def read_pickle(self, blob_name: str):
+        """Lê pickle do blob e retorna objeto Python"""
+        downloader = self._container.download_blob(blob_name)
+        buf = io.BytesIO(downloader.readall())
+        return pickle.load(buf)
+
+
+    def exists(self, blob_name: str) -> bool:
+        return self._container.get_blob_client(blob_name).exists()
+    
+    
+
+class BlobExcelClient:
+    def __init__(self, *, adls_connection_string: str = None, container: str = "luxorasap"):
+        if adls_connection_string is None:
+            adls_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+        
+        if adls_connection_string is None:
+            raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
+            
+        self._svc = BlobServiceClient.from_connection_string(adls_connection_string)
+        self._container = self._svc.get_container_client(container)
+
+
+    def write_excel(self, df: pd.DataFrame, blob_name: str, **kwargs):
+        """
+        Salva um DataFrame como arquivo Excel no blob.
+
+        Args:
+            df (pd.DataFrame): DataFrame a ser salvo
+            blob_name (str): caminho/nome do blob (ex: "reports/test.xlsx")
+            **kwargs: argumentos extras para `DataFrame.to_excel`
+        """
+        buf = io.BytesIO()
+        df.to_excel(buf, index=False, engine="openpyxl", **kwargs)
+        buf.seek(0)
+        self._container.upload_blob(name=blob_name, data=buf, overwrite=True)
+
+
+    def read_excel(self, blob_name: str, **kwargs) -> pd.DataFrame:
+        """
+        Lê um arquivo Excel do blob e retorna um DataFrame.
+
+        Args:
+            blob_name (str): caminho/nome do blob (ex: "reports/test.xlsx")
+            **kwargs: argumentos extras para `pd.read_excel`
+
+        Returns:
+            pd.DataFrame
+        """
+        downloader = self._container.download_blob(blob_name)
+        buf = io.BytesIO(downloader.readall())
+        return pd.read_excel(buf, engine="openpyxl", **kwargs)
+
+    def exists(self, blob_name: str) -> bool:
+        return self._container.get_blob_client(blob_name).exists()
+    
+ 
+    
+def list_blob_files(blob_path: str, container="luxorasap", ends_with: str = None, adls_connection_string: str = None) -> list:
+    """
+    Lista os arquivos em um diretório do blob storage.
+
+    Args:
+        blob_path (str): O caminho do diretório no blob storage.
+        ends_with (str, optional): Filtra os arquivos que terminam com esta string.(Ex.: '.parquet')
+
+    Returns:
+        list: Uma lista de nomes de blob.
+        
+    """
+    
+    if adls_connection_string is None:
+        adls_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+    if adls_connection_string is None:
+        raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
+    
+    try:
+        svc  = BlobServiceClient.from_connection_string(adls_connection_string)
+        container_client = svc.get_container_client(container)
+        blob_list = container_client.list_blobs(name_starts_with=blob_path)
+        if ends_with:
+            return [blob.name for blob in blob_list if blob.name.endswith(ends_with)]
+        return [blob.name for blob in blob_list]
+    except Exception:
+        return []
+        
+
+def delete_blob(
+    blob_name: str,
+    *,
+    adls_connection_string: str | None = None,
+    container: str = "luxorasap",
+    include_snapshots: bool = False,
+) -> None:
+    """
+    Exclui com segurança APENAS um arquivo (blob) exato do Azure Blob Storage.
+
+    Regras de segurança:
+      - Recusa nomes que terminem com "/" (prefixos de diretório virtual).
+      - Recusa curingas/shell globs (*, ?, []), para evitar exclusões indevidas.
+      - Verifica a existência do blob exato antes de remover.
+
+    Args:
+        blob_name: Caminho EXATO do blob (ex.: "enriched/parquet/tabela.parquet").
+        adls_connection_string: Se None, lê de AZURE_STORAGE_CONNECTION_STRING.
+        container: Nome do container.
+        include_snapshots: Se True, apaga snapshots vinculados ao blob.
+
+    Raises:
+        ValueError: Se o nome parecer um diretório/prefixo ou contiver curingas.
+        FileNotFoundError: Se o blob exato não existir.
+        RuntimeError: Se a conexão com o Azure não estiver configurada.
+    """
+    if adls_connection_string is None:
+        adls_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+    if adls_connection_string is None:
+        raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
+    
+    # 1) Bloqueios contra “diretórios” e curingas
+    if blob_name.endswith("/"):
+        raise ValueError("Nome termina com '/': recusa exclusão de diretórios/prefixos.")
+    if re.search(r"[\*\?\[\]]", blob_name):
+        raise ValueError("Curingas encontrados no nome do blob. Informe um arquivo exato.")
+
+    svc = BlobServiceClient.from_connection_string(adls_connection_string)
+    container_client = svc.get_container_client(container)
+    blob_client = container_client.get_blob_client(blob_name)
+
+    # 2) Checa existência do blob exato
+    try:
+        blob_client.get_blob_properties()
+    except ResourceNotFoundError:
+        raise FileNotFoundError(f"Blob não encontrado: {blob_name}")
+
+    # 3) Exclui apenas o alvo exato
+    delete_kwargs = {}
+    if include_snapshots:
+        delete_kwargs["delete_snapshots"] = "include"
+
+    blob_client.delete_blob(**delete_kwargs)
+
+    

luxorasap-0.2.0/src/luxorasap/utils/storage/change_tracker.py

@@ -0,0 +1,294 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, asdict
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from azure.core.exceptions import ResourceNotFoundError
+from azure.storage.blob import BlobServiceClient
+
+# Reuso dos utilitários que você já tem no projeto
+from luxorasap.utils.storage.blob import BlobPickleClient
+import os
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tipos de dados
+# ──────────────────────────────────────────────────────────────────────────────
+
+@dataclass(frozen=True)
+class BlobMetadata:
+    """
+    Conjunto mínimo de informações para detectar mudanças em um blob.
+    """
+    last_modified_utc: datetime   # timezone-aware, sempre em UTC
+    etag: str
+    size_bytes: int
+
+    @staticmethod
+    def from_blob_properties(props) -> "BlobMetadata":
+        """
+        Constrói BlobMetadata a partir de BlobProperties (SDK azure.storage.blob).
+        Garante que last_modified seja timezone-aware em UTC.
+        """
+        last_mod = props.last_modified
+        if last_mod.tzinfo is None:
+            last_mod = last_mod.replace(tzinfo=timezone.utc)
+        else:
+            last_mod = last_mod.astimezone(timezone.utc)
+
+        return BlobMetadata(
+            last_modified_utc=last_mod,
+            etag=props.etag,
+            size_bytes=int(props.size),
+        )
+
+    def to_dict(self) -> Dict:
+        d = asdict(self)
+        d["last_modified_utc"] = self.last_modified_utc.isoformat()
+        return d
+
+    @staticmethod
+    def from_dict(d: Dict) -> "BlobMetadata":
+        lm = d["last_modified_utc"]
+        if isinstance(lm, str):
+            lm = datetime.fromisoformat(lm)
+        if lm.tzinfo is None:
+            lm = lm.replace(tzinfo=timezone.utc)
+        else:
+            lm = lm.astimezone(timezone.utc)
+        return BlobMetadata(last_modified_utc=lm, etag=d["etag"], size_bytes=int(d["size_bytes"]))
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Watcher (com persistência em pickle no próprio ADLS)
+# ──────────────────────────────────────────────────────────────────────────────
+
+class BlobChangeWatcher:
+    """
+    Verificador de mudanças de blobs, com snapshot persistido via Pickle no ADLS.
+
+    Snapshot salvo como dict:
+        {
+          "<blob_path>": {"last_modified_utc": "...", "etag": "...", "size_bytes": int},
+          ...
+        }
+    """
+
+    def __init__(
+        self,
+        *,
+        adls_connection_string: Optional[str] = None,
+        container: str = "luxorasap",
+        snapshot_blob_path: str = "system/state",
+        watcher_id: str = "blob_change_watcher.pkl",
+        treat_missing_as_changed: bool = True,
+    ) -> None:
+        """
+        Args:
+            adls_connection_string: Se None, usa AZURE_STORAGE_CONNECTION_STRING do ambiente.
+            container: Nome do container onde estão os blobs (e onde ficará o snapshot).
+            snapshot_blob_path: Caminho do arquivo pickle (no próprio container) que guarda o snapshot.
+            treat_missing_as_changed: Se True, um blob observado pela primeira vez é considerado "mudado".
+        """
+        
+        if adls_connection_string is None:
+            adls_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+            
+        if adls_connection_string is None:
+            raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
+        
+        self._container_name = container
+        self._snapshot_blob_path = f"{snapshot_blob_path}/{watcher_id}"
+        self._treat_missing_as_changed = treat_missing_as_changed
+
+        # Clientes
+        self._blob_service = BlobServiceClient.from_connection_string(adls_connection_string)
+        self._container_client = self._blob_service.get_container_client(self._container_name)
+        self._pickle_client = BlobPickleClient(
+            adls_connection_string=adls_connection_string,
+            container=self._container_name,
+        )
+
+        # Estado em memória
+        self._snapshot: Dict[str, Dict] = {}
+
+        # Carrega snapshot na inicialização (se não existir, começa vazio)
+        self._load_snapshot()
+
+    # ───────────────────────────── Persistência do snapshot ─────────────────────────────
+
+    def _load_snapshot(self) -> None:
+        """
+        Carrega o snapshot do ADLS (pickle).
+        Se não existir ou estiver inválido, inicia com dicionário vazio.
+        """
+        try:
+            data = self._pickle_client.read_pickle(self._snapshot_blob_path)
+            self._snapshot = data if isinstance(data, dict) else {}
+        except FileNotFoundError:
+            self._snapshot = {}
+        except Exception:
+            # Corrupção/versão antiga/etc → começa do zero
+            self._snapshot = {}
+
+
+    def _save_snapshot(self) -> None:
+        """
+        Salva o snapshot atual no ADLS via pickle.
+        """
+        self._pickle_client.write_pickle(self._snapshot, self._snapshot_blob_path)
+
+    # ───────────────────────────── Acesso a propriedades remotas ─────────────────────────────
+
+    def _fetch_remote_metadata(self, blob_path: str) -> BlobMetadata:
+        """
+        Busca metadados atuais do blob no ADLS.
+        Raises:
+            ResourceNotFoundError se o blob não existir.
+        """
+        props = self._container_client.get_blob_client(blob_path).get_blob_properties()
+        return BlobMetadata.from_blob_properties(props)
+
+    def _get_snapshot_metadata(self, blob_path: str) -> Optional[BlobMetadata]:
+        """
+        Retorna o metadata salvo no snapshot (se houver).
+        """
+        raw = self._snapshot.get(blob_path)
+        return BlobMetadata.from_dict(raw) if raw else None
+
+    # ───────────────────────────── API pública ─────────────────────────────
+
+    def has_changed(
+        self,
+        blob_path: str,
+        *,
+        update_snapshot: bool = False,
+        treat_missing_as_changed: Optional[bool] = None,
+    ) -> Tuple[bool, Optional[BlobMetadata], Optional[BlobMetadata]]:
+        """
+        Verifica se o blob mudou desde o snapshot anterior.
+
+        Args:
+            blob_path: Caminho do blob (ex.: "raw/xlsx/trades.xlsx").
+            update_snapshot: Se True, grava o novo estado no snapshot quando houver mudança.
+            treat_missing_as_changed: Override local para a regra de "primeira vez conta como mudança?".
+
+        Returns:
+            (mudou?, metadata_antigo, metadata_atual)
+        """
+        if treat_missing_as_changed is None:
+            treat_missing_as_changed = self._treat_missing_as_changed
+
+        previous = self._get_snapshot_metadata(blob_path)
+
+        # Se o blob não existe mais no remoto:
+        try:
+            current = self._fetch_remote_metadata(blob_path)
+        except ResourceNotFoundError:
+            changed = previous is not None
+            if update_snapshot and changed:
+                # remove do snapshot porque o blob foi apagado
+                self._snapshot.pop(blob_path, None)
+                self._save_snapshot()
+            return changed, previous, None
+
+        # Primeira observação desse blob?
+        if previous is None:
+            changed = bool(treat_missing_as_changed)
+        else:
+            # Critério de mudança (ordem de “força”: etag > last_modified > size)
+            changed = (
+                current.etag != previous.etag
+                or current.last_modified_utc != previous.last_modified_utc
+                or current.size_bytes != previous.size_bytes
+            )
+
+        if update_snapshot and changed:
+            self._snapshot[blob_path] = current.to_dict()
+            self._save_snapshot()
+
+        return changed, previous, current
+
+
+    def update_snapshot(self, blob_path: str) -> Optional[BlobMetadata]:
+        """
+        Força a atualização do snapshot para refletir o estado atual do blob.
+        Se o blob não existir, remove do snapshot e retorna None.
+        """
+        try:
+            current = self._fetch_remote_metadata(blob_path)
+        except ResourceNotFoundError:
+            self._snapshot.pop(blob_path, None)
+            self._save_snapshot()
+            return None
+
+        self._snapshot[blob_path] = current.to_dict()
+        self._save_snapshot()
+        return current
+
+
+    def mark_as_synchronized(self, blob_path: str, metadata: Optional[BlobMetadata] = None) -> None:
+        """
+        Marca explicitamente um blob como “sincronizado” no snapshot (ex.: após processar um pipeline).
+        Se `metadata` não for informado, consulta o estado atual no ADLS.
+        """
+        if metadata is None:
+            metadata = self._fetch_remote_metadata(blob_path)
+        self._snapshot[blob_path] = metadata.to_dict()
+        self._save_snapshot()
+
+
+    def list_changed_under_prefix(
+        self,
+        prefix: str,
+        *,
+        allowed_extensions: Optional[Sequence[str]] = None,
+        update_snapshot: bool = False,
+    ) -> List[str]:
+        """
+        Varre todos os blobs sob um prefixo e retorna a lista dos que mudaram
+        segundo as regras de comparação de metadados.
+
+        Args:
+            prefix: Ex.: "enriched/parquet/fundos" (com ou sem barra final).
+            allowed_extensions: Ex.: [".parquet", ".xlsx"] para filtrar por sufixo.
+            update_snapshot: Se True, atualiza o snapshot para os que mudaram.
+
+        Returns:
+            Lista de paths de blobs que mudaram.
+        """
+        if prefix and not prefix.endswith("/"):
+            prefix += "/"
+
+        extensions = tuple(e.lower() for e in (allowed_extensions or []))
+        changed_paths: List[str] = []
+
+        for blob_item in self._container_client.list_blobs(name_starts_with=prefix):
+            name = blob_item.name
+            if name.endswith("/"):
+                continue
+            if extensions and not name.lower().endswith(extensions):
+                continue
+
+            previous = self._get_snapshot_metadata(name)
+            current = BlobMetadata.from_blob_properties(blob_item)
+
+            if previous is None:
+                has_changed = self._treat_missing_as_changed
+            else:
+                has_changed = (
+                    current.etag != previous.etag
+                    or current.last_modified_utc != previous.last_modified_utc
+                    or current.size_bytes != previous.size_bytes
+                )
+
+            if has_changed:
+                changed_paths.append(name)
+                if update_snapshot:
+                    self._snapshot[name] = current.to_dict()
+
+        if update_snapshot and changed_paths:
+            self._save_snapshot()
+
+        return changed_paths

{luxorasap-0.1.38 → luxorasap-0.2.0}/src/luxorasap.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.1.38
+Version: 0.2.0
 Summary: Toolbox da Luxor para ingestão, análise e automação de dados financeiros.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved

{luxorasap-0.1.38 → luxorasap-0.2.0}/src/luxorasap.egg-info/SOURCES.txt

@@ -22,6 +22,7 @@ src/luxorasap/utils/dataframe/reader.py
 src/luxorasap/utils/dataframe/transforms.py
 src/luxorasap/utils/storage/__init__.py
 src/luxorasap/utils/storage/blob.py
+src/luxorasap/utils/storage/change_tracker.py
 src/luxorasap/utils/tools/__init__.py
 src/luxorasap/utils/tools/excel.py
 tests/test_btgapi_auth.py
@@ -30,5 +31,7 @@ tests/test_btgapi_trades.py
 tests/test_datareader.py
 tests/test_ingest_cloud.py
 tests/test_ingest_legacy_local.py
+tests/test_utils_change_tracker.py
 tests/test_utils_dataframe.py
-tests/test_utils_storage.py
+tests/test_utils_storage.py
+tests/tests_utils_pickle_excel.py

luxorasap-0.2.0/tests/test_utils_change_tracker.py

@@ -0,0 +1,180 @@
+import datetime as dt
+from types import SimpleNamespace
+from datetime import timezone
+
+import pytest
+
+from luxorasap.utils.storage import BlobChangeWatcher, BlobMetadata
+
+
+# ------------------------ Fakes do SDK Azure ------------------------
+
+class FakeDownload:
+    def __init__(self, content: bytes):
+        self._content = content
+    def readall(self):
+        return self._content
+
+class FakeBlobClient:
+    def __init__(self, name, props=None, store=None):
+        self._name = name
+        self._props = props  # SimpleNamespace(last_modified, etag, size)
+        self._store = store  # dict[name] -> bytes (pkl snapshot)
+    def get_blob_properties(self):
+        if self._props is None:
+            from azure.core.exceptions import ResourceNotFoundError
+            raise ResourceNotFoundError("not found")
+        return self._props
+    def download_blob(self, lease=None):
+        # apenas para o snapshot (state pickle)
+        data = self._store.get(self._name, b"")
+        return FakeDownload(data)
+
+class FakeContainerClient:
+    def __init__(self, blobs, store):
+        # blobs: dict[name] -> SimpleNamespace(last_modified, etag, size)
+        self._blobs = blobs
+        self._store = store
+    def get_blob_client(self, name):
+        props = self._blobs.get(name)
+        return FakeBlobClient(name, props=props, store=self._store)
+    def list_blobs(self, name_starts_with=""):
+        for name, props in self._blobs.items():
+            if name.startswith(name_starts_with):
+                # Azure devolve itens com .name + props
+                item = SimpleNamespace(
+                    name=name,
+                    last_modified=props.last_modified,
+                    etag=props.etag,
+                    size=props.size,
+                )
+                yield item
+    def upload_blob(self, name, data, overwrite=False):
+        # usado para salvar o snapshot .pkl
+        content = data.read() if hasattr(data, "read") else data
+        self._store[name] = content
+        return SimpleNamespace()  # dummy
+
+class FakeBlobServiceClient:
+    def __init__(self, container_client):
+        self._cc = container_client
+    def get_container_client(self, container):
+        return self._cc
+
+# ------------------------ Fixtures ------------------------
+
+@pytest.fixture
+def fake_now():
+    return dt.datetime(2025, 8, 25, 12, 0, 0, tzinfo=timezone.utc)
+
+@pytest.fixture
+def azure_mocks(monkeypatch, fake_now):
+    """
+    Prepara um container fake com 2 blobs e storage em memória para o snapshot pkl.
+    """
+    # blobs existentes no "ADLS"
+    blobs = {
+        "raw/x/a.xlsx": SimpleNamespace(
+            last_modified=fake_now, etag='"v1-a"', size=100
+        ),
+        "raw/p/tb.parquet": SimpleNamespace(
+            last_modified=fake_now, etag='"v1-p"', size=500
+        ),
+    }
+    # storage em memória para o snapshot .pkl
+    store = {}
+
+    cc = FakeContainerClient(blobs=blobs, store=store)
+    bsc = FakeBlobServiceClient(container_client=cc)
+
+    # patcha o construtor real para devolver o fake
+    import luxorasap.utils.storage.change_tracker as mod
+    monkeypatch.setattr(mod, "BlobServiceClient", SimpleNamespace(from_connection_string=lambda *_args, **_kw: bsc))
+
+    return SimpleNamespace(blobs=blobs, store=store, cc=cc, bsc=bsc)
+
+
+# ------------------------ Testes ------------------------
+
+
+def test_no_change_is_false(azure_mocks):
+    watcher = BlobChangeWatcher(
+        container="luxorasap",
+        snapshot_blob_path="system/state/tests",
+        watcher_id='test_watcher.pkl',
+        treat_missing_as_changed=True,
+    )
+    # primeira vez -> muda
+    watcher.has_changed("raw/p/tb.parquet", update_snapshot=True)
+
+    # mesma versão -> não muda
+    changed, prev, curr = watcher.has_changed("raw/p/tb.parquet", update_snapshot=False)
+    assert changed is False
+    assert prev is not None and curr is not None
+    assert prev.etag == curr.etag
+    assert prev.size_bytes == curr.size_bytes
+
+
+def test_change_by_etag_or_size_is_true(monkeypatch, azure_mocks, fake_now):
+    watcher = BlobChangeWatcher(
+        container="luxorasap",
+        snapshot_blob_path="system/state/tests",
+        watcher_id='test_watcher.pkl'
+    )
+    # baseline
+    watcher.has_changed("raw/x/a.xlsx", update_snapshot=True)
+
+    # muda etag
+    azure_mocks.blobs["raw/x/a.xlsx"].etag = '"v2-a"'
+    changed, prev, curr = watcher.has_changed("raw/x/a.xlsx", update_snapshot=False)
+    assert changed is True
+
+    # aplica snapshot
+    watcher.has_changed("raw/x/a.xlsx", update_snapshot=True)
+
+    # muda apenas tamanho
+    azure_mocks.blobs["raw/x/a.xlsx"].size = 200
+    changed2, _, _ = watcher.has_changed("raw/x/a.xlsx", update_snapshot=False)
+    assert changed2 is True
+
+
+def test_deleted_blob_is_considered_changed_if_was_known(azure_mocks):
+    watcher = BlobChangeWatcher(
+        container="luxorasap",
+        snapshot_blob_path="system/state/tests",
+        watcher_id='test_watcher.pkl'
+    )
+    # primeiro registra
+    watcher.has_changed("raw/p/tb.parquet", update_snapshot=True)
+
+    # remove do conjunto remoto
+    azure_mocks.blobs.pop("raw/p/tb.parquet")
+
+    changed, prev, curr = watcher.has_changed("raw/p/tb.parquet", update_snapshot=True)
+    assert changed is True
+    assert prev is not None
+    assert curr is None  # não existe mais
+
+
+def test_list_changed_under_prefix_filters_and_updates(azure_mocks, fake_now):
+    watcher = BlobChangeWatcher(
+        container="luxorasap",
+        snapshot_blob_path="system/state/tests",
+        watcher_id='test_watcher.pkl'
+    )
+    # primeira varredura (primeira vez conta como mudança)
+    changed = watcher.list_changed_under_prefix(
+        "raw/",
+        allowed_extensions=[".xlsx"],
+        update_snapshot=True,
+    )
+    assert changed == ["raw/x/a.xlsx"]
+
+    # altera parquet, mas filtro é xlsx, então não deve aparecer
+    azure_mocks.blobs["raw/p/tb.parquet"].etag = '"v2-p"'
+    changed2 = watcher.list_changed_under_prefix(
+        "raw/",
+        allowed_extensions=[".xlsx"],
+        update_snapshot=True,
+    )
+    assert changed2 == []

luxorasap-0.2.0/tests/tests_utils_pickle_excel.py

@@ -0,0 +1,92 @@
+import io
+import pickle
+import pandas as pd
+import pytest
+from types import SimpleNamespace
+
+# Supondo que as classes estejam em luxorasap.utils.storage.blob
+from luxorasap.utils.storage import BlobPickleClient, BlobExcelClient
+
+
+# ------------------------ Fakes Azure ------------------------
+
+class FakeDownload:
+    def __init__(self, content: bytes):
+        self._content = content
+    def readall(self):
+        return self._content
+
+class FakeBlobClient:
+    def __init__(self, name, store):
+        self._name = name
+        self._store = store
+    def download_blob(self):
+        if self._name not in self._store:
+            from azure.core.exceptions import ResourceNotFoundError
+            raise ResourceNotFoundError("not found")
+        return FakeDownload(self._store[self._name])
+
+class FakeContainerClient:
+    def __init__(self, store):
+        self._store = store
+    def get_blob_client(self, name):
+        return FakeBlobClient(name, self._store)
+    def upload_blob(self, name, data, overwrite=False):
+        content = data.read() if hasattr(data, "read") else data
+        self._store[name] = content
+        return SimpleNamespace()
+
+class FakeBlobServiceClient:
+    def __init__(self, container_client):
+        self._cc = container_client
+    def get_container_client(self, container):
+        return self._cc
+
+# ------------------------ Fixtures ------------------------
+
+@pytest.fixture
+def mem_store():
+    return {}
+
+@pytest.fixture
+def patch_blob_clients(monkeypatch, mem_store):
+    # Patch para BlobPickleClient / BlobExcelClient usarem o FakeBlobServiceClient
+    import luxorasap.utils.storage.blob as mod
+    fake_bsc = FakeBlobServiceClient(FakeContainerClient(mem_store))
+    monkeypatch.setattr(mod, "BlobServiceClient", SimpleNamespace(from_connection_string=lambda *_a, **_k: fake_bsc))
+    return mem_store
+
+# ------------------------ Tests Pickle ------------------------
+
+def test_pickle_roundtrip(patch_blob_clients):
+    client = BlobPickleClient()
+    obj = {"a": 1, "b": [1, 2, 3]}
+    path = "aux/test/state.pkl"
+
+    client.write_pickle(obj, path)
+    loaded = client.read_pickle(path)
+    assert loaded == obj
+
+def test_pickle_read_missing_raises(monkeypatch, patch_blob_clients):
+    client = BlobPickleClient()
+    with pytest.raises(Exception):
+        client.read_pickle("aux/missing.pkl")
+
+# ------------------------ Tests Excel ------------------------
+
+@pytest.mark.skipif(
+    pytest.importorskip("openpyxl", reason="openpyxl é necessário para testar Excel") is None,
+    reason="openpyxl não disponível",
+)
+def test_excel_roundtrip(patch_blob_clients, tmp_path):
+    df = pd.DataFrame({"Nome": ["Ana", "Bruno"], "Idade": [28, 35]})
+    client = BlobExcelClient()
+
+    blob_path = "reports/teste.xlsx"
+    client.write_excel(df, blob_path, index=False)
+
+    df2 = client.read_excel(blob_path)
+    # Comparação tolerante a tipos (pandas pode alterar dtype ao ler)
+    assert df2.shape == df.shape
+    assert list(df2.columns) == list(df.columns)
+    assert df2.astype(str).equals(df.astype(str))

luxorasap-0.1.38/src/luxorasap/utils/storage/__init__.py

@@ -1,2 +0,0 @@
-from .blob import BlobParquetClient, BlobPickleClient
-__all__ = ["BlobParquetClient", "BlobPickleClient"]

luxorasap-0.1.38/src/luxorasap/utils/storage/blob.py

@@ -1,124 +0,0 @@
-import io, os
-from pathlib import PurePosixPath
-from datetime import timezone
-import pandas as pd
-import pyarrow as pa, pyarrow.parquet as pq
-from azure.storage.blob import BlobServiceClient
-import pickle
-
-from ..dataframe import read_bytes
-
-
-class BlobParquetClient:
-    """Leitura/gravacao de Parquet em Azure Blob – stateless & reutilizável."""
-
-    def __init__(self, container: str = "luxorasap", adls_connection_string: str = None):
-        if adls_connection_string is None:
-            adls_connection_string = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
-
-        if adls_connection_string is None:
-            raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
-        self._svc = BlobServiceClient.from_connection_string(adls_connection_string)
-        self._container = container
-
-    # ---------- API pública ----------
-    def read_df(self, blob_path: str) -> (pd.DataFrame, bool):
-        buf = io.BytesIO()
-        try:
-            self._blob(blob_path).download_blob().readinto(buf)
-            return (
-                read_bytes(buf.getvalue(), filename=PurePosixPath(blob_path).name),
-                True,
-            )
-        except Exception:
-            return None, False
-            
-
-    def write_df(self, df, blob_path: str):
-        
-        blob = self._blob(blob_path)
-        table = pa.Table.from_pandas(df)
-        buf = io.BytesIO()
-        pq.write_table(table, buf)
-        buf.seek(0)
-        blob.upload_blob(buf, overwrite=True)
-            
-        
-    def get_df_update_time(self, blob_path: str) -> float:
-        try:
-            properties = self._blob(blob_path).get_blob_properties()
-            return properties['last_modified'].replace(tzinfo=timezone.utc).timestamp()
-        except Exception:
-            return 0.0
-    
-    
-    def exists_df(self, blob_path: str) -> bool:
-        try:
-            self._blob(blob_path).get_blob_properties()
-            return True
-        except Exception:
-            return False
-
-
-    def list_blob_files(self, blob_path: str, ends_with: str = None) -> list:
-        """
-        Lista os arquivos em um diretório do blob storage.
-
-        Args:
-            blob_path (str): O caminho do diretório no blob storage.
-            ends_with (str, optional): Filtra os arquivos que terminam com esta string.(Ex.: '.parquet')
-
-        Returns:
-            list: Uma lista de nomes de blob.
-            
-        """
-        try:
-            container_client = self._svc.get_container_client(self._container)
-            blob_list = container_client.list_blobs(name_starts_with=blob_path)
-            if ends_with:
-                return [blob.name for blob in blob_list if blob.name.endswith(ends_with)]
-            return [blob.name for blob in blob_list]
-        except Exception:
-            return []
-            
-
-    def table_exists(self, table_path: str) -> bool:
-        """
-            Checa se uma tabela existe no blob storage.
-        """
-        return self.exists_df(table_path)
-    
-    
-    # ---------- interno --------------
-    def _blob(self, path: str):
-        path = str(PurePosixPath(path))
-        return self._svc.get_blob_client(self._container, path)
-    
-
-class BlobPickleClient:
-    def __init__(self, *, adls_connection_string: str = None, container: str = "luxorasap"):
-        if adls_connection_string is None:
-            adls_connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
-            
-        if adls_connection_string is None:
-            raise RuntimeError("AZURE_STORAGE_CONNECTION_STRING not set")
-        
-        self.blob_service_client = BlobServiceClient.from_connection_string(adls_connection_string)
-        self.container_client = self.blob_service_client.get_container_client(container)
-
-    def write_pickle(self, obj, blob_name: str):
-        """Salva objeto Python (ex: DataFrame) como pickle no blob"""
-        buf = io.BytesIO()
-        pickle.dump(obj, buf)
-        buf.seek(0)
-        self.container_client.upload_blob(name=blob_name, data=buf, overwrite=True)
-
-    def read_pickle(self, blob_name: str):
-        """Lê pickle do blob e retorna objeto Python"""
-        downloader = self.container_client.download_blob(blob_name)
-        buf = io.BytesIO(downloader.readall())
-        return pickle.load(buf)
-
-    def exists(self, blob_name: str) -> bool:
-        return self.container_client.get_blob_client(blob_name).exists()
-