luxorasap 0.1.17__tar.gz → 0.1.18__tar.gz

{luxorasap-0.1.17 → luxorasap-0.1.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.1.17
+Version: 0.1.18
 Summary: Toolbox da Luxor para ingestão, análise e automação de dados financeiros.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved

{luxorasap-0.1.17 → luxorasap-0.1.18}/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 #############################
 [project]
 name            = "luxorasap"
-version = "0.1.17"
+version = "0.1.18"
 description     = "Toolbox da Luxor para ingestão, análise e automação de dados financeiros."
 readme          = "README.md"
 requires-python = ">=3.9"
@@ -78,7 +78,7 @@ exclude = ["tests*"]
 # bumpver (sem-ver)
 #############################
 [tool.bumpver]
-current_version = "0.1.17"
+current_version = "0.1.18"
 version_pattern = "MAJOR.MINOR.PATCH"
 
 # regex explícito – obrigatório no bumpver 2024+

{luxorasap-0.1.17 → luxorasap-0.1.18}/src/luxorasap/__init__.py

@@ -13,7 +13,7 @@ from types import ModuleType
 try:
     __version__: str = metadata.version(__name__)
 except metadata.PackageNotFoundError:  # editable install
-    __version__ = "0.1.17"
+    __version__ = "0.1.18"
 
 # ─── Lazy loader ─────────────────────────────────────────────────
 def __getattr__(name: str) -> ModuleType:

{luxorasap-0.1.17 → luxorasap-0.1.18}/src/luxorasap/ingest/cloud/__init__.py

@@ -7,6 +7,7 @@ from luxorasap.utils.storage import BlobParquetClient
 from luxorasap.utils.dataframe import prep_for_save
 from luxorasap.datareader import LuxorQuery
 
+
 __all__ = ["save_table", "incremental_load"]
 
 _client = BlobParquetClient()   # instância única para o módulo
@@ -21,8 +22,7 @@ def save_table(
     index_name: str = "index",
     normalize_columns: bool = True,
     directory: str = "enriched/parquet",
-    override=False,
-    large_df: bool = False
+    override=False
 ):
     """Salva DataFrame como Parquet em ADLS (sobrescrevendo)."""
     
@@ -35,7 +35,10 @@ def save_table(
             return
     
     df = prep_for_save(df, index=index, index_name=index_name, normalize=normalize_columns)
-    _client.write_df(df.astype(str), f"{directory}/{table_name}.parquet", large_df=large_df)
+    
+    #_client.write_df(df.astype(str), f"{directory}/{table_name}.parquet")
+    _client.write_df(df, f"{directory}/{table_name}.parquet")
+
 
 
 def incremental_load(
@@ -47,8 +50,7 @@ def incremental_load(
     index: bool = False,
     index_name: str = "index",
     normalize_columns: bool = True,
-    directory: str = "enriched/parquet",
-    large_df: bool = False
+    directory: str = "enriched/parquet"
 ):
     """Concatena novos dados aos existentes, cortando duplicados pela data."""
     df["Last_Updated"] = dt.datetime.now()
@@ -66,6 +68,5 @@ def incremental_load(
         index_name=index_name,
         normalize_columns=normalize_columns,
         directory=directory,
-        override=True,
-        large_df=large_df
+        override=True
     )

{luxorasap-0.1.17 → luxorasap-0.1.18}/src/luxorasap/utils/dataframe/__init__.py

@@ -1,4 +1,4 @@
-from .transforms import prep_for_save, persist_column_formatting, text_to_lowercase
+from .transforms import prep_for_save, persist_column_formatting, text_to_lowercase_inplace
 from .reader import read_bytes
 
-__all__ = ["prep_for_save", "persist_column_formatting", "text_to_lowercase", "read_bytes"]
+__all__ = ["prep_for_save", "persist_column_formatting", "text_to_lowercase_inplace", "read_bytes"]

luxorasap-0.1.18/src/luxorasap/utils/dataframe/transforms.py

@@ -0,0 +1,58 @@
+import pandas as pd
+from pandas.api.types import is_object_dtype, is_string_dtype
+
+
+def text_to_lowercase_inplace(df: pd.DataFrame, cols: list[str]) -> None:
+    """
+    Converte para lower+strip apenas as células que são str.
+    Não tenta aplicar `.str` se a coluna (ou célula) não for string.
+    Opera in-place; não devolve nada.
+    """
+    for col in cols:
+        # Precisa ser coluna potencialmente textual
+        if not (is_object_dtype(df[col]) or is_string_dtype(df[col])):
+            continue
+
+        # Cria máscara com valores realmente str (ignora NaN, ints, decimals…)
+        mask = df[col].apply(lambda x: isinstance(x, str))
+
+        if mask.any():                              # só se houver algo a tratar
+            df.loc[mask, col] = (
+                df.loc[mask, col]
+                  .str.lower()
+                  .str.strip()
+            )
+
+
+def persist_column_formatting(df: pd.DataFrame,
+        columns_to_persist_override: set | None = None) -> pd.DataFrame:
+    if columns_to_persist_override is None:
+        columns_to_persist_override = set()
+
+    cols_keep_case = {
+        "Name", "Class", "Vehicles", "Segment"
+    }.union(columns_to_persist_override)
+
+    # Só colunas objeto/string candidatas
+    candidate_cols = [
+        c for c in df.columns
+        if c not in cols_keep_case and
+           (df[c].dtype == "object" or pd.api.types.is_string_dtype(df[c]))
+    ]
+
+    text_to_lowercase_inplace(df, candidate_cols)
+
+    return df  # mesma referência; alterações foram in-place
+
+
+def prep_for_save(
+    df: pd.DataFrame,
+    *,
+    index: bool = False,
+    index_name: str = "index",
+    normalize: bool = False,
+):
+    if index:
+        name = df.index.name or index_name
+        df = df.reset_index().rename(columns={"index": name})
+    return persist_column_formatting(df) if normalize else df

{luxorasap-0.1.17 → luxorasap-0.1.18}/src/luxorasap/utils/storage/blob.py

@@ -4,7 +4,7 @@ from datetime import timezone
 import pandas as pd
 import pyarrow as pa, pyarrow.parquet as pq
 from azure.storage.blob import BlobServiceClient
-import tempfile
+import io
 
 from ..dataframe import read_bytes
 
@@ -34,29 +34,14 @@ class BlobParquetClient:
             return None, False
             
 
-    def write_df(self, df, blob_path: str, large_df: bool = False):
-        if not large_df:
-            table = pa.Table.from_pandas(df)
-            buf = io.BytesIO()
-            pq.write_table(table, buf)
-            buf.seek(0)
-            self._blob(blob_path).upload_blob(buf, overwrite=True)
+    def write_df(self, df, blob_path: str):
         
-        else:
-            with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as f:
-                writer = None
-                chunk_size = 100_000
-                for i in range(0, len(df), chunk_size):
-                    chunk = pa.Table.from_pandas(df.iloc[i:i+chunk_size])
-                    if writer is None:
-                        writer = pq.ParquetWriter(f.name, chunk.schema)
-                    writer.write_table(chunk)
-                writer.close()
-                
-                with open(f.name, "rb") as f_read:
-                    self._blob(blob_path).upload_blob(f_read, overwrite=True)
-                
-                os.remove(f.name)
+        blob = self._blob(blob_path)
+        table = pa.Table.from_pandas(df, preserve_index=False)
+        buf = io.BytesIO()
+        pq.write_table(table, buf)
+        buf.seek(0)
+        blob.upload_blob(buf, overwrite=True)
             
         
     def get_df_update_time(self, blob_path: str) -> float:
@@ -102,12 +87,10 @@ class BlobParquetClient:
             Checa se uma tabela existe no blob storage.
         """
         return self.exists_df(table_path)
-        
-            
-            
     
     
     # ---------- interno --------------
     def _blob(self, path: str):
         path = str(PurePosixPath(path))
-        return self._svc.get_blob_client(self._container, path)
+        return self._svc.get_blob_client(self._container, path)
+    

{luxorasap-0.1.17 → luxorasap-0.1.18}/src/luxorasap.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.1.17
+Version: 0.1.18
 Summary: Toolbox da Luxor para ingestão, análise e automação de dados financeiros.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved

{luxorasap-0.1.17 → luxorasap-0.1.18}/tests/test_ingest_cloud.py

@@ -7,17 +7,16 @@ import luxorasap.ingest.cloud as cloud
 def test_save_table_calls_blob_client(fake_blob, monkeypatch):
     captured = {}
     
-    def fake_write(df, path, large_df):
+    def fake_write(df, path):
         captured["df"] = df.copy()
         captured["path"] = path
-        captured["large_df"] = large_df
     
     monkeypatch.setattr(cloud, "_client", SimpleNamespace(write_df=fake_write))
 
     df = pd.DataFrame({"x": [1]})
     cloud.save_table("t1", df, directory="dir")
     assert captured["path"] == "dir/t1.parquet"
-    assert captured["df"].equals(df.astype(str))
+    assert captured["df"].equals(df)
     
 
 def test_incremental_load_merges_correctly(fake_blob, monkeypatch):
@@ -30,5 +29,5 @@ def test_incremental_load_merges_correctly(fake_blob, monkeypatch):
     writes = {}
     monkeypatch.setattr(cloud, "_client", SimpleNamespace(write_df=lambda df, p: writes.setdefault("df", df)))
     new = pd.DataFrame({"Date":[dt.date(2024,1,2)], "v":[2]})
-    cloud.incremental_load(stub_lq, "prices", new, increment_column="Date", large_df=False)
+    cloud.incremental_load(stub_lq, "prices", new, increment_column="Date")
     assert len(writes["df"]) == 2

luxorasap-0.1.17/src/luxorasap/utils/dataframe/transforms.py

@@ -1,52 +0,0 @@
-import pandas as pd
-
-def text_to_lowercase(t: pd.DataFrame) -> pd.DataFrame:
-    """
-    Converte todas as colunas de texto para lowercase
-    Args:
-        t (pd.DataFrame): pandas DataFrame
-    Returns:
-        pd.DataFrame
-    """
-
-    return t.map(lambda x: x.lower().strip() if isinstance(x, str) else x)
-
-
-def persist_column_formatting(t: pd.DataFrame, columns_to_persist_override : set = {}) -> pd.DataFrame:
-    """
-    Persiste a formatacao de algumas colunas, e transforma o resto em lowercase
-    Args:
-        t (pd.DataFrame): pandas DataFrame
-    Returns:
-        pd.DataFrame
-    """
-
-    columns_to_persist = {"Name", "Class", "Vehicles", "Segment"}
-    columns_to_persist = columns_to_persist.union(columns_to_persist_override)
-
-    if len(set(t.columns).intersection(columns_to_persist)) > 0:
-        # Vamos persistir a formatacao de algumas colunas
-        columns_order = list(t.columns)
-        columns_to_persist = list(set(t.columns).intersection(columns_to_persist))
-        persistent_data = t[columns_to_persist].copy()
-
-        columns_to_normalize = list(set(columns_order) - set(columns_to_persist))
-        t = text_to_lowercase(t[columns_to_normalize])
-        t.loc[:,columns_to_persist] = persistent_data
-        return t[columns_order]
-    
-    # Nos outros casos, transformaremos tudo em lowercase
-    return text_to_lowercase(t)
-
-
-def prep_for_save(
-    df: pd.DataFrame,
-    *,
-    index: bool = False,
-    index_name: str = "index",
-    normalize: bool = False,
-):
-    if index:
-        name = df.index.name or index_name
-        df = df.reset_index().rename(columns={"index": name})
-    return persist_column_formatting(df) if normalize else df