luxorasap 0.1.15__tar.gz → 0.1.17__tar.gz

{luxorasap-0.1.15 → luxorasap-0.1.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.1.15
+Version: 0.1.17
 Summary: Toolbox da Luxor para ingestão, análise e automação de dados financeiros.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved

{luxorasap-0.1.15 → luxorasap-0.1.17}/pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 #############################
 [project]
 name            = "luxorasap"
-version = "0.1.15"
+version = "0.1.17"
 description     = "Toolbox da Luxor para ingestão, análise e automação de dados financeiros."
 readme          = "README.md"
 requires-python = ">=3.9"
@@ -78,7 +78,7 @@ exclude = ["tests*"]
 # bumpver (sem-ver)
 #############################
 [tool.bumpver]
-current_version = "0.1.15"
+current_version = "0.1.17"
 version_pattern = "MAJOR.MINOR.PATCH"
 
 # regex explícito – obrigatório no bumpver 2024+

{luxorasap-0.1.15 → luxorasap-0.1.17}/src/luxorasap/__init__.py

@@ -13,7 +13,7 @@ from types import ModuleType
 try:
     __version__: str = metadata.version(__name__)
 except metadata.PackageNotFoundError:  # editable install
-    __version__ = "0.1.15"
+    __version__ = "0.1.17"
 
 # ─── Lazy loader ─────────────────────────────────────────────────
 def __getattr__(name: str) -> ModuleType:

{luxorasap-0.1.15 → luxorasap-0.1.17}/src/luxorasap/ingest/cloud/__init__.py

@@ -21,7 +21,8 @@ def save_table(
     index_name: str = "index",
     normalize_columns: bool = True,
     directory: str = "enriched/parquet",
-    override=False
+    override=False,
+    large_df: bool = False
 ):
     """Salva DataFrame como Parquet em ADLS (sobrescrevendo)."""
     
@@ -34,7 +35,7 @@ def save_table(
             return
     
     df = prep_for_save(df, index=index, index_name=index_name, normalize=normalize_columns)
-    _client.write_df(df.astype(str), f"{directory}/{table_name}.parquet")
+    _client.write_df(df.astype(str), f"{directory}/{table_name}.parquet", large_df=large_df)
 
 
 def incremental_load(
@@ -47,6 +48,7 @@ def incremental_load(
     index_name: str = "index",
     normalize_columns: bool = True,
     directory: str = "enriched/parquet",
+    large_df: bool = False
 ):
     """Concatena novos dados aos existentes, cortando duplicados pela data."""
     df["Last_Updated"] = dt.datetime.now()
@@ -64,5 +66,6 @@ def incremental_load(
         index_name=index_name,
         normalize_columns=normalize_columns,
         directory=directory,
-        override=True
+        override=True,
+        large_df=large_df
     )

{luxorasap-0.1.15 → luxorasap-0.1.17}/src/luxorasap/utils/storage/blob.py

@@ -4,6 +4,7 @@ from datetime import timezone
 import pandas as pd
 import pyarrow as pa, pyarrow.parquet as pq
 from azure.storage.blob import BlobServiceClient
+import tempfile
 
 from ..dataframe import read_bytes
 
@@ -33,13 +34,30 @@ class BlobParquetClient:
             return None, False
             
 
-    def write_df(self, df, blob_path: str):
-        table = pa.Table.from_pandas(df)
-        buf = io.BytesIO()
-        pq.write_table(table, buf)
-        buf.seek(0)
-        self._blob(blob_path).upload_blob(buf, overwrite=True)
+    def write_df(self, df, blob_path: str, large_df: bool = False):
+        if not large_df:
+            table = pa.Table.from_pandas(df)
+            buf = io.BytesIO()
+            pq.write_table(table, buf)
+            buf.seek(0)
+            self._blob(blob_path).upload_blob(buf, overwrite=True)
         
+        else:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as f:
+                writer = None
+                chunk_size = 100_000
+                for i in range(0, len(df), chunk_size):
+                    chunk = pa.Table.from_pandas(df.iloc[i:i+chunk_size])
+                    if writer is None:
+                        writer = pq.ParquetWriter(f.name, chunk.schema)
+                    writer.write_table(chunk)
+                writer.close()
+                
+                with open(f.name, "rb") as f_read:
+                    self._blob(blob_path).upload_blob(f_read, overwrite=True)
+                
+                os.remove(f.name)
+            
         
     def get_df_update_time(self, blob_path: str) -> float:
         try:

{luxorasap-0.1.15 → luxorasap-0.1.17}/src/luxorasap.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.1.15
+Version: 0.1.17
 Summary: Toolbox da Luxor para ingestão, análise e automação de dados financeiros.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved

{luxorasap-0.1.15 → luxorasap-0.1.17}/tests/test_ingest_cloud.py

@@ -6,15 +6,19 @@ import luxorasap.ingest.cloud as cloud
 
 def test_save_table_calls_blob_client(fake_blob, monkeypatch):
     captured = {}
-    def fake_write(df, path):
+    
+    def fake_write(df, path, large_df):
         captured["df"] = df.copy()
         captured["path"] = path
+        captured["large_df"] = large_df
+    
     monkeypatch.setattr(cloud, "_client", SimpleNamespace(write_df=fake_write))
 
     df = pd.DataFrame({"x": [1]})
     cloud.save_table("t1", df, directory="dir")
     assert captured["path"] == "dir/t1.parquet"
     assert captured["df"].equals(df.astype(str))
+    
 
 def test_incremental_load_merges_correctly(fake_blob, monkeypatch):
     # stub LuxorQuery
@@ -26,5 +30,5 @@ def test_incremental_load_merges_correctly(fake_blob, monkeypatch):
     writes = {}
     monkeypatch.setattr(cloud, "_client", SimpleNamespace(write_df=lambda df, p: writes.setdefault("df", df)))
     new = pd.DataFrame({"Date":[dt.date(2024,1,2)], "v":[2]})
-    cloud.incremental_load(stub_lq, "prices", new, increment_column="Date")
+    cloud.incremental_load(stub_lq, "prices", new, increment_column="Date", large_df=False)
     assert len(writes["df"]) == 2