luxorasap 0.1.15__py3-none-any.whl → 0.1.17__py3-none-any.whl

luxorasap/__init__.py

@@ -13,7 +13,7 @@ from types import ModuleType
 try:
     __version__: str = metadata.version(__name__)
 except metadata.PackageNotFoundError:  # editable install
-    __version__ = "0.1.15"
+    __version__ = "0.1.17"
 
 # ─── Lazy loader ─────────────────────────────────────────────────
 def __getattr__(name: str) -> ModuleType:

luxorasap/ingest/cloud/__init__.py

@@ -21,7 +21,8 @@ def save_table(
     index_name: str = "index",
     normalize_columns: bool = True,
     directory: str = "enriched/parquet",
-    override=False
+    override=False,
+    large_df: bool = False
 ):
     """Salva DataFrame como Parquet em ADLS (sobrescrevendo)."""
     
@@ -34,7 +35,7 @@ def save_table(
             return
     
     df = prep_for_save(df, index=index, index_name=index_name, normalize=normalize_columns)
-    _client.write_df(df.astype(str), f"{directory}/{table_name}.parquet")
+    _client.write_df(df.astype(str), f"{directory}/{table_name}.parquet", large_df=large_df)
 
 
 def incremental_load(
@@ -47,6 +48,7 @@ def incremental_load(
     index_name: str = "index",
     normalize_columns: bool = True,
     directory: str = "enriched/parquet",
+    large_df: bool = False
 ):
     """Concatena novos dados aos existentes, cortando duplicados pela data."""
     df["Last_Updated"] = dt.datetime.now()
@@ -64,5 +66,6 @@ def incremental_load(
         index_name=index_name,
         normalize_columns=normalize_columns,
         directory=directory,
-        override=True
+        override=True,
+        large_df=large_df
     )

luxorasap/utils/storage/blob.py

@@ -4,6 +4,7 @@ from datetime import timezone
 import pandas as pd
 import pyarrow as pa, pyarrow.parquet as pq
 from azure.storage.blob import BlobServiceClient
+import tempfile
 
 from ..dataframe import read_bytes
 
@@ -33,13 +34,30 @@ class BlobParquetClient:
             return None, False
             
 
-    def write_df(self, df, blob_path: str):
-        table = pa.Table.from_pandas(df)
-        buf = io.BytesIO()
-        pq.write_table(table, buf)
-        buf.seek(0)
-        self._blob(blob_path).upload_blob(buf, overwrite=True)
+    def write_df(self, df, blob_path: str, large_df: bool = False):
+        if not large_df:
+            table = pa.Table.from_pandas(df)
+            buf = io.BytesIO()
+            pq.write_table(table, buf)
+            buf.seek(0)
+            self._blob(blob_path).upload_blob(buf, overwrite=True)
         
+        else:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".parquet") as f:
+                writer = None
+                chunk_size = 100_000
+                for i in range(0, len(df), chunk_size):
+                    chunk = pa.Table.from_pandas(df.iloc[i:i+chunk_size])
+                    if writer is None:
+                        writer = pq.ParquetWriter(f.name, chunk.schema)
+                    writer.write_table(chunk)
+                writer.close()
+                
+                with open(f.name, "rb") as f_read:
+                    self._blob(blob_path).upload_blob(f_read, overwrite=True)
+                
+                os.remove(f.name)
+            
         
     def get_df_update_time(self, blob_path: str) -> float:
         try:

{luxorasap-0.1.15.dist-info → luxorasap-0.1.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: luxorasap
-Version: 0.1.15
+Version: 0.1.17
 Summary: Toolbox da Luxor para ingestão, análise e automação de dados financeiros.
 Author-email: Luxor Group <backoffice@luxor.com.br>
 License: Proprietary – All rights reserved

{luxorasap-0.1.15.dist-info → luxorasap-0.1.17.dist-info}/RECORD

@@ -1,4 +1,4 @@
-luxorasap/__init__.py,sha256=lHZk7izy83rJh7Ij2oSrUOFc4wecgAttn7OMbkT94AA,1356
+luxorasap/__init__.py,sha256=aPCQIYh22iaw5xRNSlLfrz65PJNvNMc4NXPrDfeLITs,1356
 luxorasap/btgapi/__init__.py,sha256=QUlfb5oiBY6K1Q5x4-a-x2wECe1At5wc2962I5odOJk,620
 luxorasap/btgapi/auth.py,sha256=PvyCtbEyBO2B1CIeAlNXWugKW1OgiKfPcVzS6K5FBnQ,1872
 luxorasap/btgapi/reports.py,sha256=ZVEMLoJPXc0r3XjPJPMsKQN0zZd1Npd7umNpAj1bncs,8040
@@ -6,16 +6,16 @@ luxorasap/btgapi/trades.py,sha256=956HZ9BvN9C_VQvKTyBLN0x6ZygwVqBZN11F7OnNbDI,59
 luxorasap/datareader/__init__.py,sha256=41RAvbrQ4R6oj67S32CrKqolx0CJ2W8cbOF6g5Cqm2g,120
 luxorasap/datareader/core.py,sha256=P8AjtRFRRmUrqjbjfKRb0wTLW2eHcUva8iWid4uh4PE,155123
 luxorasap/ingest/__init__.py,sha256=XhxDTN2ar-u6UCPhnxNU_to-nWiit-SpQ6cA_N9eMSs,795
-luxorasap/ingest/cloud/__init__.py,sha256=Hm-43YkjRhu3UiFVt1Zx7DmQuNJnK2GVWfCx1T3gKNc,2052
+luxorasap/ingest/cloud/__init__.py,sha256=P4GSvfC4JFMbTSpevhfvbfyn-zqiBMIMWhc_U0mQCFc,2153
 luxorasap/ingest/legacy_local/dataloader.py,sha256=zKPhuiBSFwkuWN6d8g2s60KkbVk1R_1cGMCtQM9j-0c,11908
 luxorasap/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 luxorasap/utils/dataframe/__init__.py,sha256=dU_RwTTOi6F3mlhM-0MYWM_qexBN9BmmKc_yrDE1Lwc,207
 luxorasap/utils/dataframe/reader.py,sha256=Vzjdw-AeS1lnWEHQ8RZNh0kK93NWTp0NWVi_B6mN5N0,616
 luxorasap/utils/dataframe/transforms.py,sha256=Bm_cv9L9923QIXH82Fa_M4pM94f2AJRPu62Vv_i7tto,1684
 luxorasap/utils/storage/__init__.py,sha256=U3XRq94yzRp3kgBSUcRzs2tQgJ4o8h8a1ZzwiscA5XM,67
-luxorasap/utils/storage/blob.py,sha256=pcEixGxwXM9y5iPPpkX__ySWq0milghJGketYZlRL-0,3171
-luxorasap-0.1.15.dist-info/METADATA,sha256=klkzeKwjA2gyPqPqxeI2QqlBMgqYSbiYS2cDTOtN2gM,3804
-luxorasap-0.1.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-luxorasap-0.1.15.dist-info/entry_points.txt,sha256=XFh-dOwUhlya9DmGvgookMI0ezyUJjcOvTIHDEYS44g,52
-luxorasap-0.1.15.dist-info/top_level.txt,sha256=9YOL6bUIpzY06XFBRkUW1e4rgB32Ds91fQPGwUEjxzU,10
-luxorasap-0.1.15.dist-info/RECORD,,
+luxorasap/utils/storage/blob.py,sha256=MVGOXnZR62jJPNWIM8sH_-DF-WuxMFVlpmi1llfos78,3949
+luxorasap-0.1.17.dist-info/METADATA,sha256=aqn2ND5T-_1YUIx1HuRWAxStin3ikanLYS0e9dTJ2F4,3804
+luxorasap-0.1.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+luxorasap-0.1.17.dist-info/entry_points.txt,sha256=XFh-dOwUhlya9DmGvgookMI0ezyUJjcOvTIHDEYS44g,52
+luxorasap-0.1.17.dist-info/top_level.txt,sha256=9YOL6bUIpzY06XFBRkUW1e4rgB32Ds91fQPGwUEjxzU,10
+luxorasap-0.1.17.dist-info/RECORD,,