ygg 0.1.60__py3-none-any.whl → 0.1.65__py3-none-any.whl

{ygg-0.1.60.dist-info → ygg-0.1.65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.60
+Version: 0.1.65
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License
@@ -207,7 +207,7 @@ License:                                  Apache License
         
 Project-URL: Homepage, https://github.com/Platob/Yggdrasil
 Project-URL: Repository, https://github.com/Platob/Yggdrasil
-Project-URL: Documentation, https://github.com/Platob/Yggdrasil/tree/main/python/docs
+Project-URL: Documentation, https://github.com/Platob/Yggdrasil
 Keywords: arrow,polars,pandas,spark,databricks,typing,dataclass,serialization
 Classifier: Development Status :: 3 - Alpha
 Classifier: Programming Language :: Python

{ygg-0.1.60.dist-info → ygg-0.1.65.dist-info}/RECORD

@@ -1,9 +1,9 @@
-ygg-0.1.60.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+ygg-0.1.65.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
 yggdrasil/__init__.py,sha256=4-ghPak2S6zfMqmnlxW2GCgPb5s79znpKa2hGEGXcE4,24
 yggdrasil/exceptions.py,sha256=NEpbDFn-8ZRsLiEgJicCwrTHNMWAGtdrTJzosfAeVJo,82
-yggdrasil/version.py,sha256=RzpPAn4AEKR5U8Ey0m3Oy_SvSsWT9yeRqhNeTwbK0ks,22
+yggdrasil/version.py,sha256=P0GENqTQLndQpX5Tkuaob2sv-oNWWzdsMw2PdmqDlFY,22
 yggdrasil/ai/__init__.py,sha256=YEOVsyuvEOvPaZT8XN9xNysS_WOpHTbKgXgnA8up7x0,52
-yggdrasil/ai/session.py,sha256=X4btr4OTPLzk1rZx0pZLMJ6Gni1DfEMghAmx9qI1qdE,2579
+yggdrasil/ai/session.py,sha256=10ATAnw8FOCpfIg9sNR4meki_MRckUzKZ9Uft4IXwLA,2515
 yggdrasil/ai/sql_session.py,sha256=n92tQjHUBIey6c3EJProiEEwfAtQm07Dtmei4WXzeG0,10812
 yggdrasil/databricks/__init__.py,sha256=0GRBP930ManOvyo-Y5E7bz7F2msnvU677OH6rxzPwd8,87
 yggdrasil/databricks/exceptions.py,sha256=-ZULt0wD5_Rxww11nk4Z46DvS5j18RdKR5ISmbQfUQA,142
@@ -16,22 +16,22 @@ yggdrasil/databricks/compute/remote.py,sha256=sF99i7GXZcC0GiNgO9VO0I26rFbrtnDhK9
 yggdrasil/databricks/jobs/__init__.py,sha256=snxGSJb0M5I39v0y3IR-uEeSlZR248cQ_4DJ1sYs-h8,154
 yggdrasil/databricks/jobs/config.py,sha256=9LGeHD04hbfy0xt8_6oobC4moKJh4_DTjZiK4Q2Tqjk,11557
 yggdrasil/databricks/sql/__init__.py,sha256=PetgRp1jEj5K3TgN09FwNUVjVN8YYuGq0cDIOTqsbns,144
-yggdrasil/databricks/sql/engine.py,sha256=6PVxrO78UxDD6jHBnDjKV2KuZ9JpuvwVckQ_mjsaeKw,49558
+yggdrasil/databricks/sql/engine.py,sha256=nFWeegs91CtjCLzxgZsJwOlAXNVI1v_lfecuFVfKFFY,49979
 yggdrasil/databricks/sql/exceptions.py,sha256=srMR3Y9LQm45rkyxfyCgpgcoGtRRvGKWBEoUHf4kxsg,1762
 yggdrasil/databricks/sql/statement_result.py,sha256=01DzFX1bGDIGHj0OW2ngfVVJ1w1KHlZEfAI934E35CU,15549
 yggdrasil/databricks/sql/types.py,sha256=5G-BM9_eOsRKEMzeDTWUsWW5g4Idvs-czVCpOCrMhdA,6412
-yggdrasil/databricks/sql/warehouse.py,sha256=W045PMLgZdt7f5w8aWIIX-vSLa5GE4V8yFffyrwZSOQ,18689
+yggdrasil/databricks/sql/warehouse.py,sha256=bCMWAci_E7pxIH1-9qSgwzpLztLsyiBFjZgME9dOXC8,18971
 yggdrasil/databricks/workspaces/__init__.py,sha256=dv2zotoFVhNFlTCdRq6gwf5bEzeZkOZszoNZMs0k59g,114
 yggdrasil/databricks/workspaces/filesytem.py,sha256=Z8JXU7_XUEbw9fpTQT1avRQKi-IAP2KemXBMPkUoY4w,9805
-yggdrasil/databricks/workspaces/io.py,sha256=IHmOwX1cWksvfunwTr03BFPqhm8cWNEkCwMvM9vhM80,35162
-yggdrasil/databricks/workspaces/path.py,sha256=R6-RuMG7fZYBWS4wRmbw0bOIxiGYpwif47jFLWmcLGs,56950
+yggdrasil/databricks/workspaces/io.py,sha256=RdgN5lmEYNF5phPRkRMCVHbUl-t3ZUGkKbzgYSTKpII,37420
+yggdrasil/databricks/workspaces/path.py,sha256=k3UB0LhF4hQI-Iza50D5dVjhqNsAdP4KabKiWK7bTWM,55775
 yggdrasil/databricks/workspaces/path_kind.py,sha256=rhWe1ky7uPD0du0bZSv2S4fK4C5zWd7zAF3UeS2iiPU,283
 yggdrasil/databricks/workspaces/volumes_path.py,sha256=s8CA33cG3jpMVJy5MILLlkEBcFg_qInDCF2jozLj1Fg,2431
-yggdrasil/databricks/workspaces/workspace.py,sha256=GEUp3f15SP5lUDx0_Ujzv5QtjVQg00WJRydR4rNLdXs,30216
+yggdrasil/databricks/workspaces/workspace.py,sha256=f8Ihv3fqo6YYno1yvkXnMpZhQQWwMkklhg-C9MSVyrE,30103
 yggdrasil/dataclasses/__init__.py,sha256=_RkhfF3KC1eSORby1dzvBXQ0-UGG3u6wyUQWX2jq1Pc,108
 yggdrasil/dataclasses/dataclass.py,sha256=LxrCjwvmBnb8yRI_N-c31RHHxB4XoJPixmKg9iBIuaI,1148
 yggdrasil/libs/__init__.py,sha256=zdC9OU0Xy36CLY9mg2drxN6S7isPR8aTLzJA6xVIeLE,91
-yggdrasil/libs/databrickslib.py,sha256=t_0b_3iCGFPjBrJaIOvNzSEn5pjZBTbY_fOcDHp6qx8,1135
+yggdrasil/libs/databrickslib.py,sha256=Y99ARtrVKVBTH0qZ0njYr1Oa_757wtsVY4ywH07IdQ4,1109
 yggdrasil/libs/pandaslib.py,sha256=_U4sdFvLAFD16_65RG-RFmcx4c3fvVnALESFaAlT71M,887
 yggdrasil/libs/polarslib.py,sha256=WnnERtMTl__ZPidcZkoV7mb8-c680zcAnJgzAoD3ZE8,1437
 yggdrasil/libs/sparklib.py,sha256=FQ3W1iz2EIpQreorOiQuFt15rdhq2QhGEAWp8Zrbl9A,10177
@@ -44,19 +44,19 @@ yggdrasil/pyutils/equality.py,sha256=Xyf8D1dLUCm3spDEir8Zyj7O4US_fBJwEylJCfJ9slI
 yggdrasil/pyutils/exceptions.py,sha256=1c0xxFvGML5gkDPGzD_Tgw1ff9bGMVygH8ASgeoII2E,3889
 yggdrasil/pyutils/expiring_dict.py,sha256=pr2u25LGwPVbLfsLptiHGovUtYRRo0AMjaJtCtJl7nQ,8477
 yggdrasil/pyutils/mimetypes.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-yggdrasil/pyutils/modules.py,sha256=B7IP99YqUMW6-DIESFzBx8-09V1d0a8qrIJUDFhhL2g,11424
+yggdrasil/pyutils/modules.py,sha256=a0YWunsuA-D-Ho41LrwkN6o7e88NMk7oOzrSzlG0kPQ,11488
 yggdrasil/pyutils/parallel.py,sha256=ubuq2m9dJzWYUyKCga4Y_9bpaeMYUrleYxdp49CHr44,6781
-yggdrasil/pyutils/python_env.py,sha256=OFvM0wxVzB1iHC4BGra0tLE3sZL1ZaIr5j9dtDeLaiU,51098
+yggdrasil/pyutils/python_env.py,sha256=d9s6i_fXz6j4f5BnigKxR0dEUJ-4BmqjBD2u2ybX29U,51025
 yggdrasil/pyutils/retry.py,sha256=gXBtn1DdmIYIUmGKOUr8-SUT7MOu97LykN2YR4uocgc,11917
 yggdrasil/pyutils/waiting_config.py,sha256=WiMOiKyGR5iKr83YK4dljn7OCaDpxXMUx8cz-bUNGMg,6255
 yggdrasil/requests/__init__.py,sha256=dMesyzq97_DmI765x0TwaDPEfsxFtgGNgchk8LvEN-o,103
-yggdrasil/requests/msal.py,sha256=s2GCyzbgFdgdlJ1JqMrZ4qYVbmoG46-ZOTcaVQhZ-sQ,9220
+yggdrasil/requests/msal.py,sha256=XSuKsxEIApfygiWOBBOok_trQk3eeNb5P0f3RAUrtss,6666
 yggdrasil/requests/session.py,sha256=SLnrgHY0Lby7ZxclRFUjHdfM8euN_8bSQEWl7TkJY2U,1461
 yggdrasil/types/__init__.py,sha256=CrLiDeYNM9fO975sE5ufeVKcy7Ca702IsaG2Pk8T3YU,139
 yggdrasil/types/dummy_class.py,sha256=XXM3_ljL4XfY5LeF-WTj-myqHaKAUmWZ23cPDrXAnBM,2327
-yggdrasil/types/file_format.py,sha256=yqAadZ5z6CrctsQO0ZmEY7eGXLbhBUnvvNOwkPSk0GU,133
+yggdrasil/types/file_format.py,sha256=P-3JTa9FzhHj-ndWMGgsF0zxlR_V2q3a_p2R2CwSoRs,273
 yggdrasil/types/python_arrow.py,sha256=mOhyecAxa5u8JWsyTO26OMOWimHHgwLKWlkNSAyIVas,25636
-yggdrasil/types/python_defaults.py,sha256=GO3hZBZcwRHs9qiXes75y8l5X00kZHTfEC7el_x73uw,10184
+yggdrasil/types/python_defaults.py,sha256=kT7vuNDxzP_5tsy0aOkzVh1sZN7rKR7mky9nrYiFkl0,11063
 yggdrasil/types/cast/__init__.py,sha256=Oft3pTs2bRM5hT7YqJAuOKTYYk-SACLaMOXUVdafy_I,311
 yggdrasil/types/cast/arrow_cast.py,sha256=IZstOcHjLKPy62TFGgjMSW3ttPGt3hMi6RmDw-92T0E,41623
 yggdrasil/types/cast/cast_options.py,sha256=nDaEvCCs7TBamhTWyDrYf3LVaBWzioIP2Q5_LXrChF4,15532
@@ -67,8 +67,8 @@ yggdrasil/types/cast/registry.py,sha256=OOqIfbIjPH-a3figvu-zTvEtUDTEWhe2xIl3cCA4
 yggdrasil/types/cast/spark_cast.py,sha256=_KAsl1DqmKMSfWxqhVE7gosjYdgiL1C5bDQv6eP3HtA,24926
 yggdrasil/types/cast/spark_pandas_cast.py,sha256=BuTiWrdCANZCdD_p2MAytqm74eq-rdRXd-LGojBRrfU,5023
 yggdrasil/types/cast/spark_polars_cast.py,sha256=btmZNHXn2NSt3fUuB4xg7coaE0RezIBdZD92H8NK0Jw,9073
-ygg-0.1.60.dist-info/METADATA,sha256=LBZYw5kRHouaxOl7x_dskRBUpH-XXDk-XaFrSLGKrg0,18528
-ygg-0.1.60.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-ygg-0.1.60.dist-info/entry_points.txt,sha256=6q-vpWG3kvw2dhctQ0LALdatoeefkN855Ev02I1dKGY,70
-ygg-0.1.60.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
-ygg-0.1.60.dist-info/RECORD,,
+ygg-0.1.65.dist-info/METADATA,sha256=QQp-Hf_yN9HVD8Cjs6xgHzIaJ1g0GTJ7RODqgthqkh0,18506
+ygg-0.1.65.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ygg-0.1.65.dist-info/entry_points.txt,sha256=6q-vpWG3kvw2dhctQ0LALdatoeefkN855Ev02I1dKGY,70
+ygg-0.1.65.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
+ygg-0.1.65.dist-info/RECORD,,

yggdrasil/ai/session.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional
 
-from yggdrasil.types.dummy_class import DummyModuleClass
+from ..types.dummy_class import DummyModuleClass
 
 try:
     from openai import OpenAI
@@ -19,8 +19,6 @@ __all__ = ["AISession"]
 class AISession(ABC):
     api_key: str
     base_url: str
-
-    # Gemini default (via OpenAI-compatible gateway)
     model: str = "gemini-2.5-flash"
 
     client: OpenAI = field(init=False)

yggdrasil/databricks/sql/engine.py

@@ -17,7 +17,7 @@ import random
 import string
 import time
 from threading import Thread
-from typing import Optional, Union, Any, Dict, List, Literal
+from typing import Optional, Union, Any, Dict, List, Literal, TYPE_CHECKING
 
 import pyarrow as pa
 import pyarrow.dataset as pds
@@ -26,11 +26,10 @@ from .statement_result import StatementResult
 from .types import column_info_to_arrow_field
 from .warehouse import SQLWarehouse
 from ..workspaces import WorkspaceService, DatabricksPath
-from ...ai.sql_session import SQLAISession, SQLFlavor
 from ...libs.databrickslib import databricks_sdk, DatabricksDummyClass
 from ...libs.sparklib import SparkSession, SparkDataFrame, pyspark
 from ...pyutils.waiting_config import WaitingConfigArg
-from ...types import is_arrow_type_string_like, is_arrow_type_binary_like
+from ...types import is_arrow_type_string_like, is_arrow_type_binary_like, cast_arrow_tabular
 from ...types.cast.cast_options import CastOptions
 from ...types.cast.registry import convert
 from ...types.cast.spark_cast import cast_spark_dataframe
@@ -63,6 +62,10 @@ if pyspark is not None:
     import pyspark.sql.functions as F
 
 
+if TYPE_CHECKING:
+    from ...ai.sql_session import SQLAISession, SQLFlavor
+
+
 __all__ = [
     "SQLEngine",
     "StatementResult"
@@ -101,7 +104,7 @@ class SQLEngine(WorkspaceService):
     schema_name: Optional[str] = None
 
     _warehouse: Optional[SQLWarehouse] = dataclasses.field(default=None, repr=False, hash=False, compare=False)
-    _ai_session: Optional[SQLAISession] = dataclasses.field(default=None, repr=False, hash=False, compare=False)
+    _ai_session: Optional["SQLAISession"] = dataclasses.field(default=None, repr=False, hash=False, compare=False)
 
     def table_full_name(
         self,
@@ -198,8 +201,13 @@ class SQLEngine(WorkspaceService):
     def ai_session(
         self,
         model: str = "databricks-gemini-2-5-pro",
-        flavor: SQLFlavor = SQLFlavor.DATABRICKS
+        flavor: Optional["SQLFlavor"] = None
     ):
+        from ...ai.sql_session import SQLAISession, SQLFlavor
+
+        if flavor is None:
+            flavor = SQLFlavor.DATABRICKS
+
         return SQLAISession(
             model=model,
             api_key=self.workspace.current_token(),
@@ -224,7 +232,7 @@ class SQLEngine(WorkspaceService):
         catalog_name: Optional[str] = None,
         schema_name: Optional[str] = None,
         wait: Optional[WaitingConfigArg] = True
-    ) -> "StatementResult":
+    ) -> StatementResult:
         """Execute a SQL statement via Spark or Databricks SQL Statement Execution API.
 
         Engine resolution:
@@ -504,10 +512,13 @@ class SQLEngine(WorkspaceService):
                             logger.exception("Failed to drop table %s after auto creation error", location)
                         raise
 
-            data_tbl = convert(
-                data, pa.Table,
-                options=cast_options, target_field=existing_schema
-            )
+            cast_options = CastOptions.check_arg(options=cast_options, target_field=existing_schema)
+
+            if isinstance(data, (pa.Table, pa.RecordBatch)):
+                data_tbl = cast_arrow_tabular(data, options=cast_options)
+            else:
+                data_tbl = convert(data, pa.Table, options=cast_options)
+
             num_rows = data_tbl.num_rows
 
             logger.debug(
@@ -524,7 +535,8 @@ class SQLEngine(WorkspaceService):
                 catalog_name=catalog_name,
                 schema_name=schema_name,
                 volume_name="tmp",
-                extension="parquet"
+                extension="parquet",
+                max_lifetime=3600,
             ) if temp_volume_path is None else DatabricksPath.parse(obj=temp_volume_path, workspace=connected.workspace)
 
             logger.debug("Staging Parquet to temp volume: %s", temp_volume_path)
@@ -575,7 +587,7 @@ FROM parquet.`{temp_volume_path}`"""
             finally:
                 try:
                     Thread(
-                        target=temp_volume_path.rmdir,
+                        target=temp_volume_path.remove,
                         kwargs={
                             "recursive": True
                         }

yggdrasil/databricks/sql/warehouse.py

@@ -256,6 +256,7 @@ class SQLWarehouse(WorkspaceService):
         elif self.warehouse_id:
             return self
 
+        starter_warehouse, starter_name = None, "Serverless Starter Warehouse"
         warehouse_name = warehouse_name or self.warehouse_name or self._make_default_name(enable_serverless_compute=True)
 
         if warehouse_name:
@@ -284,8 +285,15 @@ class SQLWarehouse(WorkspaceService):
                         warehouse_name=warehouse_name,
                         warehouse_id=warehouse.warehouse_id
                     )
+
                     return warehouse
 
+                elif warehouse.warehouse_name == starter_name:
+                    starter_warehouse = warehouse
+
+        if starter_warehouse is not None:
+            return starter_warehouse
+
         if raise_error:
             v = warehouse_name or warehouse_id
 

yggdrasil/databricks/workspaces/io.py

@@ -3,28 +3,25 @@
 import base64
 import io
 import logging
+import os
 import time
 from abc import ABC, abstractmethod
+from tempfile import SpooledTemporaryFile
 from threading import Thread
-from typing import TYPE_CHECKING, Optional, IO, AnyStr, Union, Any
+from typing import TYPE_CHECKING, Optional, IO, AnyStr, Union, Any, BinaryIO
 
 import dill
 import pyarrow as pa
 import pyarrow.csv as pcsv
 import pyarrow.parquet as pq
-from pyarrow.dataset import (
-    FileFormat,
-    ParquetFileFormat,
-    CsvFileFormat,
-)
 
 from .path_kind import DatabricksPathKind
 from ...libs.databrickslib import databricks
 from ...libs.pandaslib import PandasDataFrame
 from ...libs.polarslib import polars, PolarsDataFrame
-from ...pyutils import retry
+from ...pyutils.retry import retry
 from ...types.cast.registry import convert
-from ...types.file_format import ExcelFileFormat
+from ...types.file_format import FileFormat, ParquetFileFormat, CsvFileFormat, ExcelFileFormat
 
 if databricks is not None:
     from databricks.sdk.service.workspace import ImportFormat, ExportFormat
@@ -45,7 +42,64 @@ __all__ = [
 
 
 LOGGER = logging.getLogger(__name__)
+_SPOOL_MAX = 64 * 1024 * 1024   # 64MB in RAM then spill to disk
+_COPY_CHUNK = 8 * 1024 * 1024   # 8MB chunks
+
+def _prepare_binaryio_and_size(
+    data: Union[bytes, bytearray, memoryview, BinaryIO]
+) -> tuple[int, BinaryIO, bool]:
+    """
+    Returns (size, bio, should_close).
+
+    - bytes-like -> wrap in BytesIO (closeable by us).
+    - seekable file -> compute size via fstat or seek/tell.
+    - non-seekable stream -> spool into SpooledTemporaryFile, count bytes.
+    """
+    # bytes-like
+    if isinstance(data, (bytes, bytearray, memoryview)):
+        b = bytes(data)
+        return len(b), io.BytesIO(b), True
+
+    f: BinaryIO = data
+
+    # 1) try OS-level size for real files
+    try:
+        fileno = f.fileno()  # type: ignore[attr-defined]
+    except Exception:
+        fileno = None
+
+    if fileno is not None:
+        try:
+            st = os.fstat(fileno)
+            # rewind if possible
+            try:
+                f.seek(0)
+            except Exception:
+                pass
+            return int(st.st_size), f, False
+        except Exception:
+            pass
+
+    # 2) try seek/tell (seekable streams)
+    try:
+        f.seek(0, io.SEEK_END)
+        end = f.tell()
+        f.seek(0)
+        return int(end), f, False
+    except Exception:
+        pass
 
+    # 3) non-seekable stream: spool + count
+    spooled = SpooledTemporaryFile(max_size=_SPOOL_MAX, mode="w+b")
+    size = 0
+    while True:
+        chunk = f.read(_COPY_CHUNK)
+        if not chunk:
+            break
+        spooled.write(chunk)
+        size += len(chunk)
+    spooled.seek(0)
+    return size, spooled, True
 
 class DatabricksIO(ABC, IO):
     """File-like interface for Databricks workspace, volume, or DBFS paths."""
@@ -102,7 +156,10 @@ class DatabricksIO(ABC, IO):
         return self.path.__hash__()
 
     def __str__(self):
-        return self.path.__str__()
+        return "%s(path=%s)" % (
+            self.__class__.__name__,
+            self.path.__repr__()
+        )
 
     def __repr__(self):
         return "%s(path=%s)" % (
@@ -1081,9 +1138,9 @@ class DatabricksVolumeIO(DatabricksIO):
 
         try:
             resp = client.download(full_path)
-        except Exception as e:
+        except (NotFound, ResourceDoesNotExist, BadRequest, InternalError) as e:
             # Databricks SDK exceptions vary a bit by version; keep it pragmatic.
-            if allow_not_found and any(s in str(e).lower() for s in ("not found", "not exist", "404")):
+            if allow_not_found:
                 return b""
             raise
 
@@ -1096,53 +1153,61 @@ class DatabricksVolumeIO(DatabricksIO):
         end = start + length
         return data[start:end]
 
-    @retry(exceptions=(InternalError,))
-    def write_all_bytes(self, data: Union[bytes, IO[bytes]]):
-        """Write bytes to a volume file.
-
-        Args:
-            data: Union[bytes, IO[bytes]] to write.
-
-        Returns:
-            The DatabricksVolumeIO instance.
-        """
+    def write_all_bytes(
+        self,
+        data: Union[bytes, bytearray, memoryview, BinaryIO],
+        *,
+        overwrite: bool = True,
+        part_size: Optional[int] = None,
+        use_parallel: bool = True,
+        parallelism: Optional[int] = None,
+    ):
+        """Write bytes/stream to a volume file safely (BinaryIO upload)."""
         sdk = self.workspace.sdk()
         client = sdk.files
         full_path = self.path.files_full_path()
 
-        LOGGER.debug(
-            "Writing all bytes in %s",
-            self
-        )
+        LOGGER.debug("Writing all bytes in %s", self)
 
-        try:
-            client.upload(
-                full_path,
-                io.BytesIO(data),
-                overwrite=True
-            )
-        except (NotFound, ResourceDoesNotExist, BadRequest):
-            self.path.parent.mkdir(parents=True, exist_ok=True)
+        size, bio, should_close = _prepare_binaryio_and_size(data)
 
-            client.upload(
+        def _upload():
+            return client.upload(
                 full_path,
-                io.BytesIO(data),
-                overwrite=True
+                bio,
+                overwrite=overwrite,
+                part_size=part_size,
+                use_parallel=use_parallel,
+                parallelism=parallelism,
             )
 
-        LOGGER.info(
-            "Written all bytes in %s",
-            self
-        )
+        try:
+            _ = _upload()
+        except (NotFound, ResourceDoesNotExist, BadRequest, InternalError):
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            # Important: rewind if possible before retry
+            try:
+                bio.seek(0)
+            except Exception:
+                pass
+            _ = _upload()
+        finally:
+            if should_close:
+                try:
+                    bio.close()
+                except Exception:
+                    pass
 
         self.path.reset_metadata(
             is_file=True,
             is_dir=False,
-            size=len(data),
-            mtime=time.time()
+            size=size,
+            mtime=time.time(),
         )
 
-        return self
+        LOGGER.info("Written %s bytes in %s", size or "all", self.path)
+
+        return self  # or return result if your API prefers that
 
 
 class DatabricksDBFSIO(DatabricksIO):

yggdrasil/databricks/workspaces/path.py

@@ -1,8 +1,6 @@
 """Databricks path abstraction spanning DBFS, workspace, and volumes."""
 
 # src/yggdrasil/databricks/workspaces/databricks_path.py
-from __future__ import annotations
-
 import dataclasses
 import datetime as dt
 import io
@@ -15,9 +13,7 @@ from typing import Optional, Tuple, Union, TYPE_CHECKING, List, Any, IO
 
 import dill
 import pyarrow as pa
-import pyarrow.dataset as ds
 from pyarrow import ArrowInvalid
-from pyarrow.dataset import FileFormat, ParquetFileFormat, CsvFileFormat, JsonFileFormat
 from pyarrow.fs import FileInfo, FileType, FileSystem
 
 from .io import DatabricksIO
@@ -25,12 +21,9 @@ from .path_kind import DatabricksPathKind
 from .volumes_path import get_volume_status, get_volume_metadata
 from ...libs.databrickslib import databricks
 from ...libs.pandaslib import PandasDataFrame
-from ...libs.polarslib import polars, PolarsDataFrame
-from ...types.cast.cast_options import CastOptions
-from ...types.cast.pandas_cast import pandas_converter, cast_pandas_dataframe
-from ...types.cast.polars_cast import polars_converter, cast_polars_dataframe
-from ...types.cast.registry import convert, register_converter
-from ...types.file_format import ExcelFileFormat
+from ...libs.polarslib import polars
+from ...types.cast.registry import convert
+from ...types.file_format import FileFormat, ExcelFileFormat, ParquetFileFormat, JsonFileFormat, CsvFileFormat
 
 if databricks is not None:
     from databricks.sdk.errors import InternalError
@@ -1305,6 +1298,8 @@ class DatabricksPath:
         Returns:
             A PyArrow Dataset instance.
         """
+        import pyarrow.dataset as ds
+
         filesystem = self.filesystem(workspace=workspace) if filesystem is None else filesystem
 
         return ds.dataset(
@@ -1684,32 +1679,3 @@ class DatabricksPath:
             raise ValueError(
                 "Invalid engine %s, must be in duckdb, polars" % engine
             )
-
-if databricks is not None:
-    @register_converter(DatabricksPath, ds.Dataset)
-    def databricks_path_to_arrow_table(
-        data: DatabricksPath,
-        options: Optional[CastOptions] = None,
-    ) -> ds.Dataset:
-        return data.arrow_dataset()
-
-
-    @pandas_converter(DatabricksPath, PandasDataFrame)
-    def databricks_path_to_pandas(
-        data: DatabricksPath,
-        options: Optional[CastOptions] = None,
-    ) -> PolarsDataFrame:
-        return cast_pandas_dataframe(
-            data.read_pandas(),
-            options
-        )
-
-    @polars_converter(DatabricksPath, PolarsDataFrame)
-    def databricks_path_to_polars(
-        data: DatabricksPath,
-        options: Optional[CastOptions] = None,
-    ) -> PolarsDataFrame:
-        return cast_polars_dataframe(
-            data.read_polars(),
-            options
-        )

yggdrasil/databricks/workspaces/workspace.py

@@ -520,9 +520,9 @@ class Workspace:
         Returns:
             A DatabricksPath pointing at the shared cache location.
         """
-        start = int(time.time() * 1000)
-        max_lifetime = max_lifetime or 48.0 * 3600.0
-        end = int(start + max_lifetime)
+        start = int(time.time())
+        max_lifetime = int(max_lifetime or 48 * 3600)
+        end = max(0, int(start + max_lifetime))
 
         base_path = base_path or self._base_tmp_path(
             catalog_name=catalog_name,
@@ -575,19 +575,15 @@ class Workspace:
                 base_path
             )
 
-            try:
-                for path in base_path.ls(recursive=False, allow_not_found=True):
+            for path in base_path.ls(recursive=False, allow_not_found=True):
+                if path.name.startswith("tmp"):
                     parts = path.name.split("-")
 
                     if len(parts) > 2 and parts[0] == "tmp" and parts[1].isdigit() and parts[2].isdigit():
-                        end = int(parts[2]) / 1000.0
+                        end = int(parts[2])
 
                         if end and time.time() > end:
                             path.remove(recursive=True)
-            except Exception as e:
-                if raise_error:
-                    raise e
-                LOGGER.warning(e)
 
             LOGGER.info(
                 "Cleaned temp path %s",

yggdrasil/libs/databrickslib.py

@@ -1,5 +1,5 @@
 """Optional Databricks SDK dependency helpers."""
-from yggdrasil.types.dummy_class import DummyModuleClass
+from ..types.dummy_class import DummyModuleClass
 
 
 class DatabricksDummyClass(DummyModuleClass):
@@ -25,7 +25,7 @@ def require_databricks_sdk():
 
 try:
     import databricks
-    import databricks.sdk  # type: ignore
+    import databricks.sdk
 
     from databricks.sdk import WorkspaceClient
 
@@ -34,7 +34,6 @@ try:
 except ImportError:
     databricks = DatabricksDummyClass
     databricks_sdk = DatabricksDummyClass
-
     WorkspaceClient = DatabricksDummyClass
 
 

yggdrasil/pyutils/modules.py

@@ -42,7 +42,7 @@ MODULE_PROJECT_NAMES_ALIASES = {
     "yggdrasil": "ygg",
     "jwt": "PyJWT",
 }
-
+DEFAULT_PIP_INDEX_SETTINGS = None
 
 def module_name_to_project_name(module_name: str) -> str:
     """Map module import names to PyPI project names when they differ.
@@ -264,6 +264,11 @@ class PipIndexSettings:
         Returns:
             Default PipIndexSettings instance.
         """
+        global DEFAULT_PIP_INDEX_SETTINGS
+
+        if DEFAULT_PIP_INDEX_SETTINGS is None:
+            DEFAULT_PIP_INDEX_SETTINGS = get_pip_index_settings()
+
         return DEFAULT_PIP_INDEX_SETTINGS
 
     @property
@@ -363,9 +368,3 @@ def get_pip_index_settings() -> PipIndexSettings:
             extra_index_urls.append(u)
 
     return PipIndexSettings(index_url=index_url, extra_index_urls=extra_index_urls, sources=sources)
-
-
-try:
-    DEFAULT_PIP_INDEX_SETTINGS = get_pip_index_settings()
-except:
-    DEFAULT_PIP_INDEX_SETTINGS = PipIndexSettings()

yggdrasil/pyutils/python_env.py

@@ -27,7 +27,6 @@ log = logging.getLogger(__name__)
 
 class PythonEnvError(RuntimeError):
     """Raised when Python environment operations fail."""
-
     pass
 
 
@@ -72,6 +71,9 @@ _NON_PIPABLE_RE = re.compile(
     re.IGNORECASE,
 )
 
+# Snapshot singleton (import-time)
+CURRENT_PYTHON_ENV: "PythonEnv" = None
+
 
 
 def _filter_non_pipable_linux_packages(requirements: Iterable[str]) -> List[str]:
@@ -1508,11 +1510,3 @@ print("RESULT:" + json.dumps(top_level))""".strip()
             log.error("python_env CLI error: %s", e)
             print(f"ERROR: {e}", file=sys.stderr)
             return 2
-
-
-# Snapshot singleton (import-time)
-CURRENT_PYTHON_ENV: PythonEnv = None
-
-
-if __name__ == "__main__":
-    raise SystemExit(PythonEnv.cli())

yggdrasil/requests/msal.py

@@ -3,12 +3,8 @@
 # auth_session.py
 import os
 import time
-from typing import Any, Mapping, Optional
-
-import urllib3
-
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from typing import Any, Optional
 
 from .session import YGGSession
 
@@ -38,11 +34,11 @@ class MSALAuth:
         authority: Optional authority URL override.
         scopes: List of scopes to request.
     """
-    tenant_id: Optional[str] = None
-    client_id: Optional[str] = None
-    client_secret: Optional[str] = None
-    authority: Optional[str] = None
-    scopes: list[str] | None = None
+    tenant_id: Optional[str] = field(default_factory=lambda: os.environ.get("AZURE_TENANT_ID"))
+    client_id: Optional[str] = field(default_factory=lambda: os.environ.get("AZURE_CLIENT_ID"))
+    client_secret: Optional[str] = field(default_factory=lambda: os.environ.get("AZURE_CLIENT_SECRET"))
+    authority: Optional[str] = field(default_factory=lambda: os.environ.get("AZURE_AUTHORITY"))
+    scopes: list[str] | None = field(default_factory=lambda: os.environ.get("AZURE_SCOPES"))
 
     _auth_app: ConfidentialClientApplication | None = None
     _expires_at: float | None = None
@@ -77,97 +73,15 @@ class MSALAuth:
         Returns:
             None.
         """
-        self.tenant_id = self.tenant_id or os.environ.get("AZURE_TENANT_ID")
-        self.client_id = self.client_id or os.environ.get("AZURE_CLIENT_ID")
-        self.client_secret = self.client_secret or os.environ.get("AZURE_CLIENT_SECRET")
-
-        self.authority = self.authority or os.environ.get("AZURE_AUTHORITY")
         if not self.authority:
+            assert self.tenant_id, "tenant_id is required to build authority URL"
+
             self.authority = f"https://login.microsoftonline.com/{self.tenant_id}"
 
-        self.scopes = self.scopes or os.environ.get("AZURE_SCOPES")
         if self.scopes:
             if isinstance(self.scopes, str):
                 self.scopes = self.scopes.split(",")
 
-        self._validate_config()
-
-    def _validate_config(self):
-        """Validate that all required configuration is present.
-
-        Returns:
-            None.
-        """
-        missing = []
-
-        if not self.client_id:
-            missing.append("azure_client_id (AZURE_CLIENT_ID)")
-        if not self.client_secret:
-            missing.append("azure_client_secret (AZURE_CLIENT_SECRET)")
-        if not self.tenant_id:
-            missing.append("azure_client_secret (AZURE_TENANT_ID)")
-        if not self.scopes:
-            missing.append("scopes (AZURE_SCOPES)")
-
-        if missing:
-            raise ValueError(f"Missing required configuration: {', '.join(missing)}")
-
-    @classmethod
-    def find_in_env(
-        cls,
-        env: Mapping = None,
-        prefix: Optional[str] = None
-    ) -> "MSALAuth":
-        """Return an MSALAuth built from environment variables if available.
-
-        Args:
-            env: Mapping to read variables from; defaults to os.environ.
-            prefix: Optional prefix for variable names.
-
-        Returns:
-            A configured MSALAuth instance or None.
-        """
-        if not env:
-            env = os.environ
-        prefix = prefix or "AZURE_"
-
-        required = {
-            key: env.get(prefix + key.upper())
-            for key in (
-                "client_id", "client_secret", "tenant_id", "scopes"
-            )
-        }
-
-        if all(required.values()):
-            scopes = required["scopes"].split(",") if required["scopes"] else None
-            return MSALAuth(
-                tenant_id=required["tenant_id"],
-                client_id=required["client_id"],
-                client_secret=required["client_secret"],
-                scopes=scopes,
-                authority=env.get(prefix + "AUTHORITY"),
-            )
-
-        return None
-
-    def export_to(self, to: dict = os.environ):
-        """Export the auth configuration to the provided mapping.
-
-        Args:
-            to: Mapping to populate with auth configuration values.
-
-        Returns:
-            None.
-        """
-        for key, value in (
-            ("AZURE_CLIENT_ID", self.client_id),
-            ("AZURE_CLIENT_SECRET", self.client_secret),
-            ("AZURE_AUTHORITY", self.authority),
-            ("AZURE_SCOPES", ",".join(self.scopes)),
-        ):
-            if value:
-                to[key] = value
-
     @property
     def auth_app(self) -> ConfidentialClientApplication:
         """Return or initialize the MSAL confidential client.
@@ -298,7 +212,6 @@ class MSALSession(YGGSession):
         super().__init__(*args, **kwargs)
         self.msal_auth = msal_auth
 
-
     def prepare_request(self, request):
         """Prepare the request with an Authorization header when needed.
 

yggdrasil/types/file_format.py

@@ -1,8 +1,12 @@
-from pyarrow.dataset import FileFormat
+from pyarrow.dataset import FileFormat, ParquetFileFormat, CsvFileFormat, JsonFileFormat
 
 
 __all__ = [
-    "ExcelFileFormat"
+    "FileFormat",
+    "ExcelFileFormat",
+    "ParquetFileFormat",
+    "CsvFileFormat",
+    "JsonFileFormat"
 ]
 
 

yggdrasil/types/python_defaults.py

@@ -18,84 +18,96 @@ __all__ = [
     "default_arrow_array"
 ]
 
+DEFAULT_MAPS_INITIALIZED = False
 
 _NONE_TYPE = type(None)
-_PRIMITIVE_DEFAULTS = {
-    str: "",
-    int: 0,
-    float: 0.0,
-    bool: False,
-    bytes: b"",
-}
-
-_SPECIAL_DEFAULTS = {
-    datetime.datetime: lambda: datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc),
-    datetime.date: lambda: datetime.date(1970, 1, 1),
-    datetime.time: lambda: datetime.time(0, 0, 0, tzinfo=datetime.timezone.utc),
-    datetime.timedelta: lambda: datetime.timedelta(0),
-    uuid.UUID: lambda: uuid.UUID(int=0),
-    decimal.Decimal: lambda: decimal.Decimal(0),
-}
-
-_ARROW_DEFAULTS = {
-    pa.null(): pa.scalar(None, type=pa.null()),
-
-    pa.bool_(): pa.scalar(False, type=pa.bool_()),
-
-    pa.int8(): pa.scalar(0, type=pa.int8()),
-    pa.int16(): pa.scalar(0, type=pa.int16()),
-    pa.int32(): pa.scalar(0, type=pa.int32()),
-    pa.int64(): pa.scalar(0, type=pa.int64()),
-
-    pa.uint8(): pa.scalar(0, type=pa.uint8()),
-    pa.uint16(): pa.scalar(0, type=pa.uint16()),
-    pa.uint32(): pa.scalar(0, type=pa.uint32()),
-    pa.uint64(): pa.scalar(0, type=pa.uint64()),
-
-    # pa.float16(): pa.scalar(0.0, type=pa.float16()),
-    pa.float32(): pa.scalar(0.0, type=pa.float32()),
-    pa.float64(): pa.scalar(0.0, type=pa.float64()),
-
-    pa.string(): pa.scalar("", type=pa.string()),
-    pa.string_view(): pa.scalar("", type=pa.string_view()),
-    pa.large_string(): pa.scalar("", type=pa.large_string()),
-
-    pa.binary(): pa.scalar(b"", type=pa.binary()),
-    pa.binary_view(): pa.scalar(b"", type=pa.binary_view()),
-    pa.large_binary(): pa.scalar(b"", type=pa.large_binary()),
-}
-
-
-try:
-    import polars
-
-    polars = polars
-
-    _POLARS_DEFAULTS = {
-        polars.Null(): None,
-        polars.Boolean(): False,
-
-        polars.Binary(): b"",
-
-        polars.Utf8(): "",
-
-        polars.Int8(): 0,
-        polars.Int16(): 0,
-        polars.Int32(): 0,
-        polars.Int64(): 0,
-
-        polars.UInt8(): 0,
-        polars.UInt16(): 0,
-        polars.UInt32(): 0,
-        polars.UInt64(): 0,
-
-        polars.Float32(): 0.0,
-        polars.Float64(): 0.0,
-    }
-except ImportError:
-    polars = None
-
-    _POLARS_DEFAULTS = {}
+_ARROW_DEFAULTS = {}
+_POLARS_DEFAULTS = {}
+_PRIMITIVE_DEFAULTS = {}
+_SPECIAL_DEFAULTS = {}
+
+
+def ensure_default_maps_initialized():
+    global DEFAULT_MAPS_INITIALIZED
+    global _PRIMITIVE_DEFAULTS
+    global _SPECIAL_DEFAULTS
+    global _ARROW_DEFAULTS
+    global _POLARS_DEFAULTS
+
+    if not DEFAULT_MAPS_INITIALIZED:
+        _PRIMITIVE_DEFAULTS = {
+            str: "",
+            int: 0,
+            float: 0.0,
+            bool: False,
+            bytes: b"",
+        }
+
+        _SPECIAL_DEFAULTS = {
+            datetime.datetime: lambda: datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc),
+            datetime.date: lambda: datetime.date(1970, 1, 1),
+            datetime.time: lambda: datetime.time(0, 0, 0, tzinfo=datetime.timezone.utc),
+            datetime.timedelta: lambda: datetime.timedelta(0),
+            uuid.UUID: lambda: uuid.UUID(int=0),
+            decimal.Decimal: lambda: decimal.Decimal(0),
+        }
+
+        _ARROW_DEFAULTS = {
+            pa.null(): pa.scalar(None, type=pa.null()),
+
+            pa.bool_(): pa.scalar(False, type=pa.bool_()),
+
+            pa.int8(): pa.scalar(0, type=pa.int8()),
+            pa.int16(): pa.scalar(0, type=pa.int16()),
+            pa.int32(): pa.scalar(0, type=pa.int32()),
+            pa.int64(): pa.scalar(0, type=pa.int64()),
+
+            pa.uint8(): pa.scalar(0, type=pa.uint8()),
+            pa.uint16(): pa.scalar(0, type=pa.uint16()),
+            pa.uint32(): pa.scalar(0, type=pa.uint32()),
+            pa.uint64(): pa.scalar(0, type=pa.uint64()),
+
+            # pa.float16(): pa.scalar(0.0, type=pa.float16()),
+            pa.float32(): pa.scalar(0.0, type=pa.float32()),
+            pa.float64(): pa.scalar(0.0, type=pa.float64()),
+
+            pa.string(): pa.scalar("", type=pa.string()),
+            pa.string_view(): pa.scalar("", type=pa.string_view()),
+            pa.large_string(): pa.scalar("", type=pa.large_string()),
+
+            pa.binary(): pa.scalar(b"", type=pa.binary()),
+            pa.binary_view(): pa.scalar(b"", type=pa.binary_view()),
+            pa.large_binary(): pa.scalar(b"", type=pa.large_binary()),
+        }
+
+        try:
+            import polars
+
+            _POLARS_DEFAULTS = {
+                polars.Null(): None,
+                polars.Boolean(): False,
+
+                polars.Binary(): b"",
+
+                polars.Utf8(): "",
+
+                polars.Int8(): 0,
+                polars.Int16(): 0,
+                polars.Int32(): 0,
+                polars.Int64(): 0,
+
+                polars.UInt8(): 0,
+                polars.UInt16(): 0,
+                polars.UInt32(): 0,
+                polars.UInt64(): 0,
+
+                polars.Float32(): 0.0,
+                polars.Float64(): 0.0,
+            }
+        except ImportError:
+            pass
+
+        DEFAULT_MAPS_INITIALIZED = True
 
 def _is_optional(hint) -> bool:
     """Return True when the type hint is Optional.
@@ -199,6 +211,8 @@ def default_arrow_scalar(
     Returns:
         Arrow scalar default.
     """
+    ensure_default_maps_initialized()
+
     if nullable:
         return pa.scalar(None, type=dtype)
 
@@ -307,6 +321,8 @@ def default_python_scalar(hint: Any):
     if _is_optional(hint):
         return None
 
+    ensure_default_maps_initialized()
+
     if hint in _PRIMITIVE_DEFAULTS:
         return _PRIMITIVE_DEFAULTS[hint]
 

yggdrasil/version.py

@@ -1 +1 @@
-__version__ = "0.1.60"
+__version__ = "0.1.65"