ygg 0.1.44__tar.gz → 0.1.45__tar.gz

{ygg-0.1.44 → ygg-0.1.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.44
+Version: 0.1.45
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.44 → ygg-0.1.45}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygg"
-version = "0.1.44"
+version = "0.1.45"
 description = "Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }

{ygg-0.1.44 → ygg-0.1.45}/src/ygg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.44
+Version: 0.1.45
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.44 → ygg-0.1.45}/src/ygg.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ src/yggdrasil/databricks/sql/engine.py
 src/yggdrasil/databricks/sql/exceptions.py
 src/yggdrasil/databricks/sql/statement_result.py
 src/yggdrasil/databricks/sql/types.py
+src/yggdrasil/databricks/sql/warehouse.py
 src/yggdrasil/databricks/workspaces/__init__.py
 src/yggdrasil/databricks/workspaces/filesytem.py
 src/yggdrasil/databricks/workspaces/io.py

{ygg-0.1.44 → ygg-0.1.45}/src/yggdrasil/databricks/compute/cluster.py

@@ -22,7 +22,7 @@ from typing import Any, Iterator, Optional, Union, List, Callable, Dict, ClassVa
 
 from .execution_context import ExecutionContext
 from ..workspaces.workspace import WorkspaceService, Workspace
-from ... import retry, CallableSerde
+from ... import CallableSerde
 from ...libs.databrickslib import databricks_sdk
 from ...pyutils.equality import dicts_equal, dict_diff
 from ...pyutils.expiring_dict import ExpiringDict
@@ -47,6 +47,7 @@ else:  # pragma: no cover - runtime fallback when SDK is missing
 __all__ = ["Cluster"]
 
 
+LOGGER = logging.getLogger(__name__)
 NAME_ID_CACHE: dict[str, ExpiringDict] = {}
 
 
@@ -72,9 +73,6 @@ def get_cached_cluster_id(
     return existing.get(cluster_name) if existing else None
 
 
-logger = logging.getLogger(__name__)
-
-
 # module-level mapping Databricks Runtime -> (major, minor) Python version
 _PYTHON_BY_DBR: dict[str, tuple[int, int]] = {
     "10.4": (3, 8),
@@ -363,7 +361,8 @@ class Cluster(WorkspaceService):
         tick: float = 0.5,
         timeout: Union[float, dt.timedelta] = 600,
         backoff: int = 2,
-        max_sleep_time: float = 15
+        max_sleep_time: float = 15,
+        wait_libraries: bool = True
     ):
         """Wait for the cluster to exit pending states.
 
@@ -372,6 +371,7 @@ class Cluster(WorkspaceService):
             timeout: Max seconds to wait before timing out.
             backoff: Backoff multiplier for the sleep interval.
             max_sleep_time: Maximum sleep interval in seconds.
+            wait_libraries: Wait libraries to install fully
 
         Returns:
             The current Cluster instance.
@@ -390,7 +390,8 @@ class Cluster(WorkspaceService):
 
             sleep_time = min(max_sleep_time, sleep_time * backoff)
 
-        self.wait_installed_libraries()
+        if wait_libraries:
+            self.wait_installed_libraries()
 
         self.raise_for_status()
 
@@ -638,7 +639,7 @@ class Cluster(WorkspaceService):
             if k in _CREATE_ARG_NAMES
         }
 
-        logger.debug(
+        LOGGER.debug(
             "Creating Databricks cluster %s with %s",
             update_details["cluster_name"],
             update_details,
@@ -646,7 +647,7 @@ class Cluster(WorkspaceService):
 
         self.details = self.clusters_client().create_and_wait(**update_details)
 
-        logger.info(
+        LOGGER.info(
             "Created %s",
             self
         )
@@ -699,7 +700,7 @@ class Cluster(WorkspaceService):
                 for k, v in dict_diff(existing_details, update_details, keys=_EDIT_ARG_NAMES).items()
             }
 
-            logger.debug(
+            LOGGER.debug(
                 "Updating %s with %s",
                 self, diff
             )
@@ -707,7 +708,7 @@ class Cluster(WorkspaceService):
             self.wait_for_status()
             self.clusters_client().edit(**update_details)
 
-            logger.info(
+            LOGGER.info(
                 "Updated %s",
                 self
             )
@@ -811,7 +812,7 @@ class Cluster(WorkspaceService):
         self.wait_for_status()
 
         if not self.is_running:
-            logger.info("Starting %s", self)
+            LOGGER.debug("Starting %s", self)
 
             if wait_timeout:
                 self.clusters_client().start(cluster_id=self.cluster_id)
@@ -819,6 +820,8 @@ class Cluster(WorkspaceService):
             else:
                 self.clusters_client().start(cluster_id=self.cluster_id)
 
+            LOGGER.info("Started %s", self)
+
         return self
 
     def restart(
@@ -832,7 +835,6 @@ class Cluster(WorkspaceService):
         self.wait_for_status()
 
         if self.is_running:
-            logger.info("Restarting %s", self)
             self.details = self.clusters_client().restart_and_wait(cluster_id=self.cluster_id)
             return self.wait_for_status()
 
@@ -846,8 +848,10 @@ class Cluster(WorkspaceService):
         Returns:
             The SDK delete response.
         """
-        logger.info("Deleting %s", self)
-        return self.clusters_client().delete(cluster_id=self.cluster_id)
+        if self.cluster_id:
+            LOGGER.debug("Deleting %s", self)
+            self.clusters_client().delete(cluster_id=self.cluster_id)
+            LOGGER.info("Deleted %s", self)
 
     def context(
         self,
@@ -1150,7 +1154,7 @@ class Cluster(WorkspaceService):
                 if raise_error:
                     raise DatabricksError("Libraries %s in %s failed to install" % (failed, self))
 
-                logger.warning(
+                LOGGER.exception(
                     "Libraries %s in %s failed to install",
                     failed, self
                 )
@@ -1187,7 +1191,7 @@ class Cluster(WorkspaceService):
         Returns:
             The uploaded library argument(s).
         """
-        return self.context().install_temporary_libraries(libraries=libraries)
+        return self.system_context.install_temporary_libraries(libraries=libraries)
 
     def _check_library(
         self,

{ygg-0.1.44 → ygg-0.1.45}/src/yggdrasil/databricks/compute/execution_context.py

@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Optional, Any, Callable, List, Dict, Union, It
 
 from ...libs.databrickslib import databricks_sdk
 from ...pyutils.exceptions import raise_parsed_traceback
+from ...pyutils.expiring_dict import ExpiringDict
 from ...pyutils.modules import resolve_local_lib_path
 from ...pyutils.callable_serde import CallableSerde
 
@@ -30,7 +31,7 @@ __all__ = [
     "ExecutionContext"
 ]
 
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 
 @dc.dataclass
@@ -38,7 +39,6 @@ class RemoteMetadata:
     """Metadata describing the remote cluster execution environment."""
     site_packages_path: Optional[str] = dc.field(default=None)
     os_env: Dict[str, str] = dc.field(default_factory=dict)
-    requirements: Optional[str] = dc.field(default=None)
     version_info: Tuple[int, int, int] = dc.field(default=(0, 0, 0))
 
     def os_env_diff(
@@ -80,6 +80,7 @@ class ExecutionContext:
 
     _was_connected: Optional[bool] = dc.field(default=None, repr=False)
     _remote_metadata: Optional[RemoteMetadata] = dc.field(default=None, repr=False)
+    _uploaded_package_roots: Optional[ExpiringDict] = dc.field(default_factory=ExpiringDict, repr=False)
 
     _lock: threading.RLock = dc.field(default_factory=threading.RLock, init=False, repr=False)
 
@@ -127,9 +128,7 @@ class ExecutionContext:
         with self._lock:
             # double-check after acquiring lock
             if self._remote_metadata is None:
-                cmd = r"""import glob
-import json
-import os
+                cmd = r"""import glob, json, os
 from yggdrasil.pyutils.python_env import PythonEnv
 
 current_env = PythonEnv.get_current()
@@ -144,7 +143,6 @@ os_env = meta["os_env"] = {}
 for k, v in os.environ.items():
     os_env[k] = v
     
-meta["requirements"] = current_env.requirements()
 meta["version_info"] = current_env.version_info
 
 print(json.dumps(meta))"""
@@ -191,7 +189,7 @@ print(json.dumps(meta))"""
         """
         self.cluster.ensure_running()
 
-        logger.debug(
+        LOGGER.debug(
             "Creating Databricks command execution context for %s",
             self.cluster
         )
@@ -217,7 +215,7 @@ print(json.dumps(meta))"""
             The connected ExecutionContext instance.
         """
         if self.context_id is not None:
-            logger.debug(
+            LOGGER.debug(
                 "Execution context already open for %s",
                 self
             )
@@ -235,7 +233,7 @@ print(json.dumps(meta))"""
             raise RuntimeError("Failed to create command execution context")
 
         self.context_id = context_id
-        logger.info(
+        LOGGER.info(
             "Opened execution context for %s",
             self
         )
@@ -247,13 +245,9 @@ print(json.dumps(meta))"""
         Returns:
             None.
         """
-        if self.context_id is None:
+        if not self.context_id:
             return
 
-        logger.debug(
-            "Closing execution context for %s",
-            self
-        )
         try:
             self._workspace_client().command_execution.destroy(
                 cluster_id=self.cluster.cluster_id,
@@ -349,7 +343,7 @@ print(json.dumps(meta))"""
 
         self.connect(language=Language.PYTHON)
 
-        logger.debug(
+        LOGGER.debug(
             "Executing callable %s with %s",
             getattr(func, "__name__", type(func)),
             self,
@@ -394,12 +388,18 @@ print(json.dumps(meta))"""
             module_name = module_name.split(".")[0]
 
             if module_name and "yggdrasil" not in module_name:
-                self.close()
+                LOGGER.debug(
+                    "Installing missing module %s from local environment",
+                    module_name,
+                )
 
-                self.cluster.install_libraries(
+                self.install_temporary_libraries(
                     libraries=[module_name],
-                    raise_error=True,
-                    restart=True
+                )
+
+                LOGGER.warning(
+                    "Installed missing module %s from local environment",
+                    module_name,
                 )
 
                 return self.execute_callable(
@@ -412,6 +412,7 @@ print(json.dumps(meta))"""
                     timeout=timeout,
                     command=command,
                 )
+
             raise remote_module_error
 
         return result
@@ -446,29 +447,7 @@ print(json.dumps(meta))"""
             timeout=timeout or dt.timedelta(minutes=20)
         )
 
-        try:
-            return self._decode_result(result, result_tag=result_tag, print_stdout=print_stdout)
-        except ModuleNotFoundError as remote_module_error:
-            _MOD_NOT_FOUND_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]")
-            module_name = _MOD_NOT_FOUND_RE.search(str(remote_module_error))
-            module_name = module_name.group(1) if module_name else None
-            module_name = module_name.split(".")[0]
-
-            if module_name and "yggdrasil" not in module_name:
-                self.close()
-                self.cluster.install_libraries(
-                    libraries=[module_name],
-                    raise_error=True,
-                    restart=True
-                )
-
-                return self.execute_command(
-                    command=command,
-                    timeout=timeout,
-                    result_tag=result_tag,
-                    print_stdout=print_stdout
-                )
-            raise remote_module_error
+        return self._decode_result(result, result_tag=result_tag, print_stdout=print_stdout)
 
     # ------------------------------------------------------------------
     # generic local → remote uploader, via remote python
@@ -589,16 +568,22 @@ with zipfile.ZipFile(buf, "r") as zf:
             ]
 
         resolved = resolve_local_lib_path(libraries)
+        str_resolved = str(resolved)
+        existing = self._uploaded_package_roots.get(str_resolved)
 
-        remote_site_packages_path = self.remote_metadata.site_packages_path
-        if resolved.is_dir():
-            # site-packages/<package_name>/
-            remote_target = posixpath.join(remote_site_packages_path, resolved.name)
-        else:
-            # site-packages/<module_file>
-            remote_target = posixpath.join(remote_site_packages_path, resolved.name)
+        if not existing:
+            remote_site_packages_path = self.remote_metadata.site_packages_path
+
+            if resolved.is_dir():
+                # site-packages/<package_name>/
+                remote_target = posixpath.join(remote_site_packages_path, resolved.name)
+            else:
+                # site-packages/<module_file>
+                remote_target = posixpath.join(remote_site_packages_path, resolved.name)
+
+            self.upload_local_path(resolved, remote_target)
 
-        self.upload_local_path(resolved, remote_target)
+            self._uploaded_package_roots[str_resolved] = remote_target
 
         return libraries
 

{ygg-0.1.44 → ygg-0.1.45}/src/yggdrasil/databricks/sql/engine.py

@@ -130,7 +130,10 @@ class SQLEngine(WorkspaceService):
         schema_name = schema_name or self.schema_name
         return catalog_name, schema_name, table_name
 
-    def _default_warehouse(self, cluster_size: str = "Small"):
+    def _default_warehouse(
+        self,
+        cluster_size: str = "Small"
+    ):
         """Pick a default SQL warehouse (best-effort) matching the desired size.
 
         Args:
@@ -951,7 +954,7 @@ FROM parquet.`{temp_volume_path}`"""
         """Convert an Arrow Field to a Databricks SQL column DDL fragment."""
         name = field.name
         nullable_str = " NOT NULL" if put_not_null and not field.nullable else ""
-        name_str = f"{name} " if put_name else ""
+        name_str = f"`{name}` " if put_name else ""
 
         comment_str = ""
         if put_comment and field.metadata and b"comment" in field.metadata: