ygg 0.1.30__tar.gz → 0.1.31__tar.gz

{ygg-0.1.30 → ygg-0.1.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.30
+Version: 0.1.31
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.30 → ygg-0.1.31}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygg"
-version = "0.1.30"
+version = "0.1.31"
 description = "Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }

{ygg-0.1.30 → ygg-0.1.31}/src/ygg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.30
+Version: 0.1.31
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.30 → ygg-0.1.31}/src/ygg.egg-info/SOURCES.txt

@@ -21,7 +21,10 @@ src/yggdrasil/databricks/sql/exceptions.py
 src/yggdrasil/databricks/sql/statement_result.py
 src/yggdrasil/databricks/sql/types.py
 src/yggdrasil/databricks/workspaces/__init__.py
-src/yggdrasil/databricks/workspaces/databricks_path.py
+src/yggdrasil/databricks/workspaces/filesytem.py
+src/yggdrasil/databricks/workspaces/io.py
+src/yggdrasil/databricks/workspaces/path.py
+src/yggdrasil/databricks/workspaces/path_kind.py
 src/yggdrasil/databricks/workspaces/workspace.py
 src/yggdrasil/dataclasses/__init__.py
 src/yggdrasil/dataclasses/dataclass.py

{ygg-0.1.30 → ygg-0.1.31}/src/yggdrasil/databricks/jobs/config.py

@@ -2,11 +2,11 @@ import builtins
 import dataclasses
 import datetime as dt
 import inspect
+import logging
 from dataclasses import dataclass, fields
 from enum import Enum
 from inspect import isclass
-from typing import Any, Dict, List, get_type_hints, Optional, get_origin
-import logging
+from typing import Any, Dict, List, get_type_hints, get_origin
 
 from ...libs.sparklib import SparkSession
 from ...types.cast.registry import convert
@@ -308,31 +308,3 @@ class NotebookConfig:
                 spark_session.conf.set("spark.sql.session.timeZone", "UTC")
 
         return cls.from_environment()
-
-
-class ExampleEnum(Enum):
-    """Example enum for widget demonstration"""
-    OPTION1 = "option1"
-    OPTION2 = "option2"
-    OPTION3 = "option3"
-
-
-@dataclass
-class CompleteNotebookConfig(NotebookConfig):
-    """Example JobConfig with various field types to demonstrate widget handling"""
-    # Basic types
-    text_field: str
-    integer_field: int = 42
-    float_field: float = 3.14
-    boolean_field: bool = True
-
-    # Special types
-    date_field: dt.date = dt.date(2023, 1, 1)
-    datetime_field: dt.datetime = dt.datetime(2023, 1, 1, 12, 0, 0)
-    enum_field: ExampleEnum = ExampleEnum.OPTION1
-
-    # Collection types
-    list_of_strings: List[str] = None  # Will be displayed as multiselect
-
-    # Optional fields
-    optional_text: Optional[str] = None

{ygg-0.1.30 → ygg-0.1.31}/src/yggdrasil/databricks/sql/statement_result.py

@@ -96,7 +96,7 @@ class StatementResult:
         if self.is_spark_sql:
             return self._response
 
-        if not self.done and self.statement_id and time.time() - self._response_refresh_time > delay:
+        if self.statement_id and not self.done and time.time() - self._response_refresh_time > delay:
             self.response = self.workspace.sdk().statement_execution.get_statement(self.statement_id)
 
         return self._response

{ygg-0.1.30 → ygg-0.1.31}/src/yggdrasil/databricks/sql/types.py

@@ -102,6 +102,18 @@ def _split_top_level_commas(s: str):
     return parts
 
 
+def _safe_bytes(obj):
+    if not isinstance(obj, bytes):
+        if not obj:
+            return b""
+
+        if not isinstance(obj, str):
+            obj = str(obj)
+
+        return obj.encode("utf-8")
+    return obj
+
+
 def parse_sql_type_to_pa(type_str: str) -> pa.DataType:
     """
     Adapted parser that:
@@ -170,6 +182,10 @@ def column_info_to_arrow_field(col: Union[SQLColumnInfo, CatalogColumnInfo]):
     if isinstance(col, CatalogColumnInfo):
         parsed = json.loads(col.type_json)
         md = parsed.get("metadata", {}) or {}
+        md = {
+            _safe_bytes(k): _safe_bytes(v)
+            for k, v in md.items()
+        }
         nullable = col.nullable
     elif isinstance(col, SQLColumnInfo):
         md = {}

ygg-0.1.31/src/yggdrasil/databricks/workspaces/__init__.py

@@ -0,0 +1,4 @@
+from .workspace import *
+from .path import *
+from .io import *
+

ygg-0.1.31/src/yggdrasil/databricks/workspaces/filesytem.py

@@ -0,0 +1,161 @@
+__all__ = [
+    "DatabricksFileSystem",
+    "DatabricksFileSystemHandler"
+]
+
+from typing import TYPE_CHECKING, Any, Union, List, Optional
+
+from pyarrow import PythonFile
+from pyarrow.fs import FileSystem, FileInfo, FileSelector, PyFileSystem, FileSystemHandler
+
+if TYPE_CHECKING:
+    from ..workspaces.workspace import Workspace
+    from .path import DatabricksPath
+
+
+class DatabricksFileSystemHandler(FileSystemHandler):
+
+    def __init__(
+        self,
+        workspace: "Workspace",
+    ):
+        super().__init__()
+        self.workspace = workspace
+
+    def __enter__(self):
+        return self.connect(clone=True)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.workspace.__exit__(exc_type, exc_val, exc_tb)
+
+    def _parse_path(self, obj: Any) -> "DatabricksPath":
+        from .path import DatabricksPath
+
+        return DatabricksPath.parse(obj, workspace=self.workspace)
+
+    def connect(self, clone: bool = True):
+        workspace = self.connect(clone=clone)
+
+        if clone:
+            return DatabricksFileSystemHandler(
+                workspace=workspace
+            )
+
+        self.workspace = workspace
+        return self
+
+    def close(self):
+        self.workspace.close()
+
+    def copy_file(self, src, dest, *, chunk_size: int = 4 * 1024 * 1024):
+        src = self._parse_path(src)
+        dest = self._parse_path(dest)
+
+        with src.open("rb") as r, dest.open("wb") as w:
+            while True:
+                chunk = r.read(chunk_size)
+                if not chunk:
+                    break
+                w.write(chunk)
+
+    def create_dir(self, path, *args, recursive: bool = True, **kwargs):
+        return self._parse_path(path).mkdir(parents=recursive)
+
+    def delete_dir(self, path):
+        return self._parse_path(path).rmdir(recursive=True)
+
+    def delete_dir_contents(self, path, *args, accept_root_dir: bool = False, **kwargs):
+        return self._parse_path(path).rmdir(recursive=True)
+
+    def delete_root_dir_contents(self):
+        return self.delete_dir_contents("/", accept_root_dir=True)
+
+    def delete_file(self, path):
+        return self._parse_path(path).rmfile()
+
+    def equals(self, other: FileSystem):
+        return self == other
+
+    def from_uri(self, uri):
+        uri = self._parse_path(uri)
+
+        return self.__class__(
+            workspace=uri.workspace
+        )
+
+    def get_file_info(
+        self,
+        paths_or_selector: Union[FileSelector, str, "DatabricksPath", List[Union[str, "DatabricksPath"]]]
+    ) -> Union[FileInfo, List[FileInfo]]:
+        from .path import DatabricksPath
+
+        if isinstance(paths_or_selector, (str, DatabricksPath)):
+            result = self._parse_path(paths_or_selector).file_info
+
+            return result
+
+        if isinstance(paths_or_selector, FileSelector):
+            return self.get_file_info_selector(paths_or_selector)
+
+        return [
+            self.get_file_info(obj)
+            for obj in paths_or_selector
+        ]
+
+    def get_file_info_selector(
+        self,
+        selector: FileSelector
+    ):
+        base_dir = self._parse_path(selector.base_dir)
+
+        return [
+            p.file_info
+            for p in base_dir.ls(
+                recursive=selector.recursive,
+                allow_not_found=selector.allow_not_found
+            )
+        ]
+
+    def get_type_name(self):
+        return "dbfs"
+
+    def move(self, src, dest):
+        src = self._parse_path(src)
+
+        src.copy_to(dest)
+
+        src.remove(recursive=True)
+
+    def normalize_path(self, path):
+        return self._parse_path(path).full_path()
+
+    def open(
+        self,
+        path,
+        mode: str = "r+",
+        encoding: Optional[str] = None,
+    ):
+        return self._parse_path(path).open(mode=mode, encoding=encoding, clone=False)
+
+    def open_append_stream(self, path, compression='detect', buffer_size=None, metadata=None):
+        return self._parse_path(path).open(mode="ab")
+
+    def open_input_file(self, path, mode: str = "rb", **kwargs):
+        buf = self._parse_path(path).open(mode=mode).connect(clone=True)
+
+        return PythonFile(
+            buf,
+            mode=mode
+        )
+
+    def open_input_stream(self, path, compression='detect', buffer_size=None):
+        return self._parse_path(path).open(mode="rb")
+
+    def open_output_stream(self, path, compression='detect', buffer_size=None, metadata=None):
+        return self._parse_path(path).open(mode="wb")
+
+
+class DatabricksFileSystem(PyFileSystem):
+
+    def __init__(self, handler): # real signature unknown; restored from __doc__
+        super().__init__(handler)