ygg 0.1.21__tar.gz → 0.1.24__tar.gz

{ygg-0.1.21 → ygg-0.1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.21
+Version: 0.1.24
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.21 → ygg-0.1.24}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygg"
-version = "0.1.21"
+version = "0.1.24"
 description = "Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }

{ygg-0.1.21 → ygg-0.1.24}/src/ygg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.21
+Version: 0.1.24
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.21 → ygg-0.1.24}/src/yggdrasil/databricks/compute/cluster.py

@@ -390,6 +390,8 @@ class Cluster(WorkspaceService):
         python_version: Optional[Union[str, tuple[int, ...]]] = None,
         **kwargs
     ):
+        pip_settings = PipIndexSettings.default_settings()
+
         if kwargs:
             details = ClusterDetails(**{
                 **details.as_shallow_dict(),
@@ -421,6 +423,13 @@ class Cluster(WorkspaceService):
         if details.is_single_node is not None and details.kind is None:
             details.kind = Kind.CLASSIC_PREVIEW
 
+        if pip_settings.extra_index_urls:
+            if details.spark_env_vars is None:
+                details.spark_env_vars = {}
+            str_urls = " ".join(pip_settings.extra_index_urls)
+            details.spark_env_vars["UV_EXTRA_INDEX_URL"] = details.spark_env_vars.get("UV_INDEX", str_urls)
+            details.spark_env_vars["PIP_EXTRA_INDEX_URL"] = details.spark_env_vars.get("PIP_EXTRA_INDEX_URL", str_urls)
+
         return details
 
     def create_or_update(

{ygg-0.1.21 → ygg-0.1.24}/src/yggdrasil/databricks/compute/execution_context.py

@@ -120,7 +120,7 @@ class ExecutionContext:
                 cmd = r"""import glob
 import json
 import os
-from yggdrasil.pyutils import PythonEnv
+from yggdrasil.pyutils.python_env import PythonEnv
 
 current_env = PythonEnv.get_current()
 meta = {}

{ygg-0.1.21 → ygg-0.1.24}/src/yggdrasil/pyutils/python_env.py

@@ -17,7 +17,7 @@ import zipfile
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
-from typing import Any, Iterable, Iterator, Mapping, MutableMapping, Optional, Union
+from typing import Any, Iterable, Iterator, Mapping, MutableMapping, Optional, Union, List
 
 log = logging.getLogger(__name__)
 
@@ -40,6 +40,55 @@ _LOCKS_GUARD: threading.RLock = threading.RLock()
 
 # Installed into newly created envs (and on create() upsert when env already exists)
 DEFAULT_CREATE_PACKAGES: tuple[str, ...] = ("uv", "ygg")
+_NON_PIPABLE_NAMES = (
+    "python-apt",
+    "python3-apt",
+    "unattended-upgrades",
+    "apt",
+    "apt-utils",
+    "dpkg",
+    "adduser",
+    "lsb-release",
+    "software-properties-common",
+    "systemd",
+    "udev",
+    "dbus",
+)
+
+# Matches a requirement line that starts with one of those names, followed by:
+#  - end of line
+#  - a version specifier (==,>=, etc)
+#  - extras ([...])
+#  - an environment marker (; ...)
+_NON_PIPABLE_RE = re.compile(
+    r"^\s*(?:"
+    + "|".join(re.escape(n) for n in _NON_PIPABLE_NAMES)
+    + r")(?=\s*(?:$|==|~=|!=|<=|>=|<|>|\[|;))",
+    re.IGNORECASE,
+)
+
+
+
+def _filter_non_pipable_linux_packages(requirements: Iterable[str]) -> List[str]:
+    """
+    Remove Linux OS-level packages that aren't realistically installable via pip/uv on Databricks.
+    Keeps comments/blank lines as-is.
+    """
+    out: List[str] = []
+    for line in requirements:
+        s = line.strip()
+
+        # keep empty lines and comments
+        if not s or s.startswith("#"):
+            out.append(line)
+            continue
+
+        if _NON_PIPABLE_RE.match(s):
+            continue
+
+        out.append(line)
+
+    return out
 
 
 def _is_windows() -> bool:
@@ -464,7 +513,7 @@ class PythonEnv:
 
             env_obj = cls(root)
             if not env_obj.exists():
-                raise PythonEnvError(f"Created env but python missing: {env_obj.python}")
+                raise PythonEnvError(f"Created env but python missing: {env_obj.python_executable}")
 
             env_obj.update(
                 packages=install_pkgs,
@@ -560,17 +609,18 @@ class PythonEnv:
             )
 
             extra = list(pip_args or [])
-            base = [uv, "pip", "install", "--python", str(env_obj.python)]
+            base_uv = [uv, "pip", "install", "--python", str(env_obj.python_executable)]
 
             if upgrade_pip:
                 log.info("upgrading pip in env: %s", str(env_obj.root))
-                _run_cmd(base + ["-U", "pip"] + extra, cwd=cwd, env=env, check=check)
+                _run_cmd(base_uv + ["-U", "pip"] + extra, cwd=cwd, env=env, check=check)
 
             if packages:
                 pkgs = [packages] if isinstance(packages, str) else list(packages)
+
                 if pkgs:
                     log.info("installing packages into env %s: %s", str(env_obj.root), pkgs)
-                    _run_cmd(base + ["-U"] + pkgs + extra, cwd=cwd, env=env, check=check)
+                    _run_cmd(base_uv + ["-U"] + pkgs + extra, cwd=cwd, env=env, check=check)
 
             if requirements:
                 # requirements can be:
@@ -598,7 +648,7 @@ class PythonEnv:
                         raise PythonEnvError("requirements must be a path-like string/Path or raw requirements text")
 
                     log.info("installing requirements into env %s: %s", str(env_obj.root), str(req_path))
-                    _run_cmd(base + ["-U", "-r", str(req_path)] + extra, cwd=cwd, env=env, check=check)
+                    _run_cmd(base_uv + ["-U", "-r", str(req_path)] + extra, cwd=cwd, env=env, check=check)
 
                 finally:
                     if tmp_ctx is not None:
@@ -634,12 +684,12 @@ class PythonEnv:
         return n if n else str(self.root)
 
     @property
-    def python(self) -> Path:
+    def python_executable(self) -> Path:
         exe = "python.exe" if _is_windows() else "python"
         return self.bindir / exe
 
     def exists(self) -> bool:
-        return self.python.exists()
+        return self.python_executable.exists()
 
     @property
     def version(self) -> str:
@@ -714,7 +764,7 @@ class PythonEnv:
 
         frozen_text: Optional[str] = None
         if keep_packages and self.exists():
-            p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python)], check=True)
+            p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python_executable)], check=True)
             frozen_text = p.stdout or ""
 
         with _locked_env(root):
@@ -731,7 +781,7 @@ class PythonEnv:
 
             new_env = self.__class__(root)
             if not new_env.exists():
-                raise PythonEnvError(f"Recreated env but python missing: {new_env.python}")
+                raise PythonEnvError(f"Recreated env but python missing: {new_env.python_executable}")
 
             if keep_packages and frozen_text and frozen_text.strip():
                 import datetime as _dt
@@ -740,7 +790,7 @@ class PythonEnv:
                 req_path = parent / f".{root.name}.freeze-{ts}.txt"
                 req_path.write_text(frozen_text, encoding="utf-8")
                 try:
-                    _run_cmd([uv, "pip", "install", "--python", str(new_env.python), "-r", str(req_path)], check=True)
+                    _run_cmd([uv, "pip", "install", "--python", str(new_env.python_executable), "-r", str(req_path)], check=True)
                 finally:
                     try:
                         req_path.unlink(missing_ok=True)
@@ -787,12 +837,12 @@ class PythonEnv:
         """
         if not python_versions:
             return self.export_requirements_matrix(
-                python_versions=[self.python],
+                python_versions=[self.python_executable],
                 out_dir=out_dir, base_name=base_name, include_frozen=include_frozen,
                 include_input=include_input, check=check, buffers=buffers or {},
                 uv_upgrade=uv_upgrade, uv_user=uv_user, uv_index_url=uv_index_url,
                 uv_extra_pip_args=uv_extra_pip_args
-            )[str(self.python)]
+            )[str(self.python_executable)]
 
         def _slug(s: str) -> str:
             s = (s or "").strip()
@@ -806,7 +856,7 @@ class PythonEnv:
             raise PythonEnvError("python_versions cannot be empty")
 
         if not self.exists():
-            raise PythonEnvError(f"Python executable not found in env: {self.python}")
+            raise PythonEnvError(f"Python executable not found in env: {self.python_executable}")
 
         uv = self.__class__.ensure_uv(
             check=check,
@@ -878,7 +928,9 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                     if not isinstance(top_level, list) or not all(isinstance(x, str) for x in top_level):
                         raise PythonEnvError(f"Unexpected top-level requirements payload: {top_level!r}")
 
-                    req_in_text = "\n".join(top_level) + "\n"
+                    filtered = _filter_non_pipable_linux_packages(top_level)
+
+                    req_in_text = "\n".join(filtered) + "\n"
                     req_in_path.write_text(req_in_text, encoding="utf-8")
                     if buffers is not None:
                         buffers[f"{base_name}.in"] = req_in_text
@@ -887,7 +939,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                 frozen_path = out_root / f"{base_name}.frozen.txt"
                 if include_frozen:
                     log.info("exporting frozen requirements: %s", str(frozen_path))
-                    p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python)], check=check)
+                    p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python_executable)], check=check)
                     frozen_text = p.stdout or ""
                     if write_files:
                         frozen_path.write_text(frozen_text, encoding="utf-8")
@@ -944,8 +996,8 @@ print("RESULT:" + json.dumps(top_level))""".strip()
         # pick interpreter (default = env python)
         if python is None:
             if not self.exists():
-                raise PythonEnvError(f"Python executable not found in env: {self.python}")
-            py = self.python
+                raise PythonEnvError(f"Python executable not found in env: {self.python_executable}")
+            py = self.python_executable
         else:
             py = Path(python).expanduser().resolve() if isinstance(python, Path) else Path(str(python)).expanduser().resolve()
             if not py.exists():
@@ -1277,7 +1329,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                     require_python=require_python,
                     dedupe=True,
                 ):
-                    py = str(e.python) if e.exists() else "-"
+                    py = str(e.python_executable) if e.exists() else "-"
                     print(f"{e.name}\t{e.root}\t{py}")
                 return 0
 
@@ -1323,7 +1375,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
             if args.cmd == "current":
                 cur = cls.get_current()
                 print(f"root={cur.root}")
-                print(f"python={cur.python}")
+                print(f"python={cur.python_executable}")
                 return 0
 
             raise PythonEnvError(f"Unknown command: {args.cmd}")