ygg 0.1.21__tar.gz → 0.1.23__tar.gz

{ygg-0.1.21 → ygg-0.1.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.21
+Version: 0.1.23
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.21 → ygg-0.1.23}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ygg"
-version = "0.1.21"
+version = "0.1.23"
 description = "Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { file = "LICENSE" }

{ygg-0.1.21 → ygg-0.1.23}/src/ygg.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.21
+Version: 0.1.23
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.21 → ygg-0.1.23}/src/yggdrasil/databricks/compute/cluster.py

@@ -390,6 +390,8 @@ class Cluster(WorkspaceService):
         python_version: Optional[Union[str, tuple[int, ...]]] = None,
         **kwargs
     ):
+        pip_settings = PipIndexSettings.default_settings()
+
         if kwargs:
             details = ClusterDetails(**{
                 **details.as_shallow_dict(),
@@ -421,6 +423,13 @@ class Cluster(WorkspaceService):
         if details.is_single_node is not None and details.kind is None:
             details.kind = Kind.CLASSIC_PREVIEW
 
+        if pip_settings.extra_index_urls:
+            if details.spark_env_vars is None:
+                details.spark_env_vars = {}
+            str_urls = " ".join(pip_settings.extra_index_urls)
+            details.spark_env_vars["UV_EXTRA_INDEX_URL"] = details.spark_env_vars.get("UV_INDEX", str_urls)
+            details.spark_env_vars["PIP_EXTRA_INDEX_URL"] = details.spark_env_vars.get("PIP_EXTRA_INDEX_URL", str_urls)
+
         return details
 
     def create_or_update(

{ygg-0.1.21 → ygg-0.1.23}/src/yggdrasil/databricks/compute/execution_context.py

@@ -120,7 +120,7 @@ class ExecutionContext:
                 cmd = r"""import glob
 import json
 import os
-from yggdrasil.pyutils import PythonEnv
+from yggdrasil.pyutils.python_env import PythonEnv
 
 current_env = PythonEnv.get_current()
 meta = {}

{ygg-0.1.21 → ygg-0.1.23}/src/yggdrasil/pyutils/python_env.py

@@ -40,6 +40,9 @@ _LOCKS_GUARD: threading.RLock = threading.RLock()
 
 # Installed into newly created envs (and on create() upsert when env already exists)
 DEFAULT_CREATE_PACKAGES: tuple[str, ...] = ("uv", "ygg")
+EXCLUDED_PACKAGES = {
+    "python-apt"
+}
 
 
 def _is_windows() -> bool:
@@ -464,7 +467,7 @@ class PythonEnv:
 
             env_obj = cls(root)
             if not env_obj.exists():
-                raise PythonEnvError(f"Created env but python missing: {env_obj.python}")
+                raise PythonEnvError(f"Created env but python missing: {env_obj.python_executable}")
 
             env_obj.update(
                 packages=install_pkgs,
@@ -560,17 +563,18 @@ class PythonEnv:
             )
 
             extra = list(pip_args or [])
-            base = [uv, "pip", "install", "--python", str(env_obj.python)]
+            base_uv = [uv, "pip", "install", "--python", str(env_obj.python_executable)]
 
             if upgrade_pip:
                 log.info("upgrading pip in env: %s", str(env_obj.root))
-                _run_cmd(base + ["-U", "pip"] + extra, cwd=cwd, env=env, check=check)
+                _run_cmd(base_uv + ["-U", "pip"] + extra, cwd=cwd, env=env, check=check)
 
             if packages:
                 pkgs = [packages] if isinstance(packages, str) else list(packages)
+
                 if pkgs:
                     log.info("installing packages into env %s: %s", str(env_obj.root), pkgs)
-                    _run_cmd(base + ["-U"] + pkgs + extra, cwd=cwd, env=env, check=check)
+                    _run_cmd(base_uv + ["-U"] + pkgs + extra, cwd=cwd, env=env, check=check)
 
             if requirements:
                 # requirements can be:
@@ -598,7 +602,7 @@ class PythonEnv:
                         raise PythonEnvError("requirements must be a path-like string/Path or raw requirements text")
 
                     log.info("installing requirements into env %s: %s", str(env_obj.root), str(req_path))
-                    _run_cmd(base + ["-U", "-r", str(req_path)] + extra, cwd=cwd, env=env, check=check)
+                    _run_cmd(base_uv + ["-U", "-r", str(req_path)] + extra, cwd=cwd, env=env, check=check)
 
                 finally:
                     if tmp_ctx is not None:
@@ -634,12 +638,12 @@ class PythonEnv:
         return n if n else str(self.root)
 
     @property
-    def python(self) -> Path:
+    def python_executable(self) -> Path:
         exe = "python.exe" if _is_windows() else "python"
         return self.bindir / exe
 
     def exists(self) -> bool:
-        return self.python.exists()
+        return self.python_executable.exists()
 
     @property
     def version(self) -> str:
@@ -714,7 +718,7 @@ class PythonEnv:
 
         frozen_text: Optional[str] = None
         if keep_packages and self.exists():
-            p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python)], check=True)
+            p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python_executable)], check=True)
             frozen_text = p.stdout or ""
 
         with _locked_env(root):
@@ -731,7 +735,7 @@ class PythonEnv:
 
             new_env = self.__class__(root)
             if not new_env.exists():
-                raise PythonEnvError(f"Recreated env but python missing: {new_env.python}")
+                raise PythonEnvError(f"Recreated env but python missing: {new_env.python_executable}")
 
             if keep_packages and frozen_text and frozen_text.strip():
                 import datetime as _dt
@@ -740,7 +744,7 @@ class PythonEnv:
                 req_path = parent / f".{root.name}.freeze-{ts}.txt"
                 req_path.write_text(frozen_text, encoding="utf-8")
                 try:
-                    _run_cmd([uv, "pip", "install", "--python", str(new_env.python), "-r", str(req_path)], check=True)
+                    _run_cmd([uv, "pip", "install", "--python", str(new_env.python_executable), "-r", str(req_path)], check=True)
                 finally:
                     try:
                         req_path.unlink(missing_ok=True)
@@ -787,12 +791,12 @@ class PythonEnv:
         """
         if not python_versions:
             return self.export_requirements_matrix(
-                python_versions=[self.python],
+                python_versions=[self.python_executable],
                 out_dir=out_dir, base_name=base_name, include_frozen=include_frozen,
                 include_input=include_input, check=check, buffers=buffers or {},
                 uv_upgrade=uv_upgrade, uv_user=uv_user, uv_index_url=uv_index_url,
                 uv_extra_pip_args=uv_extra_pip_args
-            )[str(self.python)]
+            )[str(self.python_executable)]
 
         def _slug(s: str) -> str:
             s = (s or "").strip()
@@ -806,7 +810,7 @@ class PythonEnv:
             raise PythonEnvError("python_versions cannot be empty")
 
         if not self.exists():
-            raise PythonEnvError(f"Python executable not found in env: {self.python}")
+            raise PythonEnvError(f"Python executable not found in env: {self.python_executable}")
 
         uv = self.__class__.ensure_uv(
             check=check,
@@ -878,7 +882,12 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                     if not isinstance(top_level, list) or not all(isinstance(x, str) for x in top_level):
                         raise PythonEnvError(f"Unexpected top-level requirements payload: {top_level!r}")
 
-                    req_in_text = "\n".join(top_level) + "\n"
+                    # exclude python-apt (yanked/unmaintained; not usable on Databricks)
+                    _pyapt_re = re.compile(r"^\s*python-apt(\s*(?:==|~=|!=|<=|>=|<|>|\[)|\s*(?:;|$))", re.IGNORECASE)
+
+                    filtered = [line for line in top_level if not _pyapt_re.match(line)]
+
+                    req_in_text = "\n".join(filtered) + "\n"
                     req_in_path.write_text(req_in_text, encoding="utf-8")
                     if buffers is not None:
                         buffers[f"{base_name}.in"] = req_in_text
@@ -887,7 +896,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                 frozen_path = out_root / f"{base_name}.frozen.txt"
                 if include_frozen:
                     log.info("exporting frozen requirements: %s", str(frozen_path))
-                    p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python)], check=check)
+                    p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python_executable)], check=check)
                     frozen_text = p.stdout or ""
                     if write_files:
                         frozen_path.write_text(frozen_text, encoding="utf-8")
@@ -944,8 +953,8 @@ print("RESULT:" + json.dumps(top_level))""".strip()
         # pick interpreter (default = env python)
         if python is None:
             if not self.exists():
-                raise PythonEnvError(f"Python executable not found in env: {self.python}")
-            py = self.python
+                raise PythonEnvError(f"Python executable not found in env: {self.python_executable}")
+            py = self.python_executable
         else:
             py = Path(python).expanduser().resolve() if isinstance(python, Path) else Path(str(python)).expanduser().resolve()
             if not py.exists():
@@ -1277,7 +1286,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                     require_python=require_python,
                     dedupe=True,
                 ):
-                    py = str(e.python) if e.exists() else "-"
+                    py = str(e.python_executable) if e.exists() else "-"
                     print(f"{e.name}\t{e.root}\t{py}")
                 return 0
 
@@ -1323,7 +1332,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
             if args.cmd == "current":
                 cur = cls.get_current()
                 print(f"root={cur.root}")
-                print(f"python={cur.python}")
+                print(f"python={cur.python_executable}")
                 return 0
 
             raise PythonEnvError(f"Unknown command: {args.cmd}")