ygg 0.1.21__py3-none-any.whl → 0.1.23__py3-none-any.whl

{ygg-0.1.21.dist-info → ygg-0.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ygg
-Version: 0.1.21
+Version: 0.1.23
 Summary: Type-friendly utilities for moving data between Python objects, Arrow, Polars, Pandas, Spark, and Databricks
 Author: Yggdrasil contributors
 License:                                  Apache License

{ygg-0.1.21.dist-info → ygg-0.1.23.dist-info}/RECORD

@@ -1,9 +1,9 @@
-ygg-0.1.21.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+ygg-0.1.23.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
 yggdrasil/__init__.py,sha256=6OPibApplA5TF4TeixkQO_qewpaAidYX-fSDvvKYcTI,91
 yggdrasil/databricks/__init__.py,sha256=aGVve5mpoQtxSK2nfzrexjRPoutCIyaOnKZijkG4_QE,92
 yggdrasil/databricks/compute/__init__.py,sha256=TVDwPmW2SOmHmnhzZhsvrWbrxZ_lEcgqe3l9BeB-oxM,218
-yggdrasil/databricks/compute/cluster.py,sha256=4Pk_s98g1-YL48SGq45CAGj9tQd17qdy__oZEb9d-yw,28067
-yggdrasil/databricks/compute/execution_context.py,sha256=_5EaGsfwgq3Dzn3zZdngyNmDS1UXx9AZe47SFDXmukw,18573
+yggdrasil/databricks/compute/cluster.py,sha256=91nBe_Ni0Yb4ECUU-YhSQeMVqy9VKhpWDAp771_1mxQ,28561
+yggdrasil/databricks/compute/execution_context.py,sha256=plutmNa03VSHlXbGeKArtc-QH9ptZrrnss2yXS2pFYA,18584
 yggdrasil/databricks/compute/remote.py,sha256=sVWBb_1YR-e33on6F5QYMLKwT6end6rolCvN8HM-9Qw,1212
 yggdrasil/databricks/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 yggdrasil/databricks/jobs/config.py,sha256=8Slfw4Wl7vu0kIlaUUqVqjjOgPwuULoo0rroENCbC20,11494
@@ -30,7 +30,7 @@ yggdrasil/pyutils/callable_serde.py,sha256=6Pw9XO2JMUSmzGp1Vj3-AN1Nhave09fNWyn1x
 yggdrasil/pyutils/exceptions.py,sha256=Kt4xY4jPv-Ld8vBJaiQy51xbewD6Q-3bZkaa3THsFNM,2891
 yggdrasil/pyutils/modules.py,sha256=r3C4TRA8C_fUTltITV0VJY0pbIdxdV1op2z8BX0FgJo,10201
 yggdrasil/pyutils/parallel.py,sha256=L2r4_iumOzDk7omWVBPYmT5U8n7suOdv1AFpa_ghHSY,5902
-yggdrasil/pyutils/python_env.py,sha256=KqFnlKcpPfskLWzSiOkEjo71KQrLmg80aRfX_wo5_QA,47892
+yggdrasil/pyutils/python_env.py,sha256=pjon-G9N-lXx64905_hLaWPa3ZAoy0AhQX3Kg-ucym8,48417
 yggdrasil/pyutils/retry.py,sha256=1zjascEsffvnkVRdHhdeoenk6tBPrzrj4VhBltbhBeU,10390
 yggdrasil/requests/__init__.py,sha256=THJz1IoZYQccwmXcQR3N8D-uWxCkfMtgeXDhONdERR8,41
 yggdrasil/requests/msal.py,sha256=ucnN45iZZpbXkByw212PX4shH4g0EeyrW8JEmfimWtY,5861
@@ -49,8 +49,8 @@ yggdrasil/types/cast/registry.py,sha256=-88mq-U1pDSGbEC9PRY0zJCzloyBodXgeSRBPb6h
 yggdrasil/types/cast/spark_cast.py,sha256=IHthM78dugabGXxNNW9sSHn-olDwzXcFdIFcPo9IiXU,23021
 yggdrasil/types/cast/spark_pandas_cast.py,sha256=8PgJItF_XbyBcNuBnXkMQU3PBy3sAPEXZT9SXL2WbU4,4200
 yggdrasil/types/cast/spark_polars_cast.py,sha256=ba1UOvY1ouGCro1Np9slXmJ4TEyWnUtwVEAwxGvPLlk,8336
-ygg-0.1.21.dist-info/METADATA,sha256=CiwzGJO6wNPbre6ZTW21BtM3_HxPXFJF6pkzyL_84fY,19204
-ygg-0.1.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ygg-0.1.21.dist-info/entry_points.txt,sha256=D0JDw2s-ZfU1GjXfniv6PvXwcyn5v9WBk4ya623Cti8,71
-ygg-0.1.21.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
-ygg-0.1.21.dist-info/RECORD,,
+ygg-0.1.23.dist-info/METADATA,sha256=KMkpdOJCjfa2pTkNNpIVPDHfIVGIhKHFu5T73Ken2XY,19204
+ygg-0.1.23.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ygg-0.1.23.dist-info/entry_points.txt,sha256=D0JDw2s-ZfU1GjXfniv6PvXwcyn5v9WBk4ya623Cti8,71
+ygg-0.1.23.dist-info/top_level.txt,sha256=iBe9Kk4VIVbLpgv_p8OZUIfxgj4dgJ5wBg6vO3rigso,10
+ygg-0.1.23.dist-info/RECORD,,

yggdrasil/databricks/compute/cluster.py

@@ -390,6 +390,8 @@ class Cluster(WorkspaceService):
         python_version: Optional[Union[str, tuple[int, ...]]] = None,
         **kwargs
     ):
+        pip_settings = PipIndexSettings.default_settings()
+
         if kwargs:
             details = ClusterDetails(**{
                 **details.as_shallow_dict(),
@@ -421,6 +423,13 @@ class Cluster(WorkspaceService):
         if details.is_single_node is not None and details.kind is None:
             details.kind = Kind.CLASSIC_PREVIEW
 
+        if pip_settings.extra_index_urls:
+            if details.spark_env_vars is None:
+                details.spark_env_vars = {}
+            str_urls = " ".join(pip_settings.extra_index_urls)
+            details.spark_env_vars["UV_EXTRA_INDEX_URL"] = details.spark_env_vars.get("UV_INDEX", str_urls)
+            details.spark_env_vars["PIP_EXTRA_INDEX_URL"] = details.spark_env_vars.get("PIP_EXTRA_INDEX_URL", str_urls)
+
         return details
 
     def create_or_update(

yggdrasil/databricks/compute/execution_context.py

@@ -120,7 +120,7 @@ class ExecutionContext:
                 cmd = r"""import glob
 import json
 import os
-from yggdrasil.pyutils import PythonEnv
+from yggdrasil.pyutils.python_env import PythonEnv
 
 current_env = PythonEnv.get_current()
 meta = {}

yggdrasil/pyutils/python_env.py

@@ -40,6 +40,9 @@ _LOCKS_GUARD: threading.RLock = threading.RLock()
 
 # Installed into newly created envs (and on create() upsert when env already exists)
 DEFAULT_CREATE_PACKAGES: tuple[str, ...] = ("uv", "ygg")
+EXCLUDED_PACKAGES = {
+    "python-apt"
+}
 
 
 def _is_windows() -> bool:
@@ -464,7 +467,7 @@ class PythonEnv:
 
             env_obj = cls(root)
             if not env_obj.exists():
-                raise PythonEnvError(f"Created env but python missing: {env_obj.python}")
+                raise PythonEnvError(f"Created env but python missing: {env_obj.python_executable}")
 
             env_obj.update(
                 packages=install_pkgs,
@@ -560,17 +563,18 @@ class PythonEnv:
             )
 
             extra = list(pip_args or [])
-            base = [uv, "pip", "install", "--python", str(env_obj.python)]
+            base_uv = [uv, "pip", "install", "--python", str(env_obj.python_executable)]
 
             if upgrade_pip:
                 log.info("upgrading pip in env: %s", str(env_obj.root))
-                _run_cmd(base + ["-U", "pip"] + extra, cwd=cwd, env=env, check=check)
+                _run_cmd(base_uv + ["-U", "pip"] + extra, cwd=cwd, env=env, check=check)
 
             if packages:
                 pkgs = [packages] if isinstance(packages, str) else list(packages)
+
                 if pkgs:
                     log.info("installing packages into env %s: %s", str(env_obj.root), pkgs)
-                    _run_cmd(base + ["-U"] + pkgs + extra, cwd=cwd, env=env, check=check)
+                    _run_cmd(base_uv + ["-U"] + pkgs + extra, cwd=cwd, env=env, check=check)
 
             if requirements:
                 # requirements can be:
@@ -598,7 +602,7 @@ class PythonEnv:
                         raise PythonEnvError("requirements must be a path-like string/Path or raw requirements text")
 
                     log.info("installing requirements into env %s: %s", str(env_obj.root), str(req_path))
-                    _run_cmd(base + ["-U", "-r", str(req_path)] + extra, cwd=cwd, env=env, check=check)
+                    _run_cmd(base_uv + ["-U", "-r", str(req_path)] + extra, cwd=cwd, env=env, check=check)
 
                 finally:
                     if tmp_ctx is not None:
@@ -634,12 +638,12 @@ class PythonEnv:
         return n if n else str(self.root)
 
     @property
-    def python(self) -> Path:
+    def python_executable(self) -> Path:
         exe = "python.exe" if _is_windows() else "python"
         return self.bindir / exe
 
     def exists(self) -> bool:
-        return self.python.exists()
+        return self.python_executable.exists()
 
     @property
     def version(self) -> str:
@@ -714,7 +718,7 @@ class PythonEnv:
 
         frozen_text: Optional[str] = None
         if keep_packages and self.exists():
-            p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python)], check=True)
+            p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python_executable)], check=True)
             frozen_text = p.stdout or ""
 
         with _locked_env(root):
@@ -731,7 +735,7 @@ class PythonEnv:
 
             new_env = self.__class__(root)
             if not new_env.exists():
-                raise PythonEnvError(f"Recreated env but python missing: {new_env.python}")
+                raise PythonEnvError(f"Recreated env but python missing: {new_env.python_executable}")
 
             if keep_packages and frozen_text and frozen_text.strip():
                 import datetime as _dt
@@ -740,7 +744,7 @@ class PythonEnv:
                 req_path = parent / f".{root.name}.freeze-{ts}.txt"
                 req_path.write_text(frozen_text, encoding="utf-8")
                 try:
-                    _run_cmd([uv, "pip", "install", "--python", str(new_env.python), "-r", str(req_path)], check=True)
+                    _run_cmd([uv, "pip", "install", "--python", str(new_env.python_executable), "-r", str(req_path)], check=True)
                 finally:
                     try:
                         req_path.unlink(missing_ok=True)
@@ -787,12 +791,12 @@ class PythonEnv:
         """
         if not python_versions:
             return self.export_requirements_matrix(
-                python_versions=[self.python],
+                python_versions=[self.python_executable],
                 out_dir=out_dir, base_name=base_name, include_frozen=include_frozen,
                 include_input=include_input, check=check, buffers=buffers or {},
                 uv_upgrade=uv_upgrade, uv_user=uv_user, uv_index_url=uv_index_url,
                 uv_extra_pip_args=uv_extra_pip_args
-            )[str(self.python)]
+            )[str(self.python_executable)]
 
         def _slug(s: str) -> str:
             s = (s or "").strip()
@@ -806,7 +810,7 @@ class PythonEnv:
             raise PythonEnvError("python_versions cannot be empty")
 
         if not self.exists():
-            raise PythonEnvError(f"Python executable not found in env: {self.python}")
+            raise PythonEnvError(f"Python executable not found in env: {self.python_executable}")
 
         uv = self.__class__.ensure_uv(
             check=check,
@@ -878,7 +882,12 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                     if not isinstance(top_level, list) or not all(isinstance(x, str) for x in top_level):
                         raise PythonEnvError(f"Unexpected top-level requirements payload: {top_level!r}")
 
-                    req_in_text = "\n".join(top_level) + "\n"
+                    # exclude python-apt (yanked/unmaintained; not usable on Databricks)
+                    _pyapt_re = re.compile(r"^\s*python-apt(\s*(?:==|~=|!=|<=|>=|<|>|\[)|\s*(?:;|$))", re.IGNORECASE)
+
+                    filtered = [line for line in top_level if not _pyapt_re.match(line)]
+
+                    req_in_text = "\n".join(filtered) + "\n"
                     req_in_path.write_text(req_in_text, encoding="utf-8")
                     if buffers is not None:
                         buffers[f"{base_name}.in"] = req_in_text
@@ -887,7 +896,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                 frozen_path = out_root / f"{base_name}.frozen.txt"
                 if include_frozen:
                     log.info("exporting frozen requirements: %s", str(frozen_path))
-                    p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python)], check=check)
+                    p = _run_cmd([uv, "pip", "freeze", "--python", str(self.python_executable)], check=check)
                     frozen_text = p.stdout or ""
                     if write_files:
                         frozen_path.write_text(frozen_text, encoding="utf-8")
@@ -944,8 +953,8 @@ print("RESULT:" + json.dumps(top_level))""".strip()
         # pick interpreter (default = env python)
         if python is None:
             if not self.exists():
-                raise PythonEnvError(f"Python executable not found in env: {self.python}")
-            py = self.python
+                raise PythonEnvError(f"Python executable not found in env: {self.python_executable}")
+            py = self.python_executable
         else:
             py = Path(python).expanduser().resolve() if isinstance(python, Path) else Path(str(python)).expanduser().resolve()
             if not py.exists():
@@ -1277,7 +1286,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
                     require_python=require_python,
                     dedupe=True,
                 ):
-                    py = str(e.python) if e.exists() else "-"
+                    py = str(e.python_executable) if e.exists() else "-"
                     print(f"{e.name}\t{e.root}\t{py}")
                 return 0
 
@@ -1323,7 +1332,7 @@ print("RESULT:" + json.dumps(top_level))""".strip()
             if args.cmd == "current":
                 cur = cls.get_current()
                 print(f"root={cur.root}")
-                print(f"python={cur.python}")
+                print(f"python={cur.python_executable}")
                 return 0
 
             raise PythonEnvError(f"Unknown command: {args.cmd}")