jupyter-storage-explorer-databricks-support 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jupyter_storage_explorer_databricks_support-0.18.0/.gitignore +76 -0
- jupyter_storage_explorer_databricks_support-0.18.0/LICENSE +21 -0
- jupyter_storage_explorer_databricks_support-0.18.0/PKG-INFO +102 -0
- jupyter_storage_explorer_databricks_support-0.18.0/README.md +79 -0
- jupyter_storage_explorer_databricks_support-0.18.0/hatch_build.py +24 -0
- jupyter_storage_explorer_databricks_support-0.18.0/pyproject.toml +67 -0
- jupyter_storage_explorer_databricks_support-0.18.0/src/jupyter_storage_explorer_databricks_support/__init__.py +30 -0
- jupyter_storage_explorer_databricks_support-0.18.0/src/jupyter_storage_explorer_databricks_support/mapper.py +102 -0
- jupyter_storage_explorer_databricks_support-0.18.0/tests/conftest.py +3 -0
- jupyter_storage_explorer_databricks_support-0.18.0/tests/test_databricks_mapper.py +137 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# --- Python ---
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
|
|
7
|
+
# Packaging
|
|
8
|
+
build/
|
|
9
|
+
dist/
|
|
10
|
+
*.egg-info/
|
|
11
|
+
*.egg
|
|
12
|
+
.eggs/
|
|
13
|
+
wheels/
|
|
14
|
+
sdist/
|
|
15
|
+
pip-wheel-metadata/
|
|
16
|
+
MANIFEST
|
|
17
|
+
|
|
18
|
+
# Virtual environments
|
|
19
|
+
.venv/
|
|
20
|
+
venv/
|
|
21
|
+
env/
|
|
22
|
+
ENV/
|
|
23
|
+
|
|
24
|
+
# uv
|
|
25
|
+
uv.lock
|
|
26
|
+
|
|
27
|
+
# Tests, coverage and type checkers
|
|
28
|
+
.pytest_cache/
|
|
29
|
+
.tox/
|
|
30
|
+
.nox/
|
|
31
|
+
.coverage
|
|
32
|
+
.coverage.*
|
|
33
|
+
htmlcov/
|
|
34
|
+
coverage.xml
|
|
35
|
+
.hypothesis/
|
|
36
|
+
.mypy_cache/
|
|
37
|
+
.pyre/
|
|
38
|
+
.pytype/
|
|
39
|
+
.ruff_cache/
|
|
40
|
+
tests/browser/smoke-*.png
|
|
41
|
+
|
|
42
|
+
# --- JupyterLab extension (launcher/labextension/) ---
|
|
43
|
+
node_modules/
|
|
44
|
+
launcher/labextension/lib/
|
|
45
|
+
launcher/labextension/.yarn/
|
|
46
|
+
*.tsbuildinfo
|
|
47
|
+
# Built into the launcher package by `jlpm build`
|
|
48
|
+
launcher/src/jupyter_storage_explorer_launcher/labextension/
|
|
49
|
+
|
|
50
|
+
# --- IPython / Jupyter ---
|
|
51
|
+
.ipynb_checkpoints/
|
|
52
|
+
*/.ipynb_checkpoints/*
|
|
53
|
+
profile_default/
|
|
54
|
+
ipython_config.py
|
|
55
|
+
.jupyter/
|
|
56
|
+
.virtual_documents/
|
|
57
|
+
|
|
58
|
+
# --- JetBrains IDEs (IntelliJ IDEA, PyCharm, ...) ---
|
|
59
|
+
.idea/
|
|
60
|
+
*.iml
|
|
61
|
+
*.ipr
|
|
62
|
+
*.iws
|
|
63
|
+
out/
|
|
64
|
+
.fleet/
|
|
65
|
+
|
|
66
|
+
# --- Environment and secrets ---
|
|
67
|
+
.env
|
|
68
|
+
.env.*
|
|
69
|
+
|
|
70
|
+
# --- OS files ---
|
|
71
|
+
.DS_Store
|
|
72
|
+
Thumbs.db
|
|
73
|
+
desktop.ini
|
|
74
|
+
|
|
75
|
+
# Others
|
|
76
|
+
notebooks/query-settings.json
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tsuyoshi SUGIBUCHI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: jupyter-storage-explorer-databricks-support
|
|
3
|
+
Version: 0.18.0
|
|
4
|
+
Summary: Databricks support for Storage Explorer
|
|
5
|
+
Project-URL: Homepage, https://github.com/sugibuchi/jupyter-storage-explorer/tree/main/plugins/databricks-support
|
|
6
|
+
Project-URL: Source, https://github.com/sugibuchi/jupyter-storage-explorer
|
|
7
|
+
Project-URL: Issues, https://github.com/sugibuchi/jupyter-storage-explorer/issues
|
|
8
|
+
Author: Tsuyoshi SUGIBUCHI
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: databricks,fsspec-databricks,jupyter,jupyter-storage-explorer,unity catalog,volumes
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Framework :: Jupyter
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: jupyter-storage-explorer==0.18.0
|
|
20
|
+
Provides-Extra: full
|
|
21
|
+
Requires-Dist: fsspec-databricks>=0.1.10; extra == 'full'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# jupyter-storage-explorer-databricks-support
|
|
25
|
+
|
|
26
|
+
Databricks support for Storage Explorer,
|
|
27
|
+
the [Jupyter Storage Explorer](https://github.com/sugibuchi/jupyter-storage-explorer) plugin for
|
|
28
|
+
a Databricks workspace's files: Unity Catalog volumes, workspace files and legacy DBFS, through
|
|
29
|
+
[fsspec-databricks](https://github.com/sugibuchi/fsspec-databricks).
|
|
30
|
+
|
|
31
|
+
> **Experimental.** fsspec-databricks is experimental, and so is this plugin: it is not recommended for production
|
|
32
|
+
> use. Feedback is welcome, in this repository's
|
|
33
|
+
> [issues](https://github.com/sugibuchi/jupyter-storage-explorer/issues).
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install "jupyter-storage-explorer[databricks]" # the plugin and its library
|
|
39
|
+
pip install "jupyter-storage-explorer[databricks-support]" # the plugin only, with the library you have
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The library is not the plugin's dependency, so that installing it never changes those of a shared environment. The
|
|
43
|
+
first command installs it, as does `jupyter-storage-explorer-databricks-support[full]`:
|
|
44
|
+
|
|
45
|
+
| Library | Needed for |
|
|
46
|
+
|---------------------|-----------------------------------------------|
|
|
47
|
+
| `fsspec-databricks` | Mandatory: the workspace's files, through fsspec |
|
|
48
|
+
|
|
49
|
+
The plugin is not in the explorer's `all` and `all-supports` extras: install it on purpose.
|
|
50
|
+
|
|
51
|
+
### Prerequisites
|
|
52
|
+
|
|
53
|
+
- fsspec-databricks reaches the workspace with the Databricks SDK's configuration: e.g. `DATABRICKS_HOST` and a
|
|
54
|
+
credential, a profile of `~/.databrickscfg`, or the notebook's own on Databricks. See
|
|
55
|
+
[Databricks unified authentication](https://docs.databricks.com/aws/en/dev-tools/auth/unified-auth).
|
|
56
|
+
|
|
57
|
+
## What it does
|
|
58
|
+
|
|
59
|
+
Give the explorer fsspec-databricks' filesystem:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from fsspec_databricks import DatabricksFileSystem
|
|
63
|
+
from jupyter_storage_explorer import StorageExplorer
|
|
64
|
+
|
|
65
|
+
StorageExplorer(filesystems={"Databricks": DatabricksFileSystem()})
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
URLs work too, e.g. `"dbfs://Volumes/main/sales"`, once `fsspec_databricks.use()` has made fsspec-databricks
|
|
69
|
+
fsspec's `dbfs` filesystem.
|
|
70
|
+
|
|
71
|
+
The plugin gives the files locations that name the workspace, by its host, besides their path, e.g.
|
|
72
|
+
`dbfs://Volumes/main/sales/raw/a.csv`:
|
|
73
|
+
|
|
74
|
+
- Two workspaces' files of the same path are told apart, also by "Paste" and the transfer policy's checks.
|
|
75
|
+
- Files copied in an explorer of another kernel are found again in the same workspace, even if this explorer did not
|
|
76
|
+
show that storage, once `fsspec_databricks.use()` is called.
|
|
77
|
+
|
|
78
|
+
The folders of volumes, catalogs, schemas, repositories and users get icons of their own, as without the plugin.
|
|
79
|
+
pyspark-support's code reads the files as Databricks names them, `dbfs:/Volumes/...`.
|
|
80
|
+
|
|
81
|
+
## Plugin options
|
|
82
|
+
|
|
83
|
+
`plugin_options` gives the plugin's options:
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
StorageExplorer(
|
|
87
|
+
filesystems={"Databricks": DatabricksFileSystem()},
|
|
88
|
+
plugin_options={"databricks-support": {"priority": 10}},
|
|
89
|
+
)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
In the launcher's tabs, `plugin_options()` in `launcher.py` gives them:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
def plugin_options():
|
|
96
|
+
return {"databricks-support": {"enabled": False}}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
| Option | Default | Meaning |
|
|
100
|
+
|------------|---------|---------------------------------------------------------------------------------------------------------|
|
|
101
|
+
| `enabled` | `True` | Whether explorers use the plugin. `False`: not even imported. |
|
|
102
|
+
| `priority` | lowest | Plugins of a higher priority come first: their mappers are tried before the others'. Equal ones by key. |
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# jupyter-storage-explorer-databricks-support
|
|
2
|
+
|
|
3
|
+
Databricks support for Storage Explorer,
|
|
4
|
+
the [Jupyter Storage Explorer](https://github.com/sugibuchi/jupyter-storage-explorer) plugin for
|
|
5
|
+
a Databricks workspace's files: Unity Catalog volumes, workspace files and legacy DBFS, through
|
|
6
|
+
[fsspec-databricks](https://github.com/sugibuchi/fsspec-databricks).
|
|
7
|
+
|
|
8
|
+
> **Experimental.** fsspec-databricks is experimental, and so is this plugin: it is not recommended for production
|
|
9
|
+
> use. Feedback is welcome, in this repository's
|
|
10
|
+
> [issues](https://github.com/sugibuchi/jupyter-storage-explorer/issues).
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install "jupyter-storage-explorer[databricks]" # the plugin and its library
|
|
16
|
+
pip install "jupyter-storage-explorer[databricks-support]" # the plugin only, with the library you have
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The library is not the plugin's dependency, so that installing it never changes those of a shared environment. The
|
|
20
|
+
first command installs it, as does `jupyter-storage-explorer-databricks-support[full]`:
|
|
21
|
+
|
|
22
|
+
| Library | Needed for |
|
|
23
|
+
|---------------------|-----------------------------------------------|
|
|
24
|
+
| `fsspec-databricks` | Mandatory: the workspace's files, through fsspec |
|
|
25
|
+
|
|
26
|
+
The plugin is not in the explorer's `all` and `all-supports` extras: install it on purpose.
|
|
27
|
+
|
|
28
|
+
### Prerequisites
|
|
29
|
+
|
|
30
|
+
- fsspec-databricks reaches the workspace with the Databricks SDK's configuration: e.g. `DATABRICKS_HOST` and a
|
|
31
|
+
credential, a profile of `~/.databrickscfg`, or the notebook's own on Databricks. See
|
|
32
|
+
[Databricks unified authentication](https://docs.databricks.com/aws/en/dev-tools/auth/unified-auth).
|
|
33
|
+
|
|
34
|
+
## What it does
|
|
35
|
+
|
|
36
|
+
Give the explorer fsspec-databricks' filesystem:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from fsspec_databricks import DatabricksFileSystem
|
|
40
|
+
from jupyter_storage_explorer import StorageExplorer
|
|
41
|
+
|
|
42
|
+
StorageExplorer(filesystems={"Databricks": DatabricksFileSystem()})
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
URLs work too, e.g. `"dbfs://Volumes/main/sales"`, once `fsspec_databricks.use()` has made fsspec-databricks
|
|
46
|
+
fsspec's `dbfs` filesystem.
|
|
47
|
+
|
|
48
|
+
The plugin gives the files locations that name the workspace, by its host, besides their path, e.g.
|
|
49
|
+
`dbfs://Volumes/main/sales/raw/a.csv`:
|
|
50
|
+
|
|
51
|
+
- Two workspaces' files of the same path are told apart, also by "Paste" and the transfer policy's checks.
|
|
52
|
+
- Files copied in an explorer of another kernel are found again in the same workspace, even if this explorer did not
|
|
53
|
+
show that storage, once `fsspec_databricks.use()` is called.
|
|
54
|
+
|
|
55
|
+
The folders of volumes, catalogs, schemas, repositories and users get icons of their own, as without the plugin.
|
|
56
|
+
pyspark-support's code reads the files as Databricks names them, `dbfs:/Volumes/...`.
|
|
57
|
+
|
|
58
|
+
## Plugin options
|
|
59
|
+
|
|
60
|
+
`plugin_options` gives the plugin's options:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
StorageExplorer(
|
|
64
|
+
filesystems={"Databricks": DatabricksFileSystem()},
|
|
65
|
+
plugin_options={"databricks-support": {"priority": 10}},
|
|
66
|
+
)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
In the launcher's tabs, `plugin_options()` in `launcher.py` gives them:
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
def plugin_options():
|
|
73
|
+
return {"databricks-support": {"enabled": False}}
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
| Option | Default | Meaning |
|
|
77
|
+
|------------|---------|---------------------------------------------------------------------------------------------------------|
|
|
78
|
+
| `enabled` | `True` | Whether explorers use the plugin. `False`: not even imported. |
|
|
79
|
+
| `priority` | lowest | Plugins of a higher priority come first: their mappers are tried before the others'. Equal ones by key. |
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""The plugin's dependencies: jupyter-storage-explorer at the plugin's own
|
|
2
|
+
version.
|
|
3
|
+
|
|
4
|
+
The two are released together, with the same version (from the git tags),
|
|
5
|
+
and the plugin may use the explorer's internals: any other version of the
|
|
6
|
+
explorer may not work with it. Its libraries are not dependencies (see the
|
|
7
|
+
"full" extra in pyproject.toml). Hatchling has resolved the version when it
|
|
8
|
+
runs this hook.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from hatchling.metadata.plugin.interface import MetadataHookInterface
|
|
16
|
+
from packaging.version import Version
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class PluginMetadataHook(MetadataHookInterface):
|
|
20
|
+
def update(self, metadata: dict[str, Any]) -> None:
|
|
21
|
+
# Without the local part (+g<commit>.d<date>) of development builds,
|
|
22
|
+
# which "==" ignores, so an editable explorer of the same commit matches.
|
|
23
|
+
version = Version(metadata["version"]).public
|
|
24
|
+
metadata["dependencies"] = [f"jupyter-storage-explorer=={version}"]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "jupyter-storage-explorer-databricks-support"
|
|
3
|
+
dynamic = ["version", "dependencies"]
|
|
4
|
+
description = "Databricks support for Storage Explorer"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Tsuyoshi SUGIBUCHI" }]
|
|
9
|
+
keywords = ["jupyter", "jupyter-storage-explorer", "databricks", "unity catalog", "volumes", "fsspec-databricks"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
# Experimental, as fsspec-databricks is.
|
|
12
|
+
"Development Status :: 3 - Alpha",
|
|
13
|
+
"Framework :: Jupyter",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
17
|
+
"Topic :: Scientific/Engineering",
|
|
18
|
+
]
|
|
19
|
+
requires-python = ">=3.11"
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Homepage = "https://github.com/sugibuchi/jupyter-storage-explorer/tree/main/plugins/databricks-support"
|
|
23
|
+
Source = "https://github.com/sugibuchi/jupyter-storage-explorer"
|
|
24
|
+
Issues = "https://github.com/sugibuchi/jupyter-storage-explorer/issues"
|
|
25
|
+
|
|
26
|
+
# Explorers load the plugin by this entry point, named by its key.
|
|
27
|
+
[project.entry-points."jupyter_storage_explorer.plugins"]
|
|
28
|
+
databricks-support = "jupyter_storage_explorer_databricks_support:DatabricksSupport"
|
|
29
|
+
|
|
30
|
+
# The libraries the plugin uses: not dependencies, so that installing the
|
|
31
|
+
# plugin never changes those of a shared environment; "full" installs them.
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
full = [
|
|
34
|
+
"fsspec-databricks>=0.1.10",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[build-system]
|
|
38
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
39
|
+
build-backend = "hatchling.build"
|
|
40
|
+
|
|
41
|
+
# The same version as jupyter-storage-explorer: from the repository's git tags.
|
|
42
|
+
[tool.hatch.version]
|
|
43
|
+
source = "vcs"
|
|
44
|
+
raw-options = { root = "../..", version_scheme = "post-release" }
|
|
45
|
+
|
|
46
|
+
# The dependencies, set by hatch_build.py: jupyter-storage-explorer at the
|
|
47
|
+
# plugin's own version.
|
|
48
|
+
[tool.hatch.metadata.hooks.custom]
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.wheel]
|
|
51
|
+
packages = ["src/jupyter_storage_explorer_databricks_support"]
|
|
52
|
+
|
|
53
|
+
[dependency-groups]
|
|
54
|
+
dev = [
|
|
55
|
+
# The tests (tests/) fake fsspec-databricks' filesystem, which needs a
|
|
56
|
+
# workspace: the library itself is not needed to run them.
|
|
57
|
+
"pytest>=9.1.1",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
[tool.uv]
|
|
61
|
+
cache-keys = [{ file = "pyproject.toml" }, { file = "hatch_build.py" }, { git = { commit = true, tags = true } }]
|
|
62
|
+
|
|
63
|
+
[tool.uv.sources]
|
|
64
|
+
jupyter-storage-explorer = { workspace = true }
|
|
65
|
+
|
|
66
|
+
[tool.pytest.ini_options]
|
|
67
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Databricks support for Storage Explorer.
|
|
2
|
+
|
|
3
|
+
A Databricks workspace's files, Unity Catalog volumes, workspace files and
|
|
4
|
+
legacy DBFS, through fsspec-databricks, with locations that name the
|
|
5
|
+
workspace. Experimental, as fsspec-databricks is.
|
|
6
|
+
|
|
7
|
+
The plugin, ``DatabricksSupport``, is found by explorers through its entry
|
|
8
|
+
point (``databricks-support``); its library is installed with the "full"
|
|
9
|
+
extra."""
|
|
10
|
+
|
|
11
|
+
from jupyter_storage_explorer import StorageExplorerPlugin
|
|
12
|
+
from jupyter_storage_explorer_databricks_support.mapper import (
|
|
13
|
+
DatabricksLocation,
|
|
14
|
+
DatabricksWorkspaceMapper,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DatabricksSupport(StorageExplorerPlugin):
|
|
19
|
+
key = "databricks-support"
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def filesystem_mappers(self):
|
|
23
|
+
return (DatabricksWorkspaceMapper(),)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"DatabricksLocation",
|
|
28
|
+
"DatabricksSupport",
|
|
29
|
+
"DatabricksWorkspaceMapper",
|
|
30
|
+
]
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Locations of a Databricks workspace's files that name the workspace.
|
|
2
|
+
|
|
3
|
+
The explorer's own ``DatabricksMapper`` gives fsspec's URLs,
|
|
4
|
+
``dbfs://Volumes/...``, which do not tell the workspace: two workspaces' files
|
|
5
|
+
of the same path look the same, and files copied from another kernel are
|
|
6
|
+
looked for among the explorer's storages only. This mapper gives
|
|
7
|
+
fsspec-databricks' filesystems ``DatabricksLocation``s, with the workspace's
|
|
8
|
+
host, so that each is told apart, and found again from another kernel.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import dataclasses
|
|
14
|
+
from typing import TYPE_CHECKING
|
|
15
|
+
from urllib.parse import urlsplit
|
|
16
|
+
|
|
17
|
+
from jupyter_storage_explorer import Location
|
|
18
|
+
from jupyter_storage_explorer.filesystems import DatabricksMapper, FileSystemMapper
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from fsspec import AbstractFileSystem
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclasses.dataclass(frozen=True, kw_only=True)
|
|
25
|
+
class DatabricksLocation(Location):
|
|
26
|
+
"""A file or folder of a Databricks workspace: a Unity Catalog volume's,
|
|
27
|
+
workspace files or legacy DBFS (``dbfs://Volumes/...``).
|
|
28
|
+
|
|
29
|
+
Attributes
|
|
30
|
+
----------
|
|
31
|
+
host : str
|
|
32
|
+
The workspace's host name, in lower case, e.g.
|
|
33
|
+
"adb-1234567890123456.7.azuredatabricks.net". Not in the URL, which
|
|
34
|
+
libraries read as they read fsspec's: it tells the workspace apart.
|
|
35
|
+
path : str
|
|
36
|
+
The path in the workspace, from "/", e.g. "/Volumes/main/sales/raw".
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
kind = "databricks"
|
|
40
|
+
|
|
41
|
+
host: str
|
|
42
|
+
path: str = "/"
|
|
43
|
+
|
|
44
|
+
def __post_init__(self) -> None:
|
|
45
|
+
object.__setattr__(self, "host", _host_name(self.host))
|
|
46
|
+
object.__setattr__(self, "path", "/" + self.path.strip("/"))
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def url(self) -> str:
|
|
50
|
+
"""``dbfs://Volumes/...``: the form fsspec reads (see the explorer's
|
|
51
|
+
``DatabricksMapper``), and pyspark-support's ``spark_url`` turns
|
|
52
|
+
into Databricks' ``dbfs:/Volumes/...``."""
|
|
53
|
+
return f"dbfs://{self.path.lstrip('/')}"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _host_name(host: str) -> str:
|
|
57
|
+
"""``host`` as a host name in lower case: without a scheme, port, path or
|
|
58
|
+
trailing dot, e.g. "https://ADB-1.azuredatabricks.net/" gives
|
|
59
|
+
"adb-1.azuredatabricks.net"."""
|
|
60
|
+
host = host.strip()
|
|
61
|
+
name = urlsplit(host if "//" in host else f"//{host}").hostname or ""
|
|
62
|
+
return name.rstrip(".")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class DatabricksWorkspaceMapper(FileSystemMapper):
|
|
66
|
+
"""fsspec-databricks' ``DatabricksFileSystem``: locations with the
|
|
67
|
+
workspace's host (``DatabricksLocation``), and the folders' levels of the
|
|
68
|
+
explorer's ``DatabricksMapper`` (volumes, catalogs, schemas, ...).
|
|
69
|
+
|
|
70
|
+
The host is the filesystem's Databricks SDK configuration's, which
|
|
71
|
+
fsspec-databricks makes once, when the filesystem first needs it (also to
|
|
72
|
+
list). A configuration always has one: without it, the SDK raises.
|
|
73
|
+
|
|
74
|
+
Not a subclass of ``DatabricksMapper``: the explorer takes the mappers of
|
|
75
|
+
fsspec's URLs (``FsspecMapper`` and its subclasses) to know nothing of
|
|
76
|
+
what an instance reaches, while this one knows its workspace, so that
|
|
77
|
+
two instances of one workspace are one storage.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
classes = ("fsspec_databricks.spec.DatabricksFileSystem",)
|
|
81
|
+
|
|
82
|
+
def __init__(self) -> None:
|
|
83
|
+
# The explorer's own: the levels, and fsspec's URLs.
|
|
84
|
+
self._fsspec = DatabricksMapper()
|
|
85
|
+
|
|
86
|
+
def level(self, fs: AbstractFileSystem, path: str) -> str | None:
|
|
87
|
+
return self._fsspec.level(fs, path)
|
|
88
|
+
|
|
89
|
+
@staticmethod
|
|
90
|
+
def host(fs: AbstractFileSystem) -> str:
|
|
91
|
+
"""The host name of ``fs``'s workspace."""
|
|
92
|
+
return _host_name(fs.config.host)
|
|
93
|
+
|
|
94
|
+
def to_location(self, fs: AbstractFileSystem, path: str) -> Location:
|
|
95
|
+
return DatabricksLocation(host=self.host(fs), path=path)
|
|
96
|
+
|
|
97
|
+
def path_of(self, fs: AbstractFileSystem, location: Location) -> str | None:
|
|
98
|
+
# Only a location of this filesystem's workspace; fsspec's URLs
|
|
99
|
+
# (another kernel without this plugin) as the explorer's mapper does.
|
|
100
|
+
if isinstance(location, DatabricksLocation):
|
|
101
|
+
return location.path if location.host == self.host(fs) else None
|
|
102
|
+
return self._fsspec.path_of(fs, location)
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""databricks-support: locations naming the workspace, on a fake of
|
|
2
|
+
fsspec-databricks' filesystem (the real one needs a workspace); its entry
|
|
3
|
+
point."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from importlib.metadata import entry_points
|
|
8
|
+
from types import SimpleNamespace
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
from fsspec import AbstractFileSystem
|
|
12
|
+
from jupyter_storage_explorer_databricks_support import (
|
|
13
|
+
DatabricksLocation,
|
|
14
|
+
DatabricksSupport,
|
|
15
|
+
DatabricksWorkspaceMapper,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
from jupyter_storage_explorer import Location
|
|
19
|
+
from jupyter_storage_explorer.filesystems import (
|
|
20
|
+
FsspecLocation,
|
|
21
|
+
MapperChain,
|
|
22
|
+
default_mappers,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
HOST = "https://ADB-1234567890123456.7.azuredatabricks.net/"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DatabricksFileSystem(AbstractFileSystem):
|
|
29
|
+
"""As fsspec-databricks' (matched by its qualified name): ``config.host``
|
|
30
|
+
is the workspace's, as the Databricks SDK's configuration gives it."""
|
|
31
|
+
|
|
32
|
+
cachable = False # each test's own instances
|
|
33
|
+
protocol = "dbfs"
|
|
34
|
+
|
|
35
|
+
def __init__(self, host=HOST, **kwargs):
|
|
36
|
+
super().__init__(**kwargs)
|
|
37
|
+
self._host = host
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def config(self):
|
|
41
|
+
return SimpleNamespace(host=self._host)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
DatabricksFileSystem.__module__ = "fsspec_databricks.spec"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@pytest.fixture
|
|
48
|
+
def chain():
|
|
49
|
+
"""The mappers of an explorer with the plugin: its first."""
|
|
50
|
+
return MapperChain([*DatabricksSupport().filesystem_mappers, *default_mappers()])
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TestLocations:
|
|
54
|
+
def test_with_the_workspaces_host(self, chain):
|
|
55
|
+
location = chain.location_of(
|
|
56
|
+
DatabricksFileSystem(), "/Volumes/main/sales/raw/a.csv"
|
|
57
|
+
)
|
|
58
|
+
assert location == DatabricksLocation(
|
|
59
|
+
host="adb-1234567890123456.7.azuredatabricks.net",
|
|
60
|
+
path="/Volumes/main/sales/raw/a.csv",
|
|
61
|
+
)
|
|
62
|
+
# The URL libraries read, as the explorer's own mapper gives it.
|
|
63
|
+
assert location.url == "dbfs://Volumes/main/sales/raw/a.csv"
|
|
64
|
+
|
|
65
|
+
@pytest.mark.parametrize(
|
|
66
|
+
("path", "level"),
|
|
67
|
+
[
|
|
68
|
+
("", "workspace"),
|
|
69
|
+
("/Volumes/main", "catalog"),
|
|
70
|
+
("/Workspace/Repos", "repos"),
|
|
71
|
+
],
|
|
72
|
+
)
|
|
73
|
+
def test_the_explorers_levels(self, chain, path, level):
|
|
74
|
+
assert chain.level_of(DatabricksFileSystem(), path) == level
|
|
75
|
+
|
|
76
|
+
def test_the_top(self, chain):
|
|
77
|
+
location = chain.location_of(DatabricksFileSystem(), "")
|
|
78
|
+
assert (location.path, location.url) == ("/", "dbfs://")
|
|
79
|
+
|
|
80
|
+
def test_through_json(self, chain):
|
|
81
|
+
location = chain.location_of(DatabricksFileSystem(), "/Workspace/Shared/x.py")
|
|
82
|
+
assert Location.from_json(location.to_json()) == location
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class TestWorkspacesToldApart:
|
|
86
|
+
def test_files_of_the_same_workspace(self, chain):
|
|
87
|
+
here, there = DatabricksFileSystem(), DatabricksFileSystem(host=HOST.lower())
|
|
88
|
+
location = chain.location_of(there, "/Volumes/main/s/v/a.csv")
|
|
89
|
+
assert chain.path_in(here, location) == "/Volumes/main/s/v/a.csv"
|
|
90
|
+
assert chain.same_filesystem(here, there)
|
|
91
|
+
|
|
92
|
+
def test_files_of_another_workspace(self, chain):
|
|
93
|
+
here = DatabricksFileSystem()
|
|
94
|
+
there = DatabricksFileSystem(host="https://dbc-other.cloud.databricks.com")
|
|
95
|
+
location = chain.location_of(there, "/Volumes/main/s/v/a.csv")
|
|
96
|
+
assert chain.path_in(here, location) is None
|
|
97
|
+
assert not chain.same_filesystem(here, there)
|
|
98
|
+
|
|
99
|
+
def test_files_pasted_from_another_kernel_found(self, chain, memfs):
|
|
100
|
+
# A location from JSON, as another kernel's explorer copied it: found
|
|
101
|
+
# on the storage of its workspace among the explorer's.
|
|
102
|
+
location = Location.from_json(
|
|
103
|
+
DatabricksLocation(host=HOST, path="/Volumes/main/s/v/a.csv").to_json()
|
|
104
|
+
)
|
|
105
|
+
fs = DatabricksFileSystem()
|
|
106
|
+
assert chain.locate(location, [memfs, fs]) == (fs, "/Volumes/main/s/v/a.csv")
|
|
107
|
+
|
|
108
|
+
def test_fsspecs_urls_still_reached(self, chain):
|
|
109
|
+
# From a kernel without the plugin: the explorer's own form.
|
|
110
|
+
location = FsspecLocation(url="dbfs://Volumes/main/s/v/a.csv")
|
|
111
|
+
assert (
|
|
112
|
+
chain.path_in(DatabricksFileSystem(), location) == "/Volumes/main/s/v/a.csv"
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@pytest.mark.parametrize(
|
|
117
|
+
("given", "host"),
|
|
118
|
+
[
|
|
119
|
+
("https://ADB-1.azuredatabricks.net/", "adb-1.azuredatabricks.net"),
|
|
120
|
+
("adb-1.azuredatabricks.net", "adb-1.azuredatabricks.net"),
|
|
121
|
+
("https://dbc-x.cloud.databricks.com:443/", "dbc-x.cloud.databricks.com"),
|
|
122
|
+
],
|
|
123
|
+
)
|
|
124
|
+
def test_hosts_normalized(given, host):
|
|
125
|
+
assert DatabricksLocation(host=given, path="Volumes/x/").host == host
|
|
126
|
+
assert DatabricksLocation(host=given, path="Volumes/x/").path == "/Volumes/x"
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_plugin_registered():
|
|
130
|
+
[entry_point] = entry_points(
|
|
131
|
+
group="jupyter_storage_explorer.plugins", name="databricks-support"
|
|
132
|
+
)
|
|
133
|
+
plugin = entry_point.load()() # loads without fsspec-databricks
|
|
134
|
+
assert isinstance(plugin, DatabricksSupport)
|
|
135
|
+
assert plugin.description == "Databricks support for Storage Explorer"
|
|
136
|
+
[mapper] = plugin.filesystem_mappers
|
|
137
|
+
assert isinstance(mapper, DatabricksWorkspaceMapper)
|