dash-ingest 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dash_ingest-0.1.1/.github/workflows/ci.yml +53 -0
- dash_ingest-0.1.1/.github/workflows/daily.yml +33 -0
- dash_ingest-0.1.1/.github/workflows/release.yml +268 -0
- dash_ingest-0.1.1/.gitignore +9 -0
- dash_ingest-0.1.1/CLAUDE.md +40 -0
- dash_ingest-0.1.1/PKG-INFO +179 -0
- dash_ingest-0.1.1/README.md +149 -0
- dash_ingest-0.1.1/dashingest/__init__.py +52 -0
- dash_ingest-0.1.1/dashingest/connectors.py +204 -0
- dash_ingest-0.1.1/dashingest/ingestor.py +279 -0
- dash_ingest-0.1.1/dashingest/readers.py +200 -0
- dash_ingest-0.1.1/dashingest/ui.py +315 -0
- dash_ingest-0.1.1/docs/api/dashingest.html +2971 -0
- dash_ingest-0.1.1/docs/api/index.html +7 -0
- dash_ingest-0.1.1/docs/api/search.js +46 -0
- dash_ingest-0.1.1/marketplace/listing.json +13 -0
- dash_ingest-0.1.1/marketplace/quickstart_notebook.py +29 -0
- dash_ingest-0.1.1/pyproject.toml +41 -0
- dash_ingest-0.1.1/tests/test_connectors.py +141 -0
- dash_ingest-0.1.1/tests/test_dashingest.py +16 -0
- dash_ingest-0.1.1/tests/test_ingestor_helpers.py +52 -0
- dash_ingest-0.1.1/tests/test_readers.py +151 -0
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
lint:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@v4
|
|
14
|
+
- uses: actions/setup-python@v5
|
|
15
|
+
with:
|
|
16
|
+
python-version: "3.11"
|
|
17
|
+
- run: pip install ruff
|
|
18
|
+
- run: ruff check dashingest/
|
|
19
|
+
|
|
20
|
+
test:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
needs: lint
|
|
23
|
+
strategy:
|
|
24
|
+
matrix:
|
|
25
|
+
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v4
|
|
28
|
+
- uses: actions/setup-python@v5
|
|
29
|
+
with:
|
|
30
|
+
python-version: ${{ matrix.python-version }}
|
|
31
|
+
- name: Install
|
|
32
|
+
run: pip install -e ".[dev]" pytest pytest-cov
|
|
33
|
+
- name: Test
|
|
34
|
+
run: pytest tests/ -v --cov=dashingest --cov-report=xml
|
|
35
|
+
- name: Upload coverage
|
|
36
|
+
uses: codecov/codecov-action@v4
|
|
37
|
+
with:
|
|
38
|
+
files: coverage.xml
|
|
39
|
+
|
|
40
|
+
build:
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
needs: test
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/checkout@v4
|
|
45
|
+
- uses: actions/setup-python@v5
|
|
46
|
+
with:
|
|
47
|
+
python-version: "3.11"
|
|
48
|
+
- run: pip install hatch
|
|
49
|
+
- run: hatch build
|
|
50
|
+
- uses: actions/upload-artifact@v4
|
|
51
|
+
with:
|
|
52
|
+
name: dist
|
|
53
|
+
path: dist/
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: Daily Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
schedule:
|
|
5
|
+
- cron: "0 6 * * *" # Every day 06:00 UTC — tests only, no commit
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
name: Test (Python ${{ matrix.python-version }})
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: ${{ matrix.python-version }}
|
|
22
|
+
|
|
23
|
+
- name: Install
|
|
24
|
+
run: pip install -e ".[dev]" pytest pytest-cov
|
|
25
|
+
|
|
26
|
+
- name: Run tests
|
|
27
|
+
run: pytest tests/ -v --cov=dashingest --cov-report=xml --cov-report=term-missing
|
|
28
|
+
|
|
29
|
+
- name: Upload coverage
|
|
30
|
+
uses: codecov/codecov-action@v4
|
|
31
|
+
with:
|
|
32
|
+
files: coverage.xml
|
|
33
|
+
fail_ci_if_error: false
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
name: Weekly Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
schedule:
|
|
5
|
+
- cron: "0 9 * * 1" # Every Monday 09:00 UTC
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
inputs:
|
|
8
|
+
release_note:
|
|
9
|
+
description: "Optional release note (shown in GitHub release body)"
|
|
10
|
+
required: false
|
|
11
|
+
default: ""
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
# ── Gate: tests must pass ────────────────────────────────────────────────
|
|
15
|
+
test:
|
|
16
|
+
name: Test (Python ${{ matrix.python-version }})
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
strategy:
|
|
19
|
+
fail-fast: true
|
|
20
|
+
matrix:
|
|
21
|
+
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{ matrix.python-version }}
|
|
28
|
+
|
|
29
|
+
- name: Install
|
|
30
|
+
run: pip install -e ".[dev]" pytest pytest-cov
|
|
31
|
+
|
|
32
|
+
- name: Run tests
|
|
33
|
+
run: pytest tests/ -v --cov=dashingest --cov-report=xml --cov-report=term-missing
|
|
34
|
+
|
|
35
|
+
- name: Upload coverage
|
|
36
|
+
uses: codecov/codecov-action@v4
|
|
37
|
+
with:
|
|
38
|
+
files: coverage.xml
|
|
39
|
+
fail_ci_if_error: false
|
|
40
|
+
|
|
41
|
+
# ── Generate docs ────────────────────────────────────────────────────────
|
|
42
|
+
docs:
|
|
43
|
+
name: Generate API docs
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
needs: test
|
|
46
|
+
steps:
|
|
47
|
+
- uses: actions/checkout@v4
|
|
48
|
+
|
|
49
|
+
- uses: actions/setup-python@v5
|
|
50
|
+
with:
|
|
51
|
+
python-version: "3.11"
|
|
52
|
+
|
|
53
|
+
- name: Install
|
|
54
|
+
run: pip install -e ".[dev]" pdoc
|
|
55
|
+
|
|
56
|
+
- name: Generate docs
|
|
57
|
+
run: |
|
|
58
|
+
pdoc dashingest --output-dir docs/api --docformat google
|
|
59
|
+
echo "Docs generated at $(date -u)" > docs/api/.generated
|
|
60
|
+
|
|
61
|
+
- name: Upload docs artifact
|
|
62
|
+
uses: actions/upload-artifact@v4
|
|
63
|
+
with:
|
|
64
|
+
name: api-docs
|
|
65
|
+
path: docs/api/
|
|
66
|
+
|
|
67
|
+
# ── Release: tag, GitHub release, commit docs ────────────────────────────
|
|
68
|
+
release:
|
|
69
|
+
name: Bump version & release
|
|
70
|
+
runs-on: ubuntu-latest
|
|
71
|
+
needs: [test, docs]
|
|
72
|
+
permissions:
|
|
73
|
+
contents: write
|
|
74
|
+
pull-requests: write
|
|
75
|
+
outputs:
|
|
76
|
+
version: ${{ steps.bump.outputs.version }}
|
|
77
|
+
steps:
|
|
78
|
+
- uses: actions/checkout@v4
|
|
79
|
+
with:
|
|
80
|
+
fetch-depth: 0
|
|
81
|
+
|
|
82
|
+
- uses: actions/setup-python@v5
|
|
83
|
+
with:
|
|
84
|
+
python-version: "3.11"
|
|
85
|
+
|
|
86
|
+
- name: Install build tools
|
|
87
|
+
run: pip install hatch pdoc
|
|
88
|
+
|
|
89
|
+
- name: Bump patch version
|
|
90
|
+
id: bump
|
|
91
|
+
run: |
|
|
92
|
+
current=$(hatch version)
|
|
93
|
+
hatch version patch
|
|
94
|
+
new=$(hatch version)
|
|
95
|
+
echo "version=$new" >> $GITHUB_OUTPUT
|
|
96
|
+
echo "prev_version=$current" >> $GITHUB_OUTPUT
|
|
97
|
+
echo "Bumped $current → $new"
|
|
98
|
+
|
|
99
|
+
- name: Regenerate docs into repo
|
|
100
|
+
run: |
|
|
101
|
+
pip install -e ".[dev]"
|
|
102
|
+
pdoc dashingest --output-dir docs/api --docformat google
|
|
103
|
+
|
|
104
|
+
- name: Build wheel + sdist
|
|
105
|
+
run: hatch build
|
|
106
|
+
|
|
107
|
+
- name: Write release notes
|
|
108
|
+
env:
|
|
109
|
+
VERSION: ${{ steps.bump.outputs.version }}
|
|
110
|
+
PREV_VERSION: ${{ steps.bump.outputs.prev_version }}
|
|
111
|
+
RELEASE_NOTE: ${{ github.event.inputs.release_note }}
|
|
112
|
+
run: |
|
|
113
|
+
cat > RELEASE_NOTES.md << EOF
|
|
114
|
+
## DashIngest — Data Ingestion v${VERSION}
|
|
115
|
+
|
|
116
|
+
**Released:** $(date -u '+%Y-%m-%d')
|
|
117
|
+
**Previous:** v${PREV_VERSION}
|
|
118
|
+
|
|
119
|
+
$( [ -n "${RELEASE_NOTE}" ] && echo "### Notes" && echo "${RELEASE_NOTE}" || true )
|
|
120
|
+
|
|
121
|
+
### What's included
|
|
122
|
+
- All tests passing across Python 3.9, 3.10, 3.11, 3.12
|
|
123
|
+
- API documentation regenerated (see \`docs/api/\`)
|
|
124
|
+
- Published to PyPI and Databricks Marketplace
|
|
125
|
+
|
|
126
|
+
### Install
|
|
127
|
+
\`\`\`bash
|
|
128
|
+
pip install dash-ingest==${VERSION}
|
|
129
|
+
\`\`\`
|
|
130
|
+
|
|
131
|
+
### Quick Start (Databricks notebook)
|
|
132
|
+
\`\`\`python
|
|
133
|
+
%pip install dash-ingest==${VERSION}
|
|
134
|
+
import dashingest
|
|
135
|
+
dashingest.launch()
|
|
136
|
+
\`\`\`
|
|
137
|
+
EOF
|
|
138
|
+
|
|
139
|
+
- name: Commit version bump + docs to a release branch
|
|
140
|
+
env:
|
|
141
|
+
VERSION: ${{ steps.bump.outputs.version }}
|
|
142
|
+
run: |
|
|
143
|
+
git config user.name "github-actions[bot]"
|
|
144
|
+
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
145
|
+
git push origin --delete "refs/tags/v${VERSION}" 2>/dev/null || true
|
|
146
|
+
git push origin --delete "release/v${VERSION}" 2>/dev/null || true
|
|
147
|
+
git tag -d "v${VERSION}" 2>/dev/null || true
|
|
148
|
+
git checkout -b "release/v${VERSION}"
|
|
149
|
+
git add .
|
|
150
|
+
git commit -m "release: v${VERSION} — tests passed, docs updated"
|
|
151
|
+
git tag "v${VERSION}"
|
|
152
|
+
git push origin "release/v${VERSION}"
|
|
153
|
+
git push origin "v${VERSION}"
|
|
154
|
+
|
|
155
|
+
- name: Open and auto-merge release PR
|
|
156
|
+
continue-on-error: true
|
|
157
|
+
env:
|
|
158
|
+
GH_TOKEN: ${{ secrets.RELEASE_TOKEN || github.token }}
|
|
159
|
+
VERSION: ${{ steps.bump.outputs.version }}
|
|
160
|
+
run: |
|
|
161
|
+
gh pr create --base main --head "release/v${VERSION}" \
|
|
162
|
+
--title "release: v${VERSION}" \
|
|
163
|
+
--body "Automated release PR — tests passed, docs regenerated, version bumped to v${VERSION}." \
|
|
164
|
+
2>/dev/null || true
|
|
165
|
+
gh pr merge "release/v${VERSION}" --merge --admin --delete-branch
|
|
166
|
+
|
|
167
|
+
- name: Create GitHub Release
|
|
168
|
+
uses: softprops/action-gh-release@v2
|
|
169
|
+
with:
|
|
170
|
+
tag_name: "v${{ steps.bump.outputs.version }}"
|
|
171
|
+
body_path: RELEASE_NOTES.md
|
|
172
|
+
files: dist/*
|
|
173
|
+
|
|
174
|
+
- name: Upload dist artifact for PyPI job
|
|
175
|
+
uses: actions/upload-artifact@v4
|
|
176
|
+
with:
|
|
177
|
+
name: dist
|
|
178
|
+
path: dist/
|
|
179
|
+
|
|
180
|
+
# ── Publish to PyPI (Trusted Publisher / OIDC — no token needed) ─────────
|
|
181
|
+
publish-pypi:
|
|
182
|
+
name: Publish to PyPI
|
|
183
|
+
runs-on: ubuntu-latest
|
|
184
|
+
needs: release
|
|
185
|
+
permissions:
|
|
186
|
+
id-token: write # required for OIDC trusted publisher
|
|
187
|
+
environment:
|
|
188
|
+
name: pypi
|
|
189
|
+
url: https://pypi.org/project/dash-ingest
|
|
190
|
+
steps:
|
|
191
|
+
- name: Download dist
|
|
192
|
+
uses: actions/download-artifact@v4
|
|
193
|
+
with:
|
|
194
|
+
name: dist
|
|
195
|
+
path: dist/
|
|
196
|
+
|
|
197
|
+
- name: Publish to PyPI
|
|
198
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
199
|
+
|
|
200
|
+
# ── Package for Databricks Marketplace ───────────────────────────────────
|
|
201
|
+
publish-databricks:
|
|
202
|
+
name: Package for Databricks Marketplace
|
|
203
|
+
runs-on: ubuntu-latest
|
|
204
|
+
needs: release
|
|
205
|
+
steps:
|
|
206
|
+
- uses: actions/checkout@v4
|
|
207
|
+
with:
|
|
208
|
+
ref: "v${{ needs.release.outputs.version }}"
|
|
209
|
+
|
|
210
|
+
- name: Download dist
|
|
211
|
+
uses: actions/download-artifact@v4
|
|
212
|
+
with:
|
|
213
|
+
name: dist
|
|
214
|
+
path: dist/
|
|
215
|
+
|
|
216
|
+
- name: Build Marketplace bundle
|
|
217
|
+
env:
|
|
218
|
+
VERSION: ${{ needs.release.outputs.version }}
|
|
219
|
+
run: |
|
|
220
|
+
mkdir -p marketplace-bundle/files
|
|
221
|
+
|
|
222
|
+
# Copy wheel
|
|
223
|
+
cp dist/*.whl marketplace-bundle/files/
|
|
224
|
+
|
|
225
|
+
# Generate companion notebook
|
|
226
|
+
cat > marketplace-bundle/files/DashIngest_Quickstart.py << NBEOF
|
|
227
|
+
# Databricks notebook source
|
|
228
|
+
# MAGIC %md
|
|
229
|
+
# MAGIC # DashIngest — Data Ingestion v${VERSION} for Databricks
|
|
230
|
+
# MAGIC Install and launch the interactive UI.
|
|
231
|
+
|
|
232
|
+
# COMMAND ----------
|
|
233
|
+
# MAGIC %pip install dash-ingest==${VERSION}
|
|
234
|
+
|
|
235
|
+
# COMMAND ----------
|
|
236
|
+
dbutils.library.restartPython()
|
|
237
|
+
|
|
238
|
+
# COMMAND ----------
|
|
239
|
+
import dashingest
|
|
240
|
+
dashingest.launch()
|
|
241
|
+
NBEOF
|
|
242
|
+
|
|
243
|
+
# Generate listing metadata
|
|
244
|
+
cat > marketplace-bundle/listing.json << LEOF
|
|
245
|
+
{
|
|
246
|
+
"listing_name": "DashIngest — Data Ingestion",
|
|
247
|
+
"version": "${VERSION}",
|
|
248
|
+
"short_description": "ADF-style ingestion from Volumes, ADLS, S3, databases and REST APIs into Delta tables",
|
|
249
|
+
"long_description": "DashIngest provides an ipywidgets UI inside Databricks notebooks to ingest data from Databricks Volumes, ADLS Gen2, Amazon S3, DBFS, relational databases, and REST APIs into Delta tables. No coding required for business users.",
|
|
250
|
+
"categories": ["Data Engineering,ETL"],
|
|
251
|
+
"tags": ["ingestion", "etl", "databricks", "unity-catalog", "pyspark", "jdbc"],
|
|
252
|
+
"provider": "dash-libs",
|
|
253
|
+
"documentation_url": "https://github.com/dash-libs/dash-ingest",
|
|
254
|
+
"source_url": "https://github.com/dash-libs/dash-ingest",
|
|
255
|
+
"pypi_package": "dash-ingest==${VERSION}"
|
|
256
|
+
}
|
|
257
|
+
LEOF
|
|
258
|
+
|
|
259
|
+
# Zip the bundle
|
|
260
|
+
cd marketplace-bundle && zip -r ../dashingest-marketplace-${VERSION}.zip .
|
|
261
|
+
echo "Bundle created: dashingest-marketplace-${VERSION}.zip"
|
|
262
|
+
|
|
263
|
+
- name: Upload Marketplace bundle artifact
|
|
264
|
+
uses: actions/upload-artifact@v4
|
|
265
|
+
with:
|
|
266
|
+
name: marketplace-bundle
|
|
267
|
+
path: dashingest-marketplace-*.zip
|
|
268
|
+
retention-days: 90
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# CLAUDE.md — dash-ingest
|
|
2
|
+
|
|
3
|
+
Part of the **Dashlibs** suite. See ~/dashlibs for the full context.
|
|
4
|
+
|
|
5
|
+
## Purpose
|
|
6
|
+
ADF-style ingestion from Databricks Volumes, ADLS Gen2, S3, DBFS, JDBC
|
|
7
|
+
databases, and REST APIs into Delta tables — pick a source kind, fill a
|
|
8
|
+
few plain fields (no hand-written URIs/JDBC strings), run.
|
|
9
|
+
|
|
10
|
+
## Structure
|
|
11
|
+
- `/connectors.py` — pure logic: source/target dataclasses (including
|
|
12
|
+
advanced JDBC options — SSL, fetch size, partitioned parallel reads,
|
|
13
|
+
raw `connection_properties`; and REST API auth/pagination), path building
|
|
14
|
+
(`resolve_path`), format inference (`infer_format_from_path`), JDBC URL/
|
|
15
|
+
driver construction (`build_jdbc_url`/`jdbc_driver`) — no Spark, fully
|
|
16
|
+
unit-tested
|
|
17
|
+
- `/readers.py` — pure logic: per-file-format reader option dataclasses
|
|
18
|
+
(`CsvReaderOptions`, `ExcelReaderOptions`, ...) and `build_reader_options()`
|
|
19
|
+
translating them into real Spark/spark-excel option keys — no Spark
|
|
20
|
+
- `/ingestor.py` — Spark/JDBC/REST-touching glue: `run_ingestion()`,
|
|
21
|
+
`test_connection()` (lightweight reachability check per source kind, no
|
|
22
|
+
data read — uses Spark's Hadoop filesystem API directly for path-based
|
|
23
|
+
sources, not `dbutils`), `preview()` (first N rows without writing
|
|
24
|
+
anywhere). Only its handful of pure helpers (`_extract_json_path`,
|
|
25
|
+
`_rest_auth_headers`, `_rest_basic_auth`) are unit-tested; the rest needs
|
|
26
|
+
a live SparkSession.
|
|
27
|
+
- `/ui.py` — ipywidgets UI (built on `dashui`, including
|
|
28
|
+
`dashui.editable_table()` for connection properties/headers/params),
|
|
29
|
+
`launch()` entrypoint
|
|
30
|
+
- `tests/` — pytest, no Spark dependency for unit tests
|
|
31
|
+
|
|
32
|
+
## Key Design Rules
|
|
33
|
+
- Never import Spark at module level — always inside functions
|
|
34
|
+
- UI calls core classes; never contains business logic
|
|
35
|
+
- `launch()` is always the public entrypoint for business users
|
|
36
|
+
|
|
37
|
+
## CI
|
|
38
|
+
- `ci.yml` — PR gate: lint → test → build
|
|
39
|
+
- `daily.yml` — 06:00 UTC: tests + .health/log.txt commit
|
|
40
|
+
- `release.yml`— Monday 09:00 UTC: patch bump + GitHub release
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dash-ingest
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: ADF-style data ingestion for Databricks — pick a source (Volume, ADLS, S3, Database, REST API), fill a few fields, run
|
|
5
|
+
Project-URL: Homepage, https://github.com/dash-libs/dash-ingest
|
|
6
|
+
Author-email: Darshan Shah <darshan.innovation@gmail.com>
|
|
7
|
+
License: Apache-2.0
|
|
8
|
+
Keywords: adls,databricks,delta,etl,ingestion,jdbc,pyspark
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Information Technology
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Requires-Dist: dash-uis==0.2.3
|
|
21
|
+
Requires-Dist: ipywidgets>=8.0
|
|
22
|
+
Requires-Dist: requests>=2.28
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: hatch; extra == 'dev'
|
|
25
|
+
Requires-Dist: pdoc; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
28
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# DashIngest — Databricks Library
|
|
32
|
+
|
|
33
|
+
[](https://github.com/dash-libs/dash-ingest/actions)
|
|
34
|
+
[](https://pypi.org/project/dash-ingest/)
|
|
35
|
+
[](LICENSE)
|
|
36
|
+
|
|
37
|
+
Part of the **[Dashlibs](https://github.com/dash-libs)** suite — Databricks libraries built for business users.
|
|
38
|
+
|
|
39
|
+
ADF-style data ingestion: pick a source kind, fill a few plain fields — no
|
|
40
|
+
hand-written `abfss://` URIs or JDBC connection strings — and run.
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
%pip install dash-ingest
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Quick Start
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
import dashingest
|
|
52
|
+
dashingest.launch() # Opens interactive UI in your Databricks notebook
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Or drive it directly from code:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from dashingest import ADLSSource, IngestTarget, run_ingestion
|
|
59
|
+
|
|
60
|
+
source = ADLSSource(storage_account="myacct", container="raw", path="sales/2024.csv")
|
|
61
|
+
target = IngestTarget(table="main.bronze.sales", write_mode="merge", merge_keys=["order_id"])
|
|
62
|
+
result = run_ingestion(source, target)
|
|
63
|
+
result.display()
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Sources
|
|
67
|
+
|
|
68
|
+
| Kind | What you provide |
|
|
69
|
+
|---|---|
|
|
70
|
+
| Databricks Volume | catalog, schema, volume, path |
|
|
71
|
+
| ADLS Gen2 | storage account, container, path |
|
|
72
|
+
| Amazon S3 | bucket, path |
|
|
73
|
+
| DBFS | path |
|
|
74
|
+
| Database (JDBC) | engine (postgres/mysql/sqlserver/oracle/snowflake), host, database, table or query |
|
|
75
|
+
| REST API | URL, optional JSON path to the records |
|
|
76
|
+
|
|
77
|
+
File format (csv/json/parquet/excel/avro/orc/text) is inferred from the
|
|
78
|
+
path's extension if not set explicitly — most ingestions need zero format
|
|
79
|
+
options.
|
|
80
|
+
|
|
81
|
+
## File format readers
|
|
82
|
+
|
|
83
|
+
Each format has its own options dataclass with real per-format defaults —
|
|
84
|
+
not a generic options dict. Excel gets the most coverage, since vanilla
|
|
85
|
+
Spark has no native Excel reader and a raw file path alone doesn't tell it
|
|
86
|
+
which sheet to read, where the header starts, or whether the workbook is
|
|
87
|
+
password-protected:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from dashingest import ExcelReaderOptions, VolumeSource
|
|
91
|
+
|
|
92
|
+
source = VolumeSource(
|
|
93
|
+
catalog="main", schema_name="bronze", volume="landing",
|
|
94
|
+
path="regional_sales.xlsx",
|
|
95
|
+
reader_options=ExcelReaderOptions(
|
|
96
|
+
sheet_name="Q1 Actuals",
|
|
97
|
+
header_row=2, # skips two title/banner rows above the header
|
|
98
|
+
workbook_password="secret", # optional
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Set `sheet_names=["Jan", "Feb", "Mar"]` instead of `sheet_name` to read and
|
|
104
|
+
stack several same-shaped sheets into one DataFrame — the common "one tab
|
|
105
|
+
per month" spreadsheet layout.
|
|
106
|
+
|
|
107
|
+
`CsvReaderOptions` (delimiter, quote/escape chars, encoding, null markers,
|
|
108
|
+
date/timestamp formats, parse mode), `JsonReaderOptions`,
|
|
109
|
+
`ParquetReaderOptions`/`OrcReaderOptions` (schema merging), and
|
|
110
|
+
`TextReaderOptions` are also available — pass any of them via
|
|
111
|
+
`reader_options=` on a source.
|
|
112
|
+
|
|
113
|
+
## Write modes
|
|
114
|
+
|
|
115
|
+
`append` · `overwrite` · `merge` (upsert into Delta by `merge_keys`, with
|
|
116
|
+
schema evolution where the runtime supports it).
|
|
117
|
+
|
|
118
|
+
## Test Connection & Preview
|
|
119
|
+
|
|
120
|
+
Both the UI and the API let you check a source before committing to a full
|
|
121
|
+
run — the same pattern ADF's linked-service "Test Connection" and
|
|
122
|
+
dataset "Data preview" use:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from dashingest import test_connection, preview
|
|
126
|
+
|
|
127
|
+
test_connection(source).display() # reachability/credentials check, no data read
|
|
128
|
+
preview(source, limit=10) # pandas DataFrame of the first N rows
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
`test_connection` runs a lightweight check per source kind: `SELECT 1` for
|
|
132
|
+
databases, an HTTP request for REST APIs, a filesystem existence check for
|
|
133
|
+
Volumes/ADLS/S3/DBFS (no `dbutils` needed — it uses Spark's Hadoop
|
|
134
|
+
filesystem API directly, so it works the same way across all of them).
|
|
135
|
+
|
|
136
|
+
## Advanced database & REST options
|
|
137
|
+
|
|
138
|
+
`DatabaseSource` supports SSL, JDBC fetch size, parallel reads (split a
|
|
139
|
+
large table by `partition_column` across `num_partitions`), and a raw
|
|
140
|
+
`connection_properties` escape hatch:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from dashingest import DatabaseSource
|
|
144
|
+
|
|
145
|
+
source = DatabaseSource(
|
|
146
|
+
engine="postgresql", host="db.internal", database="analytics",
|
|
147
|
+
table="events", user="svc", password="...",
|
|
148
|
+
ssl=True, num_partitions=8, partition_column="id",
|
|
149
|
+
lower_bound=0, upper_bound=10_000_000,
|
|
150
|
+
)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
`RestApiSource` supports auth (`bearer` / `api_key` / `basic`) and
|
|
154
|
+
pagination (`page_param` or `cursor`-based, up to `max_pages`):
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from dashingest import RestApiSource
|
|
158
|
+
|
|
159
|
+
source = RestApiSource(
|
|
160
|
+
url="https://api.example.com/records",
|
|
161
|
+
auth_type="bearer", bearer_token="...",
|
|
162
|
+
pagination="cursor", cursor_json_path="meta.next_cursor", max_pages=50,
|
|
163
|
+
)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Part of Dashlibs
|
|
167
|
+
|
|
168
|
+
| Library | Purpose |
|
|
169
|
+
|---|---|
|
|
170
|
+
| dash-dq | Data Quality |
|
|
171
|
+
| dash-synthetic | Synthetic Data Generation |
|
|
172
|
+
| dash-ml | ML Lifecycle Management |
|
|
173
|
+
| dash-ingest | Data Ingestion |
|
|
174
|
+
| dash-gov | Data Governance |
|
|
175
|
+
| dash-ontology | Ontology & Lineage for AI |
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
Apache 2.0
|