dcmassist 0.1.15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcmassist-0.1.15/.github/workflows/ci.yml +155 -0
- dcmassist-0.1.15/.gitignore +224 -0
- dcmassist-0.1.15/.pre-commit-config.yaml +17 -0
- dcmassist-0.1.15/.python-version +1 -0
- dcmassist-0.1.15/CLAUDE.md +75 -0
- dcmassist-0.1.15/CONTRIBUTING.md +132 -0
- dcmassist-0.1.15/LICENSE +21 -0
- dcmassist-0.1.15/PKG-INFO +146 -0
- dcmassist-0.1.15/README.md +115 -0
- dcmassist-0.1.15/docs/superpowers/plans/2026-05-23-dcmexporter.md +3403 -0
- dcmassist-0.1.15/docs/superpowers/plans/2026-05-23-real-snowflake-discovery-fixes.md +1141 -0
- dcmassist-0.1.15/docs/superpowers/plans/2026-05-24-chunk-definitions-into-numbered-files.md +901 -0
- dcmassist-0.1.15/docs/superpowers/plans/2026-05-24-rich-status-dashboard.md +640 -0
- dcmassist-0.1.15/docs/superpowers/specs/2026-05-23-dcmexporter-design.md +311 -0
- dcmassist-0.1.15/pyproject.toml +57 -0
- dcmassist-0.1.15/src/dcmassist/__init__.py +3 -0
- dcmassist-0.1.15/src/dcmassist/__main__.py +13 -0
- dcmassist-0.1.15/src/dcmassist/chunking.py +56 -0
- dcmassist-0.1.15/src/dcmassist/cli.py +158 -0
- dcmassist-0.1.15/src/dcmassist/config.py +91 -0
- dcmassist-0.1.15/src/dcmassist/connection.py +75 -0
- dcmassist-0.1.15/src/dcmassist/log.py +48 -0
- dcmassist-0.1.15/src/dcmassist/makefile.py +31 -0
- dcmassist-0.1.15/src/dcmassist/manifest.py +97 -0
- dcmassist-0.1.15/src/dcmassist/objects/__init__.py +25 -0
- dcmassist-0.1.15/src/dcmassist/objects/_base.py +161 -0
- dcmassist-0.1.15/src/dcmassist/objects/_schema_ddl.py +58 -0
- dcmassist-0.1.15/src/dcmassist/objects/_show_paging.py +58 -0
- dcmassist-0.1.15/src/dcmassist/objects/_stage_ddl.py +81 -0
- dcmassist-0.1.15/src/dcmassist/objects/_unimplemented.py +44 -0
- dcmassist-0.1.15/src/dcmassist/objects/file_format.py +32 -0
- dcmassist-0.1.15/src/dcmassist/objects/schema.py +78 -0
- dcmassist-0.1.15/src/dcmassist/objects/sequence.py +30 -0
- dcmassist-0.1.15/src/dcmassist/objects/stage.py +45 -0
- dcmassist-0.1.15/src/dcmassist/objects/table.py +39 -0
- dcmassist-0.1.15/src/dcmassist/objects/tag.py +32 -0
- dcmassist-0.1.15/src/dcmassist/objects/view.py +30 -0
- dcmassist-0.1.15/src/dcmassist/orchestrator.py +208 -0
- dcmassist-0.1.15/src/dcmassist/plugin.py +61 -0
- dcmassist-0.1.15/src/dcmassist/render.py +68 -0
- dcmassist-0.1.15/src/dcmassist/rewrite.py +108 -0
- dcmassist-0.1.15/src/dcmassist/status.py +147 -0
- dcmassist-0.1.15/src/dcmassist/types.py +74 -0
- dcmassist-0.1.15/tests/__init__.py +0 -0
- dcmassist-0.1.15/tests/conftest.py +22 -0
- dcmassist-0.1.15/tests/test_chunking.py +91 -0
- dcmassist-0.1.15/tests/test_cli.py +212 -0
- dcmassist-0.1.15/tests/test_config.py +118 -0
- dcmassist-0.1.15/tests/test_connection.py +79 -0
- dcmassist-0.1.15/tests/test_log.py +40 -0
- dcmassist-0.1.15/tests/test_makefile.py +75 -0
- dcmassist-0.1.15/tests/test_manifest.py +107 -0
- dcmassist-0.1.15/tests/test_objects/__init__.py +0 -0
- dcmassist-0.1.15/tests/test_objects/test_base_helpers.py +105 -0
- dcmassist-0.1.15/tests/test_objects/test_file_format.py +85 -0
- dcmassist-0.1.15/tests/test_objects/test_registry.py +20 -0
- dcmassist-0.1.15/tests/test_objects/test_schema.py +93 -0
- dcmassist-0.1.15/tests/test_objects/test_schema_ddl.py +81 -0
- dcmassist-0.1.15/tests/test_objects/test_sequence.py +85 -0
- dcmassist-0.1.15/tests/test_objects/test_show_paging.py +114 -0
- dcmassist-0.1.15/tests/test_objects/test_stage.py +112 -0
- dcmassist-0.1.15/tests/test_objects/test_stage_ddl.py +69 -0
- dcmassist-0.1.15/tests/test_objects/test_table.py +83 -0
- dcmassist-0.1.15/tests/test_objects/test_tag.py +83 -0
- dcmassist-0.1.15/tests/test_objects/test_view.py +83 -0
- dcmassist-0.1.15/tests/test_orchestrator.py +298 -0
- dcmassist-0.1.15/tests/test_plugin.py +72 -0
- dcmassist-0.1.15/tests/test_render.py +87 -0
- dcmassist-0.1.15/tests/test_rewrite.py +117 -0
- dcmassist-0.1.15/tests/test_status.py +99 -0
- dcmassist-0.1.15/tests/test_types.py +81 -0
- dcmassist-0.1.15/uv.lock +1202 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
inputs:
|
|
10
|
+
release_notes:
|
|
11
|
+
description: "Text to prepend to the GitHub release notes"
|
|
12
|
+
required: false
|
|
13
|
+
default: ""
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
test:
|
|
17
|
+
strategy:
|
|
18
|
+
fail-fast: false
|
|
19
|
+
matrix:
|
|
20
|
+
os: [ubuntu-latest, macos-latest, windows-latest]
|
|
21
|
+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
22
|
+
runs-on: ${{ matrix.os }}
|
|
23
|
+
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v6
|
|
26
|
+
|
|
27
|
+
- name: Install uv
|
|
28
|
+
uses: astral-sh/setup-uv@v8.1.0
|
|
29
|
+
with:
|
|
30
|
+
enable-cache: true
|
|
31
|
+
|
|
32
|
+
- name: Set up Python
|
|
33
|
+
run: uv python install ${{ matrix.python-version }}
|
|
34
|
+
|
|
35
|
+
- name: Install dependencies
|
|
36
|
+
run: uv sync --all-extras --python ${{ matrix.python-version }}
|
|
37
|
+
|
|
38
|
+
- name: Ruff format
|
|
39
|
+
run: uv run --python ${{ matrix.python-version }} ruff format --check src/ tests/
|
|
40
|
+
|
|
41
|
+
- name: Ruff check
|
|
42
|
+
run: uv run --python ${{ matrix.python-version }} ruff check src/ tests/
|
|
43
|
+
|
|
44
|
+
- name: Run tests
|
|
45
|
+
run: uv run --python ${{ matrix.python-version }} pytest tests/ -v
|
|
46
|
+
|
|
47
|
+
build:
|
|
48
|
+
needs: test
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
outputs:
|
|
51
|
+
version: ${{ steps.stamp.outputs.version }}
|
|
52
|
+
|
|
53
|
+
steps:
|
|
54
|
+
- uses: actions/checkout@v6
|
|
55
|
+
|
|
56
|
+
- name: Install uv
|
|
57
|
+
uses: astral-sh/setup-uv@v8.1.0
|
|
58
|
+
with:
|
|
59
|
+
enable-cache: true
|
|
60
|
+
|
|
61
|
+
- name: Set up Python
|
|
62
|
+
run: uv python install 3.12
|
|
63
|
+
|
|
64
|
+
- name: Install dependencies
|
|
65
|
+
run: uv sync --all-extras
|
|
66
|
+
|
|
67
|
+
- name: Stamp version
|
|
68
|
+
id: stamp
|
|
69
|
+
env:
|
|
70
|
+
RUN_NUMBER: ${{ github.run_number }}
|
|
71
|
+
run: |
|
|
72
|
+
python - <<'PY'
|
|
73
|
+
import os, re, pathlib
|
|
74
|
+
path = pathlib.Path("pyproject.toml")
|
|
75
|
+
text = path.read_text()
|
|
76
|
+
match = re.search(r'^version = "(\d+\.\d+)"', text, re.MULTILINE)
|
|
77
|
+
if not match:
|
|
78
|
+
raise SystemExit("pyproject.toml must contain 'version = \"MAJOR.MINOR\"'")
|
|
79
|
+
base = match.group(1)
|
|
80
|
+
full = f'{base}.{os.environ["RUN_NUMBER"]}'
|
|
81
|
+
path.write_text(
|
|
82
|
+
re.sub(
|
|
83
|
+
r'^version = "\d+\.\d+"',
|
|
84
|
+
f'version = "{full}"',
|
|
85
|
+
text,
|
|
86
|
+
count=1,
|
|
87
|
+
flags=re.MULTILINE,
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
print(f"Stamped version: {full}")
|
|
91
|
+
with open(os.environ["GITHUB_OUTPUT"], "a") as fh:
|
|
92
|
+
fh.write(f"version={full}\n")
|
|
93
|
+
PY
|
|
94
|
+
|
|
95
|
+
- name: Build sdist and wheel
|
|
96
|
+
run: uv build
|
|
97
|
+
|
|
98
|
+
- name: Upload build artifacts
|
|
99
|
+
uses: actions/upload-artifact@v7
|
|
100
|
+
with:
|
|
101
|
+
name: dist
|
|
102
|
+
path: dist/
|
|
103
|
+
if-no-files-found: error
|
|
104
|
+
|
|
105
|
+
publish:
|
|
106
|
+
needs: build
|
|
107
|
+
if: github.event_name == 'workflow_dispatch'
|
|
108
|
+
runs-on: ubuntu-latest
|
|
109
|
+
environment: pypi
|
|
110
|
+
permissions:
|
|
111
|
+
id-token: write
|
|
112
|
+
|
|
113
|
+
steps:
|
|
114
|
+
- name: Download build artifacts
|
|
115
|
+
uses: actions/download-artifact@v8
|
|
116
|
+
with:
|
|
117
|
+
name: dist
|
|
118
|
+
path: dist/
|
|
119
|
+
|
|
120
|
+
- name: Publish to PyPI
|
|
121
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
122
|
+
|
|
123
|
+
release:
|
|
124
|
+
needs: [build, publish]
|
|
125
|
+
if: github.event_name == 'workflow_dispatch'
|
|
126
|
+
runs-on: ubuntu-latest
|
|
127
|
+
permissions:
|
|
128
|
+
contents: write
|
|
129
|
+
|
|
130
|
+
steps:
|
|
131
|
+
- uses: actions/checkout@v6
|
|
132
|
+
|
|
133
|
+
- name: Download build artifacts
|
|
134
|
+
uses: actions/download-artifact@v8
|
|
135
|
+
with:
|
|
136
|
+
name: dist
|
|
137
|
+
path: dist/
|
|
138
|
+
|
|
139
|
+
- name: Create GitHub release
|
|
140
|
+
env:
|
|
141
|
+
GH_TOKEN: ${{ github.token }}
|
|
142
|
+
VERSION: ${{ needs.build.outputs.version }}
|
|
143
|
+
RELEASE_NOTES: ${{ inputs.release_notes }}
|
|
144
|
+
run: |
|
|
145
|
+
pypi_line="Published to PyPI: https://pypi.org/project/dcmassist/${VERSION}/"
|
|
146
|
+
if [ -n "${RELEASE_NOTES}" ]; then
|
|
147
|
+
notes="${RELEASE_NOTES}"$'\n\n'"${pypi_line}"
|
|
148
|
+
else
|
|
149
|
+
notes="${pypi_line}"
|
|
150
|
+
fi
|
|
151
|
+
gh release create "v${VERSION}" \
|
|
152
|
+
--target "${GITHUB_SHA}" \
|
|
153
|
+
--title "v${VERSION}" \
|
|
154
|
+
--notes "${notes}" \
|
|
155
|
+
dist/*
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
# Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
# poetry.lock
|
|
109
|
+
# poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
# pdm.lock
|
|
116
|
+
# pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
# pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# Redis
|
|
135
|
+
*.rdb
|
|
136
|
+
*.aof
|
|
137
|
+
*.pid
|
|
138
|
+
|
|
139
|
+
# RabbitMQ
|
|
140
|
+
mnesia/
|
|
141
|
+
rabbitmq/
|
|
142
|
+
rabbitmq-data/
|
|
143
|
+
|
|
144
|
+
# ActiveMQ
|
|
145
|
+
activemq-data/
|
|
146
|
+
|
|
147
|
+
# SageMath parsed files
|
|
148
|
+
*.sage.py
|
|
149
|
+
|
|
150
|
+
# Environments
|
|
151
|
+
.env
|
|
152
|
+
.envrc
|
|
153
|
+
.venv
|
|
154
|
+
env/
|
|
155
|
+
venv/
|
|
156
|
+
ENV/
|
|
157
|
+
env.bak/
|
|
158
|
+
venv.bak/
|
|
159
|
+
|
|
160
|
+
# Spyder project settings
|
|
161
|
+
.spyderproject
|
|
162
|
+
.spyproject
|
|
163
|
+
|
|
164
|
+
# Rope project settings
|
|
165
|
+
.ropeproject
|
|
166
|
+
|
|
167
|
+
# mkdocs documentation
|
|
168
|
+
/site
|
|
169
|
+
|
|
170
|
+
# mypy
|
|
171
|
+
.mypy_cache/
|
|
172
|
+
.dmypy.json
|
|
173
|
+
dmypy.json
|
|
174
|
+
|
|
175
|
+
# Pyre type checker
|
|
176
|
+
.pyre/
|
|
177
|
+
|
|
178
|
+
# pytype static type analyzer
|
|
179
|
+
.pytype/
|
|
180
|
+
|
|
181
|
+
# Cython debug symbols
|
|
182
|
+
cython_debug/
|
|
183
|
+
|
|
184
|
+
# PyCharm
|
|
185
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
186
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
188
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
|
+
# .idea/
|
|
190
|
+
|
|
191
|
+
# Abstra
|
|
192
|
+
# Abstra is an AI-powered process automation framework.
|
|
193
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
194
|
+
# Learn more at https://abstra.io/docs
|
|
195
|
+
.abstra/
|
|
196
|
+
|
|
197
|
+
# Visual Studio Code
|
|
198
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
199
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
200
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
201
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
202
|
+
# .vscode/
|
|
203
|
+
# Temporary file for partial code execution
|
|
204
|
+
tempCodeRunnerFile.py
|
|
205
|
+
|
|
206
|
+
# Ruff stuff:
|
|
207
|
+
.ruff_cache/
|
|
208
|
+
|
|
209
|
+
# PyPI configuration file
|
|
210
|
+
.pypirc
|
|
211
|
+
|
|
212
|
+
# Marimo
|
|
213
|
+
marimo/_static/
|
|
214
|
+
marimo/_lsp/
|
|
215
|
+
__marimo__/
|
|
216
|
+
|
|
217
|
+
# Streamlit
|
|
218
|
+
.streamlit/secrets.toml
|
|
219
|
+
|
|
220
|
+
# dcmassist default output folder
|
|
221
|
+
out/
|
|
222
|
+
|
|
223
|
+
# Claude Code local settings
|
|
224
|
+
.claude/
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
+
rev: v0.15.11
|
|
4
|
+
hooks:
|
|
5
|
+
- id: ruff-format
|
|
6
|
+
args: [src/, tests/]
|
|
7
|
+
- id: ruff
|
|
8
|
+
args: [--fix, src/, tests/]
|
|
9
|
+
|
|
10
|
+
- repo: local
|
|
11
|
+
hooks:
|
|
12
|
+
- id: mypy
|
|
13
|
+
name: mypy
|
|
14
|
+
entry: uv run mypy src/
|
|
15
|
+
language: system
|
|
16
|
+
types: [python]
|
|
17
|
+
pass_filenames: false
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Project
|
|
6
|
+
|
|
7
|
+
`dcmassist` is a CLI that exports Snowflake object definitions (tables, views, schemas, stages, …) into the on-disk layout expected by a Snowflake DCM (Declarative Change Management) project: `manifest.yml`, `Makefile`, `sources/definitions/<type>.sql`, and optionally `sources/macros/<type>.sql`.
|
|
8
|
+
|
|
9
|
+
The exported DDL is committed to a DCM project (i.e. git). Stage DDL is synthesized from `DESC STAGE` and **must never include AWS credentials** — credentials belong to the storage integration, not the stage.
|
|
10
|
+
|
|
11
|
+
## Commands
|
|
12
|
+
|
|
13
|
+
The project uses `uv` for everything; do not invoke `pip` or `python` directly.
|
|
14
|
+
|
|
15
|
+
| Task | Command |
|
|
16
|
+
|------|---------|
|
|
17
|
+
| Run the CLI | `uv run dcmassist export --database <DB> [--force]` |
|
|
18
|
+
| Run all tests | `uv run pytest` |
|
|
19
|
+
| Run one file | `uv run pytest tests/test_orchestrator.py -v` |
|
|
20
|
+
| Run one test | `uv run pytest tests/test_orchestrator.py::test_export_writes_full_layout -v` |
|
|
21
|
+
| Lint | `uv run ruff check src tests` |
|
|
22
|
+
| Format check | `uv run ruff format --check src tests` |
|
|
23
|
+
| Type check | `uv run mypy src` |
|
|
24
|
+
|
|
25
|
+
A pre-commit hook runs `ruff format`, `ruff` (with `--fix`), and `mypy src` across the whole tree on every commit. **Never bypass it with `--no-verify`** — if a hook fails, fix the underlying issue. If a refactor breaks a different file's call site, that's a sign two changes need to land in one commit (this has happened before with the orchestrator/render and orchestrator/status pairs).
|
|
26
|
+
|
|
27
|
+
The hook does NOT run pytest — run it manually before committing.
|
|
28
|
+
|
|
29
|
+
## Architecture
|
|
30
|
+
|
|
31
|
+
### Pipeline (orchestrator-driven)
|
|
32
|
+
|
|
33
|
+
`cli.py` builds a frozen `Config`, then `orchestrator.export(cfg)` runs the entire pipeline:
|
|
34
|
+
|
|
35
|
+
1. `resolve_objects_per_file()` — read `DCMASSIST_EXPORT_OBJECTS_PER_FILE` BEFORE touching the output folder (a bad value must not blow away the user's directory).
|
|
36
|
+
2. `prepare_out_folder()` — refuse non-empty unless `--force`; clear and recreate.
|
|
37
|
+
3. Open `RunLog` at `<out>/dcmassist.log` and dump every Config field so users see (and discover) available options.
|
|
38
|
+
4. Inside a `StatusDashboard` (Rich `Live` panel), connect to Snowflake and iterate `filter_types(cfg)`.
|
|
39
|
+
5. For each type, dispatch to its plugin: `discover()` → per-FQN `get_ddl()` → `to_define_and_invocation()`.
|
|
40
|
+
6. Chunk per-type blocks via `chunk_blocks()` into `dict[filename, body]` keyed by FULL filename (`table.sql`, `table2.sql`, …). The first chunk keeps the unsuffixed name so small exports look identical to before chunking existed.
|
|
41
|
+
7. `write_outputs()` writes the dict to disk. It is mechanical — naming logic lives in the orchestrator; macros are still keyed by slug.
|
|
42
|
+
|
|
43
|
+
Exit codes: `0` ok, `4` exported nothing but had errors, `5` config/setup error (bad env var or non-empty out folder).
|
|
44
|
+
|
|
45
|
+
### Plugin system (`src/dcmassist/objects/`)
|
|
46
|
+
|
|
47
|
+
Plugins are auto-discovered: `objects/__init__.py:build_registry()` imports every non-underscore module and calls `registry.register(module.plugin)`. `_unimplemented.py` is a special case that registers a list of stub plugins for types DCM supports but dcmassist doesn't yet handle.
|
|
48
|
+
|
|
49
|
+
To add a new v1 type: create `objects/<type>.py`, subclass `V1ObjectPlugin` from `_base.py`, set `type_name`, `file_slug`, `SHOW_FORM`, `GET_DDL_TYPE`, `MACRO_BODY`, instantiate as module-level `plugin`. Add the canonical name to `V1_TYPES` in `types.py`.
|
|
50
|
+
|
|
51
|
+
`V1ObjectPlugin` (`objects/_base.py`) implements the standard recipe: schema-iterated `SHOW` discovery, `GET_DDL(<TYPE>, <quoted_fqn>, TRUE)`, and a rewrite chain (`create_to_define` → `inject_comment_if_missing` → `parameterise_database`). Override only what's special. `Schema` and `Stage` synthesize their DDL directly from `SHOW`/`DESC STAGE` rows (see `_schema_ddl.py`, `_stage_ddl.py`) because `GET_DDL` either hangs (recursive Schema) or is unsupported (Stage).
|
|
52
|
+
|
|
53
|
+
### Snowflake quirks worth knowing
|
|
54
|
+
|
|
55
|
+
- `SHOW … IN DATABASE` is capped at 10K rows. Use `paginated_show()` from `objects/_show_paging.py`, which appends `LIMIT N FROM '<last_name>'`.
|
|
56
|
+
- The cursor is forced to `DictCursor` in `connection.py`; `_rows_to_fqns` rejects non-dict rows defensively.
|
|
57
|
+
- Identifiers must be quoted in `GET_DDL`/`DESC STAGE` (see `FQN.quoted` in `types.py`) — leading-digit and lowercase names break otherwise.
|
|
58
|
+
- `GET_DDL(..., TRUE)` returns fully-qualified names; `parameterise_database` then rewrites the literal database to `{{ database }}` so the same export can be run against multiple environments.
|
|
59
|
+
- The error string `002003 / "does not exist or not authorized"` is treated as a non-fatal `skipped_missing` count — common in shared databases where the role lacks privileges on a few objects.
|
|
60
|
+
- `Database` is intentionally NOT in `V1_TYPES` — DCM rejects `DEFINE DATABASE` for the project's parent database. The Makefile prints an analogous warning for the parent schema (which the exporter can't auto-detect).
|
|
61
|
+
|
|
62
|
+
### Status & logging split
|
|
63
|
+
|
|
64
|
+
- `StatusDashboard` (`status.py`) — Rich `Live` panel on stderr, transient (disappears at end). Silent on non-TTY, `NO_COLOR=1`, or `DCMASSIST_NO_STATUS=1`. `.log()` writes above the panel; falls back to plain stderr when disabled so warnings still surface in CI.
|
|
65
|
+
- `RunLog` (`log.py`) — per-run timestamped log file at `<out-folder>/dcmassist.log`. Truncated on each run. This is what users tail when something goes wrong.
|
|
66
|
+
|
|
67
|
+
The dashboard answers "what's happening right now"; the log answers "what happened to which object and why".
|
|
68
|
+
|
|
69
|
+
## Conventions
|
|
70
|
+
|
|
71
|
+
- Default to no comments. Add one only when the **why** is non-obvious (a Snowflake quirk, a hidden invariant, a workaround for a specific error). Don't comment what well-named code already shows.
|
|
72
|
+
- Module docstrings should explain why the module exists, not list its contents.
|
|
73
|
+
- Don't add backwards-compatibility shims, dead defensive code, or feature flags for hypothetical futures. The codebase is pre-1.0 and prefers clean breaks over migration scaffolding.
|
|
74
|
+
- Plans and specs live under `docs/superpowers/plans/` and `docs/superpowers/specs/` and are written via the `superpowers:writing-plans` and `superpowers:brainstorming` skills.
|
|
75
|
+
- The tool was renamed from `dcmexporter` → `dcmassist`. Older plans/specs under `docs/superpowers/` still use the old name (they're snapshots) — don't update them retroactively.
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# Contributing to dcmassist
|
|
2
|
+
|
|
3
|
+
Thanks for your interest in contributing. This document covers everything you need to make a clean, mergeable change.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
Requires Python ≥ 3.10 and [`uv`](https://docs.astral.sh/uv/).
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
git clone https://github.com/jamiekt/dcmassist
|
|
11
|
+
cd dcmassist
|
|
12
|
+
uv sync # installs runtime + dev deps + the package itself, editable
|
|
13
|
+
uv run pre-commit install # wires the pre-commit hook
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
`uv` is the only supported tool — please don't introduce `pip`, `python -m venv`, or `poetry` into the workflow.
|
|
17
|
+
|
|
18
|
+
## Common commands
|
|
19
|
+
|
|
20
|
+
| Task | Command |
|
|
21
|
+
|------|---------|
|
|
22
|
+
| Run the CLI | `uv run dcmassist export --database <DB>` |
|
|
23
|
+
| Run all tests | `uv run pytest` |
|
|
24
|
+
| Run one file | `uv run pytest tests/test_orchestrator.py -v` |
|
|
25
|
+
| Run one test | `uv run pytest tests/test_orchestrator.py::test_export_writes_full_layout -v` |
|
|
26
|
+
| Lint | `uv run ruff check src tests` |
|
|
27
|
+
| Format check | `uv run ruff format --check src tests` |
|
|
28
|
+
| Auto-fix | `uv run ruff check --fix src tests && uv run ruff format src tests` |
|
|
29
|
+
| Type check | `uv run mypy src` |
|
|
30
|
+
|
|
31
|
+
## Pre-commit hook
|
|
32
|
+
|
|
33
|
+
The pre-commit hook runs `ruff format`, `ruff --fix`, and `mypy src` across the whole tree on every commit.
|
|
34
|
+
|
|
35
|
+
- **Never bypass it with `--no-verify`.** If the hook fails, fix the underlying issue.
|
|
36
|
+
- The hook does **not** run pytest. Run `uv run pytest` manually before pushing.
|
|
37
|
+
- If a refactor breaks a different file's call site, that's a sign two changes need to land in one commit.
|
|
38
|
+
|
|
39
|
+
## Workflow
|
|
40
|
+
|
|
41
|
+
1. Open an issue describing what you want to change (skip for small fixes).
|
|
42
|
+
2. Branch from `main`.
|
|
43
|
+
3. TDD where it makes sense — write the test, watch it fail, write the code, watch it pass.
|
|
44
|
+
4. Keep commits focused. Frequent small commits beat one giant one.
|
|
45
|
+
5. Run the full test suite (`uv run pytest`) before pushing.
|
|
46
|
+
6. Open a PR. Reference any related issue.
|
|
47
|
+
|
|
48
|
+
## Code conventions
|
|
49
|
+
|
|
50
|
+
These are enforced by review (and partly by tooling):
|
|
51
|
+
|
|
52
|
+
- **Default to no comments.** Add one only when the *why* is non-obvious — a Snowflake quirk, a hidden invariant, a workaround for a specific error. Don't comment what well-named code already shows.
|
|
53
|
+
- **Module docstrings explain why the module exists**, not what it contains.
|
|
54
|
+
- **No backwards-compatibility shims, dead defensive code, or feature flags for hypothetical futures.** The codebase is pre-1.0 and prefers clean breaks over migration scaffolding.
|
|
55
|
+
- **No mocking the database in tests that exercise the orchestrator end-to-end.** Use the cursor-mocking pattern in `tests/test_orchestrator.py`.
|
|
56
|
+
- **Quote Snowflake identifiers** in any DDL you generate — leading-digit and lowercase names break otherwise.
|
|
57
|
+
- **Never include credentials in synthesized DDL**, especially stage DDL. Credentials belong on storage integrations.
|
|
58
|
+
|
|
59
|
+
## Architecture overview
|
|
60
|
+
|
|
61
|
+
### Pipeline
|
|
62
|
+
|
|
63
|
+
`cli.py` builds a frozen `Config`, then `orchestrator.export(cfg)` runs the entire pipeline:
|
|
64
|
+
|
|
65
|
+
1. `resolve_objects_per_file()` — read `DCMASSIST_EXPORT_OBJECTS_PER_FILE` *before* touching the output folder (a bad value must not blow away the user's directory).
|
|
66
|
+
2. `prepare_out_folder()` — refuse non-empty unless `--force`; clear and recreate.
|
|
67
|
+
3. Open `RunLog` at `<out>/dcmassist.log` and dump every Config field so users see (and discover) available options.
|
|
68
|
+
4. Inside a `StatusDashboard` (Rich `Live` panel), connect to Snowflake and iterate `filter_types(cfg)`.
|
|
69
|
+
5. For each type, dispatch to its plugin: `discover()` → per-FQN `get_ddl()` → `to_define_and_invocation()`.
|
|
70
|
+
6. Chunk per-type blocks via `chunk_blocks()` into `dict[filename, body]` keyed by full filename (`table.sql`, `table2.sql`, …). The first chunk keeps the unsuffixed name so small exports look identical to before chunking existed.
|
|
71
|
+
7. `write_outputs()` writes the dict to disk. It is mechanical — naming logic lives in the orchestrator.
|
|
72
|
+
|
|
73
|
+
Exit codes: `0` ok, `4` exported nothing but had errors, `5` config/setup error.
|
|
74
|
+
|
|
75
|
+
### Plugin system
|
|
76
|
+
|
|
77
|
+
Plugins live in `src/dcmassist/objects/` and are auto-discovered: `objects/__init__.py:build_registry()` imports every non-underscore module and calls `registry.register(module.plugin)`.
|
|
78
|
+
|
|
79
|
+
To add a new v1 type:
|
|
80
|
+
|
|
81
|
+
1. Create `src/dcmassist/objects/<type>.py`.
|
|
82
|
+
2. Subclass `V1ObjectPlugin` from `_base.py`.
|
|
83
|
+
3. Set `type_name`, `file_slug`, `SHOW_FORM`, `GET_DDL_TYPE`, `MACRO_BODY`.
|
|
84
|
+
4. Instantiate as a module-level `plugin`.
|
|
85
|
+
5. Add the canonical name to `V1_TYPES` in `types.py`.
|
|
86
|
+
6. Write tests in `tests/test_objects/test_<type>.py`.
|
|
87
|
+
|
|
88
|
+
`V1ObjectPlugin` implements the standard recipe: schema-iterated `SHOW` discovery, `GET_DDL(<TYPE>, <quoted_fqn>, TRUE)`, and a rewrite chain (`create_to_define` → `inject_comment_if_missing` → `parameterise_database`). Override only what's special.
|
|
89
|
+
|
|
90
|
+
`Schema` and `Stage` synthesize their DDL directly from `SHOW`/`DESC STAGE` rows because `GET_DDL` either hangs (recursive Schema) or is unsupported (Stage).
|
|
91
|
+
|
|
92
|
+
### Snowflake quirks
|
|
93
|
+
|
|
94
|
+
- `SHOW … IN DATABASE` is capped at 10K rows. Use `paginated_show()` from `objects/_show_paging.py`, which appends `LIMIT N FROM '<last_name>'`.
|
|
95
|
+
- The cursor is forced to `DictCursor` in `connection.py`; `_rows_to_fqns` rejects non-dict rows defensively.
|
|
96
|
+
- Identifiers must be quoted in `GET_DDL`/`DESC STAGE` (see `FQN.quoted` in `types.py`).
|
|
97
|
+
- `GET_DDL(..., TRUE)` returns fully-qualified names; `parameterise_database` rewrites the literal database to `{{ database }}`.
|
|
98
|
+
- The error string `002003 / "does not exist or not authorized"` is treated as a non-fatal `skipped_missing` — common in shared databases where the role lacks privileges on a few objects.
|
|
99
|
+
- `Database` is intentionally NOT in `V1_TYPES` — DCM rejects `DEFINE DATABASE` for the project's parent database.
|
|
100
|
+
|
|
101
|
+
### Status & logging split
|
|
102
|
+
|
|
103
|
+
- **`StatusDashboard`** (`status.py`) — Rich `Live` panel on stderr, transient. Silent on non-TTY, `NO_COLOR=1`, or `DCMASSIST_NO_STATUS=1`. `.log()` writes above the panel; falls back to plain stderr when disabled so warnings still surface in CI.
|
|
104
|
+
- **`RunLog`** (`log.py`) — per-run timestamped log file at `<out>/dcmassist.log`. Truncated on each run. This is what users tail when something goes wrong.
|
|
105
|
+
|
|
106
|
+
The dashboard answers "what's happening right now"; the log answers "what happened to which object and why".
|
|
107
|
+
|
|
108
|
+
## Testing notes
|
|
109
|
+
|
|
110
|
+
- Tests in `tests/test_orchestrator.py` mock the Snowflake cursor end-to-end via the pattern in `_cfg()` and `execute_side_effect`. Reuse that pattern for new orchestrator-level tests.
|
|
111
|
+
- Plugin-level tests (`tests/test_objects/test_*.py`) test discovery and rewrite chains in isolation, with no cursor at all where possible.
|
|
112
|
+
- `tmp_path` from pytest is fine for output-folder fixtures.
|
|
113
|
+
- If you're adding a new env var, write a test that sets it via `monkeypatch.setenv` and confirm both the success path and an invalid value.
|
|
114
|
+
|
|
115
|
+
## Plans and specs
|
|
116
|
+
|
|
117
|
+
Larger features start with a spec in `docs/superpowers/specs/` and a plan in `docs/superpowers/plans/`, written via the [superpowers](https://github.com/anthropics/superpowers) `brainstorming` and `writing-plans` skills. You don't need superpowers to contribute — short PRs can skip the doc — but for anything non-trivial please open an issue or draft PR first so the design can be discussed before code lands.
|
|
118
|
+
|
|
119
|
+
## Reporting bugs
|
|
120
|
+
|
|
121
|
+
Open an issue at [github.com/jamiekt/dcmassist/issues](https://github.com/jamiekt/dcmassist/issues) with:
|
|
122
|
+
|
|
123
|
+
- The command you ran.
|
|
124
|
+
- The relevant section of `dcmassist.log`.
|
|
125
|
+
- Snowflake account region (if reproducible only on certain regions).
|
|
126
|
+
- Whether the same export works against a different database / role.
|
|
127
|
+
|
|
128
|
+
Don't paste DDL that contains real customer data or credentials.
|
|
129
|
+
|
|
130
|
+
## License
|
|
131
|
+
|
|
132
|
+
By contributing, you agree your contributions will be licensed under the project's MIT license.
|
dcmassist-0.1.15/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jamie Thomson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|