dataform-dependency-visualizer 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataform_dependency_visualizer-0.1.0.dist-info/METADATA +170 -0
- dataform_dependency_visualizer-0.1.0.dist-info/RECORD +13 -0
- dataform_dependency_visualizer-0.1.0.dist-info/WHEEL +5 -0
- dataform_dependency_visualizer-0.1.0.dist-info/entry_points.txt +2 -0
- dataform_dependency_visualizer-0.1.0.dist-info/licenses/LICENSE +21 -0
- dataform_dependency_visualizer-0.1.0.dist-info/top_level.txt +1 -0
- dataform_viz/__init__.py +14 -0
- dataform_viz/cli.py +159 -0
- dataform_viz/dataform_check.py +175 -0
- dataform_viz/master_index.py +278 -0
- dataform_viz/parser.py +69 -0
- dataform_viz/svg_generator.py +381 -0
- dataform_viz/visualizer.py +140 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Generate a master index.html to view all dependency SVGs from all schemas
|
|
4
|
+
"""
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
def collect_all_svgs():
|
|
9
|
+
"""Collect all SVG files from dependencies_* folders"""
|
|
10
|
+
schemas = {}
|
|
11
|
+
|
|
12
|
+
output_path = Path('output')
|
|
13
|
+
if not output_path.exists():
|
|
14
|
+
output_path = Path('.')
|
|
15
|
+
|
|
16
|
+
for folder in output_path.glob('dependencies_*'):
|
|
17
|
+
if not folder.is_dir():
|
|
18
|
+
continue
|
|
19
|
+
|
|
20
|
+
schema_name = folder.name.replace('dependencies_', '')
|
|
21
|
+
svgs = []
|
|
22
|
+
|
|
23
|
+
for svg_file in sorted(folder.glob('*.svg')):
|
|
24
|
+
table_name = svg_file.stem.replace('_', '.')
|
|
25
|
+
# Use relative path from output folder
|
|
26
|
+
relative_path = f"{folder.name}/{svg_file.name}"
|
|
27
|
+
svgs.append({
|
|
28
|
+
'file': relative_path,
|
|
29
|
+
'name': table_name,
|
|
30
|
+
'display_name': table_name.split('.')[-1] if '.' in table_name else table_name
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
if svgs:
|
|
34
|
+
schemas[schema_name] = {
|
|
35
|
+
'folder': str(folder),
|
|
36
|
+
'svgs': svgs
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return schemas
|
|
40
|
+
|
|
41
|
+
def generate_master_index(schemas):
|
|
42
|
+
"""Generate master index.html"""
|
|
43
|
+
|
|
44
|
+
html_lines = [
|
|
45
|
+
'<!DOCTYPE html>',
|
|
46
|
+
'<html>',
|
|
47
|
+
'<head>',
|
|
48
|
+
' <meta charset="utf-8">',
|
|
49
|
+
' <title>All Dependencies - Master Index</title>',
|
|
50
|
+
' <style>',
|
|
51
|
+
' * { box-sizing: border-box; }',
|
|
52
|
+
' body { ',
|
|
53
|
+
' font-family: Arial, sans-serif; ',
|
|
54
|
+
' margin: 0; ',
|
|
55
|
+
' padding: 0; ',
|
|
56
|
+
' background: #f5f5f5; ',
|
|
57
|
+
' display: flex;',
|
|
58
|
+
' height: 100vh;',
|
|
59
|
+
' }',
|
|
60
|
+
' .sidebar {',
|
|
61
|
+
' width: 300px;',
|
|
62
|
+
' background: white;',
|
|
63
|
+
' border-right: 1px solid #ddd;',
|
|
64
|
+
' overflow-y: auto;',
|
|
65
|
+
' padding: 20px;',
|
|
66
|
+
' }',
|
|
67
|
+
' .content {',
|
|
68
|
+
' flex: 1;',
|
|
69
|
+
' overflow: auto;',
|
|
70
|
+
' padding: 20px;',
|
|
71
|
+
' background: white;',
|
|
72
|
+
' }',
|
|
73
|
+
' h1 { ',
|
|
74
|
+
' color: #333; ',
|
|
75
|
+
' margin: 0 0 20px 0;',
|
|
76
|
+
' font-size: 24px;',
|
|
77
|
+
' }',
|
|
78
|
+
' .schema-section {',
|
|
79
|
+
' margin-bottom: 30px;',
|
|
80
|
+
' }',
|
|
81
|
+
' .schema-title {',
|
|
82
|
+
' font-size: 16px;',
|
|
83
|
+
' font-weight: bold;',
|
|
84
|
+
' color: #1976d2;',
|
|
85
|
+
' margin: 15px 0 10px 0;',
|
|
86
|
+
' padding: 8px;',
|
|
87
|
+
' background: #e3f2fd;',
|
|
88
|
+
' border-radius: 4px;',
|
|
89
|
+
' cursor: pointer;',
|
|
90
|
+
' user-select: none;',
|
|
91
|
+
' }',
|
|
92
|
+
' .schema-title:hover {',
|
|
93
|
+
' background: #bbdefb;',
|
|
94
|
+
' }',
|
|
95
|
+
' .table-list {',
|
|
96
|
+
' list-style: none;',
|
|
97
|
+
' padding: 0;',
|
|
98
|
+
' margin: 0 0 0 15px;',
|
|
99
|
+
' }',
|
|
100
|
+
' .table-list.collapsed {',
|
|
101
|
+
' display: none;',
|
|
102
|
+
' }',
|
|
103
|
+
' .table-item {',
|
|
104
|
+
' padding: 6px 10px;',
|
|
105
|
+
' cursor: pointer;',
|
|
106
|
+
' border-radius: 4px;',
|
|
107
|
+
' margin: 2px 0;',
|
|
108
|
+
' font-size: 13px;',
|
|
109
|
+
' }',
|
|
110
|
+
' .table-item:hover {',
|
|
111
|
+
' background: #f5f5f5;',
|
|
112
|
+
' }',
|
|
113
|
+
' .table-item.active {',
|
|
114
|
+
' background: #1976d2;',
|
|
115
|
+
' color: white;',
|
|
116
|
+
' }',
|
|
117
|
+
' .viewer {',
|
|
118
|
+
' text-align: center;',
|
|
119
|
+
' }',
|
|
120
|
+
' .viewer img {',
|
|
121
|
+
' max-width: 100%;',
|
|
122
|
+
' height: auto;',
|
|
123
|
+
' box-shadow: 0 2px 8px rgba(0,0,0,0.1);',
|
|
124
|
+
' }',
|
|
125
|
+
' .viewer-title {',
|
|
126
|
+
' font-size: 20px;',
|
|
127
|
+
' color: #333;',
|
|
128
|
+
' margin-bottom: 20px;',
|
|
129
|
+
' padding: 15px;',
|
|
130
|
+
' background: #f5f5f5;',
|
|
131
|
+
' border-radius: 4px;',
|
|
132
|
+
' }',
|
|
133
|
+
' .viewer-subtitle {',
|
|
134
|
+
' font-size: 14px;',
|
|
135
|
+
' color: #666;',
|
|
136
|
+
' margin-top: 5px;',
|
|
137
|
+
' }',
|
|
138
|
+
' .empty-state {',
|
|
139
|
+
' text-align: center;',
|
|
140
|
+
' padding: 100px 20px;',
|
|
141
|
+
' color: #999;',
|
|
142
|
+
' }',
|
|
143
|
+
' .stats {',
|
|
144
|
+
' padding: 15px;',
|
|
145
|
+
' background: #f5f5f5;',
|
|
146
|
+
' border-radius: 4px;',
|
|
147
|
+
' margin-bottom: 20px;',
|
|
148
|
+
' font-size: 13px;',
|
|
149
|
+
' color: #666;',
|
|
150
|
+
' }',
|
|
151
|
+
' .collapse-icon {',
|
|
152
|
+
' float: right;',
|
|
153
|
+
' font-size: 12px;',
|
|
154
|
+
' }',
|
|
155
|
+
' </style>',
|
|
156
|
+
'</head>',
|
|
157
|
+
'<body>',
|
|
158
|
+
' <div class="sidebar">',
|
|
159
|
+
' <h1>Table Dependencies</h1>',
|
|
160
|
+
' <div class="stats">',
|
|
161
|
+
]
|
|
162
|
+
|
|
163
|
+
# Count totals
|
|
164
|
+
total_schemas = len(schemas)
|
|
165
|
+
total_tables = sum(len(s['svgs']) for s in schemas.values())
|
|
166
|
+
|
|
167
|
+
html_lines.append(f' <strong>{total_schemas}</strong> schemas<br>')
|
|
168
|
+
html_lines.append(f' <strong>{total_tables}</strong> tables')
|
|
169
|
+
html_lines.append(' </div>')
|
|
170
|
+
|
|
171
|
+
# Generate schema sections
|
|
172
|
+
for schema_name, schema_data in sorted(schemas.items()):
|
|
173
|
+
html_lines.append(f' <div class="schema-section">')
|
|
174
|
+
html_lines.append(f' <div class="schema-title" onclick="toggleSchema(\'{schema_name}\')">')
|
|
175
|
+
html_lines.append(f' {schema_name}')
|
|
176
|
+
html_lines.append(f' <span class="collapse-icon" id="icon-{schema_name}">▼</span>')
|
|
177
|
+
html_lines.append(f' </div>')
|
|
178
|
+
html_lines.append(f' <ul class="table-list" id="list-{schema_name}">')
|
|
179
|
+
|
|
180
|
+
for svg in schema_data['svgs']:
|
|
181
|
+
safe_id = svg['name'].replace('.', '_').replace('-', '_')
|
|
182
|
+
html_lines.append(f' <li class="table-item" id="item-{safe_id}" ')
|
|
183
|
+
html_lines.append(f' onclick="showDiagram(\'{svg["file"]}\', \'{svg["name"]}\', \'{schema_name}\', \'{safe_id}\')">')
|
|
184
|
+
html_lines.append(f' {svg["display_name"]}')
|
|
185
|
+
html_lines.append(f' </li>')
|
|
186
|
+
|
|
187
|
+
html_lines.append(' </ul>')
|
|
188
|
+
html_lines.append(' </div>')
|
|
189
|
+
|
|
190
|
+
html_lines.extend([
|
|
191
|
+
' </div>',
|
|
192
|
+
' <div class="content">',
|
|
193
|
+
' <div id="viewer" class="empty-state">',
|
|
194
|
+
' <h2>Select a table to view its dependencies</h2>',
|
|
195
|
+
' <p>Choose from the list on the left</p>',
|
|
196
|
+
' </div>',
|
|
197
|
+
' </div>',
|
|
198
|
+
' <script>',
|
|
199
|
+
' let currentItem = null;',
|
|
200
|
+
' ',
|
|
201
|
+
' function showDiagram(file, name, schema, itemId) {',
|
|
202
|
+
' // Update active state',
|
|
203
|
+
' if (currentItem) {',
|
|
204
|
+
' document.getElementById(currentItem).classList.remove("active");',
|
|
205
|
+
' }',
|
|
206
|
+
' currentItem = "item-" + itemId;',
|
|
207
|
+
' document.getElementById(currentItem).classList.add("active");',
|
|
208
|
+
' ',
|
|
209
|
+
' // Show diagram',
|
|
210
|
+
' const viewer = document.getElementById("viewer");',
|
|
211
|
+
' viewer.className = "viewer";',
|
|
212
|
+
' viewer.innerHTML = `',
|
|
213
|
+
' <div class="viewer-title">',
|
|
214
|
+
' ${name}',
|
|
215
|
+
' <div class="viewer-subtitle">${schema}</div>',
|
|
216
|
+
' </div>',
|
|
217
|
+
' <img src="${file}" alt="${name} dependencies">',
|
|
218
|
+
' `;',
|
|
219
|
+
' }',
|
|
220
|
+
' ',
|
|
221
|
+
' function toggleSchema(schemaName) {',
|
|
222
|
+
' const list = document.getElementById("list-" + schemaName);',
|
|
223
|
+
' const icon = document.getElementById("icon-" + schemaName);',
|
|
224
|
+
' ',
|
|
225
|
+
' if (list.classList.contains("collapsed")) {',
|
|
226
|
+
' list.classList.remove("collapsed");',
|
|
227
|
+
' icon.textContent = "▼";',
|
|
228
|
+
' } else {',
|
|
229
|
+
' list.classList.add("collapsed");',
|
|
230
|
+
' icon.textContent = "▶";',
|
|
231
|
+
' }',
|
|
232
|
+
' }',
|
|
233
|
+
' ',
|
|
234
|
+
' // Auto-expand first schema and select first table',
|
|
235
|
+
' window.addEventListener("load", function() {',
|
|
236
|
+
' const firstTable = document.querySelector(".table-item");',
|
|
237
|
+
' if (firstTable) {',
|
|
238
|
+
' firstTable.click();',
|
|
239
|
+
' }',
|
|
240
|
+
' });',
|
|
241
|
+
' </script>',
|
|
242
|
+
'</body>',
|
|
243
|
+
'</html>',
|
|
244
|
+
])
|
|
245
|
+
|
|
246
|
+
return '\n'.join(html_lines)
|
|
247
|
+
|
|
248
|
+
def main():
|
|
249
|
+
print("Collecting SVG files from all schemas...")
|
|
250
|
+
schemas = collect_all_svgs()
|
|
251
|
+
|
|
252
|
+
if not schemas:
|
|
253
|
+
print("No dependency folders found. Run split_dependencies_svg.py first.")
|
|
254
|
+
return
|
|
255
|
+
|
|
256
|
+
print(f"\nFound {len(schemas)} schemas:")
|
|
257
|
+
for schema_name, schema_data in sorted(schemas.items()):
|
|
258
|
+
print(f" - {schema_name}: {len(schema_data['svgs'])} tables")
|
|
259
|
+
|
|
260
|
+
print("\nGenerating master index.html...")
|
|
261
|
+
html_content = generate_master_index(schemas)
|
|
262
|
+
|
|
263
|
+
output_folder = Path('output')
|
|
264
|
+
output_folder.mkdir(exist_ok=True)
|
|
265
|
+
output_file = output_folder / 'dependencies_master_index.html'
|
|
266
|
+
with open(output_file, 'w', encoding='utf-8') as f:
|
|
267
|
+
f.write(html_content)
|
|
268
|
+
|
|
269
|
+
print(f"\nMaster index created: {output_file}")
|
|
270
|
+
print(f"Total tables: {sum(len(s['svgs']) for s in schemas.values())}")
|
|
271
|
+
|
|
272
|
+
# Open in browser
|
|
273
|
+
import subprocess
|
|
274
|
+
subprocess.run(['start', str(output_file)], shell=True)
|
|
275
|
+
print("\nOpening in browser...")
|
|
276
|
+
|
|
277
|
+
if __name__ == "__main__":
|
|
278
|
+
main()
|
dataform_viz/parser.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Parser for Dataform dependencies report
|
|
3
|
+
"""
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Dict, List, Tuple
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def parse_dependencies_report(report_path: str) -> Dict[str, dict]:
|
|
10
|
+
"""
|
|
11
|
+
Parse the dependencies_report.txt file
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
report_path: Path to dependencies report file
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
Dictionary mapping table names to their info (type, dependencies, dependents)
|
|
18
|
+
"""
|
|
19
|
+
tables = {}
|
|
20
|
+
current_table = None
|
|
21
|
+
|
|
22
|
+
# Try different encodings
|
|
23
|
+
encodings = ['utf-8', 'utf-16', 'cp1252', 'latin-1']
|
|
24
|
+
content = None
|
|
25
|
+
|
|
26
|
+
report_file = Path(report_path)
|
|
27
|
+
if not report_file.exists():
|
|
28
|
+
raise FileNotFoundError(f"Report file not found: {report_path}")
|
|
29
|
+
|
|
30
|
+
for encoding in encodings:
|
|
31
|
+
try:
|
|
32
|
+
with open(report_file, 'r', encoding=encoding) as f:
|
|
33
|
+
content = f.read()
|
|
34
|
+
break
|
|
35
|
+
except (UnicodeDecodeError, UnicodeError):
|
|
36
|
+
continue
|
|
37
|
+
|
|
38
|
+
if content is None:
|
|
39
|
+
raise ValueError("Could not decode file with any supported encoding")
|
|
40
|
+
|
|
41
|
+
for line in content.split('\n'):
|
|
42
|
+
line = line.rstrip()
|
|
43
|
+
|
|
44
|
+
# Match table definition line
|
|
45
|
+
table_match = re.match(r'^Table: (.+?) \((\w+)\)$', line)
|
|
46
|
+
if table_match:
|
|
47
|
+
table_name = table_match.group(1)
|
|
48
|
+
table_type = table_match.group(2)
|
|
49
|
+
current_table = table_name
|
|
50
|
+
tables[current_table] = {
|
|
51
|
+
'type': table_type,
|
|
52
|
+
'dependencies': [],
|
|
53
|
+
'dependents': []
|
|
54
|
+
}
|
|
55
|
+
continue
|
|
56
|
+
|
|
57
|
+
# Match dependency line
|
|
58
|
+
if current_table and '<-' in line:
|
|
59
|
+
dep = line.strip().replace('<- ', '').strip()
|
|
60
|
+
if dep:
|
|
61
|
+
tables[current_table]['dependencies'].append(dep)
|
|
62
|
+
|
|
63
|
+
# Match dependent line
|
|
64
|
+
if current_table and '->' in line:
|
|
65
|
+
dep = line.strip().replace('-> ', '').strip()
|
|
66
|
+
if dep:
|
|
67
|
+
tables[current_table]['dependents'].append(dep)
|
|
68
|
+
|
|
69
|
+
return tables
|