dataform-dependency-visualizer 0.1.0__py3-none-any.whl

dataform_dependency_visualizer-0.1.0.dist-info/METADATA

@@ -0,0 +1,170 @@
+Metadata-Version: 2.4
+Name: dataform-dependency-visualizer
+Version: 0.1.0
+Summary: Visualize Dataform table dependencies as interactive SVG diagrams
+Author-email: Thamo <thamo@example.com>
+License: MIT
+Project-URL: Homepage, https://github.com/thamo/dataform-dependency-visualizer
+Project-URL: Repository, https://github.com/thamo/dataform-dependency-visualizer
+Keywords: dataform,dependencies,visualization,bigquery,sql
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Database
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Dataform Dependency Visualizer
+
+Generate beautiful, interactive SVG diagrams showing dependencies between Dataform tables.
+
+## Features
+
+- 📊 **Individual table diagrams** - One SVG per table showing immediate dependencies
+- 🎨 **Color-coded by type** - Tables, views, and operations visually distinct
+- 🔍 **Master index viewer** - Browse all tables in single interface
+- 📁 **Schema organization** - Organized by schema with collapsible sections
+- ⚡ **Pure Python SVGs** - No Graphviz required
+- 🎯 **Orthogonal routing** - Clean, professional arrow paths
+- 📝 **Smart text wrapping** - Long table names split across lines
+
+## Installation
+
+```bash
+pip install dataform-dependency-visualizer
+```
+
+## Quick Start
+
+### 1. Generate dependency report
+
+In your Dataform project directory:
+
+```bash
+dataform compile --json > dependencies_report.txt
+```
+
+### 2. Generate SVG diagrams
+
+```bash
+# Generate for specific schema
+dataform-deps generate dashboard_wwim
+
+# Generate for all schemas (excluding refined_*)
+dataform-deps generate-all
+
+# Generate master index
+dataform-deps index
+```
+
+### 3. View diagrams
+
+Open `output/dependencies_master_index.html` in your browser to browse all tables and their dependencies.
+
+## Usage
+
+### Command Line
+
+**Generate diagrams for a specific schema:**
+
+```bash
+dataform-deps generate SCHEMA_NAME
+```
+
+Example:
+```bash
+dataform-deps generate dashboard_wwim
+```
+
+**Generate for all schemas:**
+
+```bash
+dataform-deps generate-all
+```
+
+By default, schemas starting with `refined_` are excluded. To include them:
+
+```bash
+dataform-deps generate-all --include-refined
+```
+
+**Generate master index:**
+
+```bash
+dataform-deps index
+```
+
+Creates `output/dependencies_master_index.html` with links to all tables.
+
+**Check prerequisites:**
+
+```bash
+dataform-deps check
+```
+
+Verifies that:
+- You're in a Dataform project directory
+- `dependencies_report.txt` exists
+- The report contains valid dependency data
+
+## Example Output
+
+### Individual Table Diagram
+
+Each table gets its own SVG showing:
+- **Dependencies (upstream)** - Tables this table depends on
+- **Dependents (downstream)** - Tables that depend on this table
+- **Color coding** - Tables (blue), views (green), operations (orange)
+
+### Master Index
+
+The master index organizes all tables by schema with:
+- Clickable table names that open their SVG
+- Expandable/collapsible schemas
+- Type badges (table/view/incremental)
+- Search functionality
+
+## Requirements
+
+- Python 3.8+
+- Dataform project with compiled dependencies
+
+## How It Works
+
+1. Parse `dependencies_report.txt` generated by Dataform
+2. Extract table dependencies and metadata
+3. Generate SVG diagrams with orthogonal routing
+4. Create master index HTML for easy navigation
+
+## Project Structure
+
+```
+output/
+├── dependencies_master_index.html    # Main entry point
+└── dependencies/
+    ├── schema1_table1.svg
+    ├── schema1_table2.svg
+    └── ...
+```
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Contributing
+
+Contributions welcome! Please open an issue or PR on GitHub.
+
+## Links
+
+- [GitHub Repository](https://github.com/yourusername/dataform-dependency-visualizer)
+- [Issue Tracker](https://github.com/yourusername/dataform-dependency-visualizer/issues)

dataform_dependency_visualizer-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+dataform_dependency_visualizer-0.1.0.dist-info/licenses/LICENSE,sha256=ixWsdND9GXUi7Eb1toJqBGEpQox6S5UWDFlMUrgwTLY,1089
+dataform_viz/__init__.py,sha256=tspWZ_DIMIQ7-SbhaHBnui7wmJgnDEn1q_OWHJcwolQ,379
+dataform_viz/cli.py,sha256=BQ8UKFqpF21WyVGyJmmjqtLmNWxeDte-ysithtUeTHA,4577
+dataform_viz/dataform_check.py,sha256=A40KXcbIpHQTB1kgiLHGMXMV878Pj3OLMEYGSbx9VUI,6684
+dataform_viz/master_index.py,sha256=KT8VbjVVi5MJR0_l6a6Y4dlR46_-iCihS-Mixgd6XWk,10735
+dataform_viz/parser.py,sha256=qLjif5TwKoNqXbXudPSIhNmADfjGrjVWZUK9ZpBbwBg,2133
+dataform_viz/svg_generator.py,sha256=BlgzyEQ6iBUHXdrAFiE5JN_UAHwJd9Pg152T5-roNho,15816
+dataform_viz/visualizer.py,sha256=n3DVfBOYebE66Nun8rdU9gVsqmoUAr-jk2JT9ygw5T4,4645
+dataform_dependency_visualizer-0.1.0.dist-info/METADATA,sha256=_ErrVSh7LByFIZpAcg2um_c9r8ZZeCG7cacDyfuT8_A,4509
+dataform_dependency_visualizer-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataform_dependency_visualizer-0.1.0.dist-info/entry_points.txt,sha256=M1Ue84hbvcL7duLuI9ieyX0d0hSXnzOpsiaskhCt-1Y,56
+dataform_dependency_visualizer-0.1.0.dist-info/top_level.txt,sha256=EIiWhSvCyeLRkhNmH1r-NChI-pZ0NbECn4Ya0YsF91U,13
+dataform_dependency_visualizer-0.1.0.dist-info/RECORD,,

dataform_dependency_visualizer-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dataform_dependency_visualizer-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dataform-deps = dataform_viz.cli:main

dataform_dependency_visualizer-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 [Your Name]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dataform_dependency_visualizer-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dataform_viz

dataform_viz/__init__.py

@@ -0,0 +1,14 @@
+"""
+Dataform Dependency Visualizer
+
+Generate interactive SVG diagrams showing table dependencies in Dataform projects.
+"""
+
+__version__ = "0.1.0"
+__author__ = "Your Name"
+__email__ = "your.email@example.com"
+
+from .visualizer import DependencyVisualizer
+from .parser import parse_dependencies_report
+
+__all__ = ["DependencyVisualizer", "parse_dependencies_report"]

dataform_viz/cli.py

@@ -0,0 +1,159 @@
+"""
+Command-line interface for dataform-dependency-visualizer
+"""
+import sys
+import argparse
+from pathlib import Path
+from .visualizer import DependencyVisualizer
+
+
+def cmd_generate(args):
+    """Generate SVGs for a specific schema"""
+    viz = DependencyVisualizer(args.report)
+    
+    try:
+        count = viz.generate_schema_svgs(
+            args.schema,
+            output_dir=args.output
+        )
+        print(f"✓ Generated {count} SVG diagrams for {args.schema}")
+        print(f"  Output: {args.output}/dependencies_{args.schema}/")
+        return 0
+    except Exception as e:
+        print(f"✗ Error: {e}", file=sys.stderr)
+        return 1
+
+
+def cmd_generate_all(args):
+    """Generate SVGs for all schemas"""
+    viz = DependencyVisualizer(args.report)
+    
+    try:
+        results = viz.generate_all_schemas(
+            output_dir=args.output,
+            exclude_patterns=args.exclude or ['refined_*']
+        )
+        
+        total = sum(results.values())
+        print(f"\n✓ Generated {total} SVG diagrams across {len(results)} schemas:")
+        for schema, count in sorted(results.items()):
+            print(f"  - {schema}: {count} tables")
+        print(f"\nOutput directory: {args.output}/")
+        return 0
+    except Exception as e:
+        print(f"✗ Error: {e}", file=sys.stderr)
+        return 1
+
+
+def cmd_index(args):
+    """Generate master index"""
+    viz = DependencyVisualizer(args.report)
+    
+    try:
+        index_file = viz.generate_master_index(output_dir=args.output)
+        print(f"✓ Master index created: {index_file}")
+        
+        if args.open:
+            import subprocess
+            subprocess.run(['start', str(index_file)], shell=True)
+        
+        return 0
+    except Exception as e:
+        print(f"✗ Error: {e}", file=sys.stderr)
+        return 1
+
+
+def cmd_setup(args):
+    """Full setup pipeline"""
+    from .dataform_check import check_prerequisites
+    
+    print("=" * 60)
+    print("DATAFORM DEPENDENCIES VISUALIZATION SETUP")
+    print("=" * 60)
+    
+    if not check_prerequisites():
+        print("\n⚠ Prerequisites check failed")
+        return 1
+    
+    # Generate all
+    args_all = argparse.Namespace(
+        report=args.report,
+        output=args.output,
+        exclude=args.exclude
+    )
+    if cmd_generate_all(args_all) != 0:
+        return 1
+    
+    # Generate index
+    args_idx = argparse.Namespace(
+        report=args.report,
+        output=args.output,
+        open=True
+    )
+    return cmd_index(args_idx)
+
+
+def main():
+    """Main CLI entry point"""
+    parser = argparse.ArgumentParser(
+        prog='dataform-deps',
+        description='Generate interactive SVG diagrams for Dataform table dependencies'
+    )
+    
+    parser.add_argument(
+        '--report',
+        default='dependencies_report.txt',
+        help='Path to dependencies report file (default: dependencies_report.txt)'
+    )
+    
+    parser.add_argument(
+        '--output',
+        default='output',
+        help='Output directory (default: output)'
+    )
+    
+    subparsers = parser.add_subparsers(dest='command', help='Commands')
+    
+    # Generate command
+    gen_parser = subparsers.add_parser('generate', help='Generate SVGs for a schema')
+    gen_parser.add_argument('schema', help='Schema name to generate')
+    gen_parser.set_defaults(func=cmd_generate)
+    
+    # Generate-all command
+    gen_all_parser = subparsers.add_parser('generate-all', help='Generate SVGs for all schemas')
+    gen_all_parser.add_argument(
+        '--exclude',
+        nargs='+',
+        help='Schema patterns to exclude (default: refined_*)'
+    )
+    gen_all_parser.set_defaults(func=cmd_generate_all)
+    
+    # Index command
+    idx_parser = subparsers.add_parser('index', help='Generate master index')
+    idx_parser.add_argument(
+        '--open',
+        action='store_true',
+        help='Open index in browser'
+    )
+    idx_parser.set_defaults(func=cmd_index)
+    
+    # Setup command
+    setup_parser = subparsers.add_parser('setup', help='Full setup pipeline')
+    setup_parser.add_argument(
+        '--exclude',
+        nargs='+',
+        help='Schema patterns to exclude (default: refined_*)'
+    )
+    setup_parser.set_defaults(func=cmd_setup)
+    
+    args = parser.parse_args()
+    
+    if not args.command:
+        parser.print_help()
+        return 1
+    
+    return args.func(args)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

dataform_viz/dataform_check.py

@@ -0,0 +1,175 @@
+import subprocess
+import json
+import sys
+import os
+
+def get_dataform_graph():
+    print("Compiling Dataform graph (this may take a moment)...")
+    
+    # Try to find local dataform binary first (most reliable)
+    dataform_cmd = "dataform"
+    local_bin = os.path.join(os.getcwd(), "node_modules", ".bin", "dataform.cmd")
+    
+    if os.path.exists(local_bin):
+        dataform_cmd = f'"{local_bin}"'
+    else:
+        # Fallback to npx logic...
+        # helper to find npx
+        npx_cmd = "npx"
+        possible_paths = [
+            r"C:\Program Files\nodejs\npx.cmd",
+            r"C:\Program Files (x86)\nodejs\npx.cmd"
+        ]
+        
+        # Check if 'npx' is in path, if not try absolute paths
+        from shutil import which
+        if which("npx") is None:
+            for p in possible_paths:
+                if os.path.exists(p):
+                    npx_cmd = f'"{p}"' # wrapper quotes for shell
+                    break
+        
+        dataform_cmd = f"{npx_cmd} dataform"
+
+    cmd = f"{dataform_cmd} compile --json"
+    
+    # Run dataform compile and capture JSON output
+    # shell=True is often required on Windows to find npx.cmd
+    try:
+        # We assume npx is in the path.
+        result = subprocess.run(cmd, 
+                                capture_output=True, 
+                                text=True,
+                                encoding='utf-8',
+                                errors='ignore',
+                                shell=True, 
+                                check=True)
+        if not result.stdout:
+            print("No output from dataform compile")
+            return None
+        
+        # Dataform outputs log message then JSON on the same line
+        # Format: {"level":"INFO",...} { "tables": ...}
+        # We need to find the part that starts with { "tables"
+        output = result.stdout
+        
+        # Find the position where the actual compilation result starts
+        # Look for '{ "tables"' or '{\n    "tables"'
+        tables_pos = output.find('"tables"')
+        if tables_pos == -1:
+            print("Could not find 'tables' key in dataform output")
+            return None
+        
+        # Walk backwards to find the opening brace of this object
+        brace_pos = output.rfind('{', 0, tables_pos)
+        if brace_pos == -1:
+            print("Could not find JSON object start")
+            return None
+        
+        json_str = output[brace_pos:]
+        return json.loads(json_str)
+        
+    except subprocess.CalledProcessError as e:
+        print(f"Error running dataform: {e.stderr}")
+        return None
+    except json.JSONDecodeError as e:
+        print(f"Error decoding dataform output: {e}")
+        print("First 500 chars of attempted parse:", json_str[:500] if 'json_str' in locals() else "N/A")
+        return None
+    except Exception as e:
+        print(f"Error processing dataform output: {e}")
+        return None
+
+def normalize_name(target):
+    """Helper to get a consistent name string from a target object {database, schema, name}"""
+    # Adjust this based on how you prefer to see names (e.g., schema.name or just name)
+    if not target:
+        return "UNKNOWN"
+    return f"{target.get('schema', '')}.{target.get('name', '')}"
+
+def main():
+    graph = get_dataform_graph()
+    if not graph:
+        return
+
+    tables = graph.get("tables", [])
+    
+    # build a mapping of standard_name -> table_info
+    # Dataform JSON output for 'target' usually has schema and name. 
+    # The 'dependencies' list is usually an array of strings representing the names of the targets.
+    
+    table_lookup = {}
+    
+    # First pass: Index all tables by their full name and short name
+    for t in tables:
+        tgt = t.get("target", {})
+        full_name = normalize_name(tgt) # e.g. "dataset.table"
+        short_name = tgt.get("name")
+        
+        # Store using full name as key
+        table_lookup[full_name] = {
+            "type": t.get("type"),
+            "dependencies": t.get("dependencyTargets", []), # dataform 2.x often uses dependencyTargets (objects) or dependencies (strings)
+            # recent dataform versions might strictly use dependencyTargets. Let's check both or inspect.
+            # If dependencyTargets matches the current structure, otherwise fallback.
+            "dependents": []
+        }
+        
+        # Also store short name pointer if it doesn't conflict? 
+        # For safety, let's just stick to iterating for search.
+
+    # Note: 'dependencies' key in the compiled JSON often contains the list of resolved target names.
+    # Let's inspect what we actually have. 
+    
+    # Calculate dependents
+    for t in tables:
+        tgt = t.get("target", {})
+        this_full_name = normalize_name(tgt)
+        
+        # 'dependencyTargets' is preferred in newer dataform, it's a list of {schema, name, database...}
+        deps = t.get("dependencyTargets") 
+        if deps is None:
+             # Fallback to 'dependencies' if dependencyTargets is missing (older CLI)
+             # But 'dependencies' might just be strings.
+             pass
+        
+        if deps:
+            for d in deps:
+                dep_full_name = normalize_name(d)
+                if dep_full_name in table_lookup:
+                    table_lookup[dep_full_name]["dependents"].append(this_full_name)
+
+    # Search functionality
+    search_term = ""
+    if len(sys.argv) > 1:
+        search_term = sys.argv[1].lower()
+    
+    found_count = 0
+    print(f"\n--- Dataform Dependency Analysis ---")
+    
+    for name, info in table_lookup.items():
+        if not search_term or search_term in name.lower():
+            found_count += 1
+            print(f"\nTable: {name} ({info['type']})")
+            
+            deps = info['dependencies']
+            print(f"  Dependencies ({len(deps)}):")
+            for d in deps:
+                print(f"    <- {normalize_name(d)}")
+                
+            depts = info['dependents']
+            print(f"  Dependents ({len(depts)}):")
+            for d in depts:
+                print(f"    -> {d}")
+
+    if found_count == 0 and search_term:
+        print(f"No tables found matching '{search_term}'")
+    elif not search_term:
+        print("\nTip: Pass a table name as an argument to filter results.")
+        print("Example: python utility_check_dependencies.py my_table_name")
+
+if __name__ == "__main__":
+    # Ensure output directory exists
+    from pathlib import Path
+    Path('output').mkdir(exist_ok=True)
+    main()