truthound 1.0.8__py3-none-any.whl

truthound/lineage/visualization/renderers/graphviz.py

@@ -0,0 +1,276 @@
+"""Graphviz DOT renderer for lineage graphs.
+
+Produces DOT format output for use with Graphviz tools.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from truthound.lineage.visualization.protocols import (
+    IGraphRenderer,
+    RenderFormat,
+    RenderConfig,
+)
+
+if TYPE_CHECKING:
+    from truthound.lineage.base import LineageGraph
+
+
+class GraphvizRenderer(IGraphRenderer):
+    """Renders lineage graphs to Graphviz DOT format.
+
+    DOT format can be used with Graphviz tools to generate
+    SVG, PNG, PDF and other image formats.
+
+    Example:
+        >>> renderer = GraphvizRenderer()
+        >>> dot_output = renderer.render(graph)
+        >>> # Save and render: dot -Tpng output.dot -o output.png
+    """
+
+    # Node shapes by type
+    NODE_SHAPES = {
+        "SOURCE": "cylinder",
+        "TABLE": "box3d",
+        "FILE": "note",
+        "STREAM": "parallelogram",
+        "TRANSFORMATION": "component",
+        "VALIDATION": "diamond",
+        "MODEL": "hexagon",
+        "REPORT": "tab",
+        "EXTERNAL": "oval",
+        "VIRTUAL": "none",
+    }
+
+    # Edge styles by type
+    EDGE_STYLES = {
+        "DERIVED_FROM": "solid",
+        "VALIDATED_BY": "dashed",
+        "USED_BY": "dotted",
+        "TRANSFORMED_TO": "solid",
+        "JOINED_WITH": "bold",
+        "AGGREGATED_TO": "solid",
+        "FILTERED_TO": "dashed",
+        "DEPENDS_ON": "dotted",
+    }
+
+    @property
+    def format(self) -> RenderFormat:
+        return RenderFormat.DOT
+
+    def render(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig | None = None,
+    ) -> str:
+        """Render graph to DOT format.
+
+        Args:
+            graph: Lineage graph to render
+            config: Render configuration
+
+        Returns:
+            DOT format string
+        """
+        config = config or RenderConfig()
+        return self._build_dot(graph, config)
+
+    def render_subgraph(
+        self,
+        graph: "LineageGraph",
+        root_node_id: str,
+        direction: str = "downstream",
+        max_depth: int = -1,
+        config: RenderConfig | None = None,
+    ) -> str:
+        """Render subgraph from root node.
+
+        Args:
+            graph: Full lineage graph
+            root_node_id: Root node ID
+            direction: "upstream" or "downstream"
+            max_depth: Maximum depth
+            config: Render configuration
+
+        Returns:
+            DOT format string
+        """
+        config = config or RenderConfig()
+
+        if direction == "upstream":
+            subgraph = graph.get_upstream(root_node_id, max_depth)
+        else:
+            subgraph = graph.get_downstream(root_node_id, max_depth)
+
+        config.highlight_nodes = config.highlight_nodes + [root_node_id]
+
+        return self._build_dot(subgraph, config)
+
+    def _build_dot(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig,
+    ) -> str:
+        """Build DOT format string."""
+        lines = []
+
+        # Graph header
+        lines.append("digraph LineageGraph {")
+        lines.append("    // Graph settings")
+        lines.append(f"    rankdir={config.orientation};")
+        lines.append("    node [fontname=\"Arial\", fontsize=10];")
+        lines.append("    edge [fontname=\"Arial\", fontsize=8];")
+        lines.append("")
+
+        # Node definitions
+        lines.append("    // Nodes")
+        node_ids = set()
+
+        # Handle both list and dict node formats
+        if isinstance(graph.nodes, dict):
+            node_items = graph.nodes.items()
+        else:
+            node_items = [(node.id, node) for node in graph.nodes]
+
+        for node_id, node in node_items:
+            if config.filter_node_types and node.node_type.value not in config.filter_node_types:
+                continue
+
+            node_ids.add(node_id)
+
+            shape = self.NODE_SHAPES.get(node.node_type.value, "ellipse")
+            color = config.get_node_color(node.node_type.value)
+            label = self._escape_label(node.name)
+
+            # Build node attributes
+            attrs = [
+                f'label="{label}"',
+                f'shape={shape}',
+                f'fillcolor="{color}"',
+                'style=filled',
+            ]
+
+            # Highlight
+            if node_id in config.highlight_nodes:
+                attrs.append('penwidth=3')
+                attrs.append('color="red"')
+
+            # Tooltip with metadata
+            if config.include_metadata and hasattr(node, "metadata"):
+                tooltip = f"{node.name}\\nType: {node.node_type.value}"
+                if node.metadata.description:
+                    tooltip += f"\\n{node.metadata.description}"
+                attrs.append(f'tooltip="{self._escape_label(tooltip)}"')
+
+            safe_id = self._safe_id(node_id)
+            lines.append(f'    {safe_id} [{", ".join(attrs)}];')
+
+        lines.append("")
+
+        # Edge definitions
+        lines.append("    // Edges")
+
+        for edge in graph.edges:
+            if config.filter_edge_types and edge.edge_type.value not in config.filter_edge_types:
+                continue
+
+            # Handle both source/source_id attribute names
+            source_id = getattr(edge, 'source_id', None) or getattr(edge, 'source', None)
+            target_id = getattr(edge, 'target_id', None) or getattr(edge, 'target', None)
+
+            if source_id not in node_ids or target_id not in node_ids:
+                continue
+
+            style = self.EDGE_STYLES.get(edge.edge_type.value, "solid")
+            color = config.get_edge_color(edge.edge_type.value)
+
+            attrs = [
+                f'style={style}',
+                f'color="{color}"',
+                f'label="{edge.edge_type.value}"',
+            ]
+
+            safe_source = self._safe_id(source_id)
+            safe_target = self._safe_id(target_id)
+            lines.append(f'    {safe_source} -> {safe_target} [{", ".join(attrs)}];')
+
+        lines.append("}")
+
+        return "\n".join(lines)
+
+    def _escape_label(self, text: str) -> str:
+        """Escape text for DOT labels."""
+        return text.replace('"', '\\"').replace("\n", "\\n")
+
+    def _safe_id(self, node_id: str) -> str:
+        """Convert node ID to safe DOT identifier."""
+        # Replace non-alphanumeric with underscore
+        safe = "".join(c if c.isalnum() else "_" for c in node_id)
+        # Ensure it starts with letter
+        if safe and not safe[0].isalpha():
+            safe = "n_" + safe
+        return safe or "node"
+
+    def render_svg(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig | None = None,
+    ) -> str:
+        """Render graph to SVG using Graphviz.
+
+        Requires graphviz package to be installed.
+
+        Args:
+            graph: Lineage graph
+            config: Render configuration
+
+        Returns:
+            SVG string
+        """
+        try:
+            import graphviz
+        except ImportError:
+            raise ImportError(
+                "graphviz package is required for SVG rendering. "
+                "Install with: pip install graphviz"
+            )
+
+        dot = self.render(graph, config)
+
+        # Create graphviz source
+        source = graphviz.Source(dot)
+        svg = source.pipe(format="svg").decode("utf-8")
+
+        return svg
+
+    def render_png(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig | None = None,
+    ) -> bytes:
+        """Render graph to PNG using Graphviz.
+
+        Requires graphviz package to be installed.
+
+        Args:
+            graph: Lineage graph
+            config: Render configuration
+
+        Returns:
+            PNG bytes
+        """
+        try:
+            import graphviz
+        except ImportError:
+            raise ImportError(
+                "graphviz package is required for PNG rendering. "
+                "Install with: pip install graphviz"
+            )
+
+        dot = self.render(graph, config)
+
+        source = graphviz.Source(dot)
+        png = source.pipe(format="png")
+
+        return png

truthound/lineage/visualization/renderers/mermaid.py

@@ -0,0 +1,308 @@
+"""Mermaid renderer for lineage graphs.
+
+Produces Mermaid diagram syntax for Markdown-friendly visualization.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from truthound.lineage.visualization.protocols import (
+    IGraphRenderer,
+    RenderFormat,
+    RenderConfig,
+)
+
+if TYPE_CHECKING:
+    from truthound.lineage.base import LineageGraph
+
+
+class MermaidRenderer(IGraphRenderer):
+    """Renders lineage graphs to Mermaid diagram syntax.
+
+    Mermaid diagrams can be embedded in Markdown files and
+    rendered by GitHub, GitLab, and other platforms.
+
+    Example:
+        >>> renderer = MermaidRenderer()
+        >>> mermaid = renderer.render(graph)
+        >>> # Embed in Markdown: ```mermaid\\n{mermaid}\\n```
+    """
+
+    # Node shapes in Mermaid syntax
+    NODE_SHAPES = {
+        "SOURCE": ("[(", ")]"),  # Cylindrical
+        "TABLE": ("[", "]"),  # Rectangle
+        "FILE": ("{{", "}}"),  # Hexagon
+        "STREAM": ("[/", "/]"),  # Parallelogram
+        "TRANSFORMATION": ("[[", "]]"),  # Subroutine
+        "VALIDATION": ("{", "}"),  # Diamond-like
+        "MODEL": ("([", "])"),  # Stadium
+        "REPORT": ("[", "]"),  # Rectangle
+        "EXTERNAL": ("((", "))"),  # Circle
+        "VIRTUAL": ("[", "]"),  # Rectangle
+    }
+
+    # Edge arrow styles
+    EDGE_ARROWS = {
+        "DERIVED_FROM": "-->",
+        "VALIDATED_BY": "-.->",
+        "USED_BY": "-.->",
+        "TRANSFORMED_TO": "==>",
+        "JOINED_WITH": "<-->",
+        "AGGREGATED_TO": "-->",
+        "FILTERED_TO": "-.->",
+        "DEPENDS_ON": "-.-",
+    }
+
+    @property
+    def format(self) -> RenderFormat:
+        return RenderFormat.MERMAID
+
+    def render(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig | None = None,
+    ) -> str:
+        """Render graph to Mermaid syntax.
+
+        Args:
+            graph: Lineage graph to render
+            config: Render configuration
+
+        Returns:
+            Mermaid diagram string
+        """
+        config = config or RenderConfig()
+        return self._build_mermaid(graph, config)
+
+    def render_subgraph(
+        self,
+        graph: "LineageGraph",
+        root_node_id: str,
+        direction: str = "downstream",
+        max_depth: int = -1,
+        config: RenderConfig | None = None,
+    ) -> str:
+        """Render subgraph from root node.
+
+        Args:
+            graph: Full lineage graph
+            root_node_id: Root node ID
+            direction: "upstream" or "downstream"
+            max_depth: Maximum depth
+            config: Render configuration
+
+        Returns:
+            Mermaid diagram string
+        """
+        config = config or RenderConfig()
+
+        if direction == "upstream":
+            subgraph = graph.get_upstream(root_node_id, max_depth)
+        else:
+            subgraph = graph.get_downstream(root_node_id, max_depth)
+
+        config.highlight_nodes = config.highlight_nodes + [root_node_id]
+
+        return self._build_mermaid(subgraph, config)
+
+    def _build_mermaid(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig,
+    ) -> str:
+        """Build Mermaid diagram string."""
+        lines = []
+
+        # Diagram header
+        lines.append(f"graph {config.orientation}")
+        lines.append("")
+
+        # Group nodes by type for styling
+        node_by_type: dict[str, list[str]] = {}
+        node_ids = set()
+
+        # Handle both list and dict node formats
+        if isinstance(graph.nodes, dict):
+            node_items = graph.nodes.items()
+        else:
+            node_items = [(node.id, node) for node in graph.nodes]
+
+        # Node definitions
+        for node_id, node in node_items:
+            if config.filter_node_types and node.node_type.value not in config.filter_node_types:
+                continue
+
+            node_ids.add(node_id)
+
+            node_type = node.node_type.value
+            if node_type not in node_by_type:
+                node_by_type[node_type] = []
+            node_by_type[node_type].append(node_id)
+
+            # Build node definition
+            safe_id = self._safe_id(node_id)
+            label = self._escape_label(node.name)
+            left, right = self.NODE_SHAPES.get(node_type, ("[", "]"))
+
+            lines.append(f"    {safe_id}{left}{label}{right}")
+
+        lines.append("")
+
+        # Edge definitions
+        for edge in graph.edges:
+            if config.filter_edge_types and edge.edge_type.value not in config.filter_edge_types:
+                continue
+
+            # Handle both source/source_id attribute names
+            source_id = getattr(edge, 'source_id', None) or getattr(edge, 'source', None)
+            target_id = getattr(edge, 'target_id', None) or getattr(edge, 'target', None)
+
+            if source_id not in node_ids or target_id not in node_ids:
+                continue
+
+            safe_source = self._safe_id(source_id)
+            safe_target = self._safe_id(target_id)
+            arrow = self.EDGE_ARROWS.get(edge.edge_type.value, "-->")
+
+            # Add edge label if not too long
+            edge_label = edge.edge_type.value
+            if len(edge_label) <= 15:
+                lines.append(f"    {safe_source} {arrow}|{edge_label}| {safe_target}")
+            else:
+                lines.append(f"    {safe_source} {arrow} {safe_target}")
+
+        lines.append("")
+
+        # Styling by node type
+        lines.append("    %% Styling")
+        for node_type, ids in node_by_type.items():
+            color = config.get_node_color(node_type)
+            safe_ids = ",".join(self._safe_id(nid) for nid in ids)
+            lines.append(f"    style {safe_ids} fill:{color}")
+
+        # Highlight nodes
+        if config.highlight_nodes:
+            for node_id in config.highlight_nodes:
+                if node_id in node_ids:
+                    safe_id = self._safe_id(node_id)
+                    lines.append(f"    style {safe_id} stroke:#ff0000,stroke-width:3px")
+
+        return "\n".join(lines)
+
+    def _escape_label(self, text: str) -> str:
+        """Escape text for Mermaid labels."""
+        # Remove special characters that break Mermaid syntax
+        return text.replace('"', "'").replace("[", "(").replace("]", ")").replace("{", "(").replace("}", ")")
+
+    def _safe_id(self, node_id: str) -> str:
+        """Convert node ID to safe Mermaid identifier."""
+        # Replace non-alphanumeric with underscore
+        safe = "".join(c if c.isalnum() else "_" for c in node_id)
+        return safe or "node"
+
+    def render_markdown(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig | None = None,
+        title: str = "Lineage Graph",
+    ) -> str:
+        """Render graph as Markdown with embedded Mermaid diagram.
+
+        Args:
+            graph: Lineage graph
+            config: Render configuration
+            title: Document title
+
+        Returns:
+            Markdown string
+        """
+        config = config or RenderConfig()
+        mermaid = self.render(graph, config)
+
+        md_lines = [
+            f"# {title}",
+            "",
+            "```mermaid",
+            mermaid,
+            "```",
+            "",
+            "## Legend",
+            "",
+            "| Symbol | Node Type |",
+            "|--------|-----------|",
+        ]
+
+        # Add legend
+        for node_type, (left, right) in self.NODE_SHAPES.items():
+            md_lines.append(f"| `{left}...{right}` | {node_type} |")
+
+        md_lines.extend([
+            "",
+            "## Edge Types",
+            "",
+            "| Arrow | Relationship |",
+            "|-------|--------------|",
+        ])
+
+        for edge_type, arrow in self.EDGE_ARROWS.items():
+            md_lines.append(f"| `{arrow}` | {edge_type} |")
+
+        return "\n".join(md_lines)
+
+    def render_html(
+        self,
+        graph: "LineageGraph",
+        config: RenderConfig | None = None,
+    ) -> str:
+        """Render complete HTML page with Mermaid diagram.
+
+        Args:
+            graph: Lineage graph
+            config: Render configuration
+
+        Returns:
+            Complete HTML page
+        """
+        config = config or RenderConfig()
+        mermaid = self.render(graph, config)
+
+        return f"""<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <title>Lineage Graph - Mermaid</title>
+    <script src="https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.min.js"></script>
+    <style>
+        body {{
+            margin: 0;
+            padding: 20px;
+            font-family: Arial, sans-serif;
+            display: flex;
+            justify-content: center;
+        }}
+        .mermaid {{
+            background: white;
+            padding: 20px;
+            border-radius: 8px;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+        }}
+    </style>
+</head>
+<body>
+    <div class="mermaid">
+{mermaid}
+    </div>
+    <script>
+        mermaid.initialize({{
+            startOnLoad: true,
+            theme: 'default',
+            flowchart: {{
+                useMaxWidth: true,
+                htmlLabels: true,
+            }}
+        }});
+    </script>
+</body>
+</html>"""

truthound/maskers.py

@@ -0,0 +1,113 @@
+"""Data masking utilities for anonymizing sensitive data."""
+
+import hashlib
+import random
+import string
+
+import polars as pl
+
+from truthound.scanners import scan_pii
+
+
+def mask_data(
+    lf: pl.LazyFrame,
+    columns: list[str] | None = None,
+    strategy: str = "redact",
+) -> pl.DataFrame:
+    """Mask sensitive data in a LazyFrame.
+
+    Args:
+        lf: Polars LazyFrame to mask.
+        columns: Optional list of columns to mask. If None, auto-detects PII.
+        strategy: Masking strategy - "redact", "hash", or "fake".
+
+    Returns:
+        Polars DataFrame with masked values.
+
+    Raises:
+        ValueError: If an invalid strategy is provided.
+    """
+    if strategy not in ("redact", "hash", "fake"):
+        raise ValueError(f"Invalid strategy: {strategy}. Use 'redact', 'hash', or 'fake'.")
+
+    df = lf.collect()
+
+    # Auto-detect PII columns if not specified
+    if columns is None:
+        pii_findings = scan_pii(lf)
+        columns = [f["column"] for f in pii_findings]
+
+    if not columns:
+        return df
+
+    # Apply masking to each column
+    for col in columns:
+        if col not in df.columns:
+            continue
+
+        if strategy == "redact":
+            df = _apply_redact(df, col)
+        elif strategy == "hash":
+            df = _apply_hash(df, col)
+        elif strategy == "fake":
+            df = _apply_fake(df, col)
+
+    return df
+
+
+def _apply_redact(df: pl.DataFrame, col: str) -> pl.DataFrame:
+    """Apply redaction masking - replaces characters with asterisks."""
+
+    def redact_value(val: str | None) -> str | None:
+        if val is None:
+            return None
+        # Keep structure hints for common formats
+        if "@" in val:  # Email
+            parts = val.split("@")
+            if len(parts) == 2:
+                local = "*" * len(parts[0])
+                domain_parts = parts[1].split(".")
+                domain = ".".join("*" * len(p) for p in domain_parts)
+                return f"{local}@{domain}"
+        if "-" in val and len(val.replace("-", "").replace(" ", "")) in (9, 10, 16):  # SSN or phone or CC
+            return "".join("*" if c.isalnum() else c for c in val)
+        # Default: replace all characters
+        return "*" * len(val)
+
+    return df.with_columns(pl.col(col).map_elements(redact_value, return_dtype=pl.String).alias(col))
+
+
+def _apply_hash(df: pl.DataFrame, col: str) -> pl.DataFrame:
+    """Apply hash masking - replaces values with SHA256 hash."""
+
+    def hash_value(val: str | None) -> str | None:
+        if val is None:
+            return None
+        return hashlib.sha256(val.encode()).hexdigest()[:16]
+
+    return df.with_columns(pl.col(col).map_elements(hash_value, return_dtype=pl.String).alias(col))
+
+
+def _apply_fake(df: pl.DataFrame, col: str) -> pl.DataFrame:
+    """Apply fake data masking - replaces with realistic-looking fake data."""
+    col_lower = col.lower()
+
+    def generate_fake(val: str | None) -> str | None:
+        if val is None:
+            return None
+
+        # Generate appropriate fake data based on column name or value pattern
+        if "email" in col_lower or ("@" in val and "." in val):
+            random_str = "".join(random.choices(string.ascii_lowercase, k=8))
+            return f"user{random_str}@masked.com"
+
+        if "phone" in col_lower or (val.replace("-", "").replace(" ", "").replace("+", "").isdigit() and len(val) >= 7):
+            return f"+1-555-{random.randint(100, 999)}-{random.randint(1000, 9999)}"
+
+        if "ssn" in col_lower or (len(val) == 11 and val.count("-") == 2):
+            return f"{random.randint(100, 999)}-{random.randint(10, 99)}-{random.randint(1000, 9999)}"
+
+        # Default: generate random string of same length
+        return "".join(random.choices(string.ascii_letters + string.digits, k=len(val)))
+
+    return df.with_columns(pl.col(col).map_elements(generate_fake, return_dtype=pl.String).alias(col))