dbt-tui 0.1.0__tar.gz → 0.2.0__tar.gz

{dbt_tui-0.1.0 → dbt_tui-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-tui
-Version: 0.1.0
+Version: 0.2.0
 Summary: Terminal UI for exploring and managing dbt projects
 License: MIT
 Keywords: dbt,tui,terminal,data,analytics,sql

{dbt_tui-0.1.0 → dbt_tui-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dbt-tui"
-version = "0.1.0"
+version = "0.2.0"
 description = "Terminal UI for exploring and managing dbt projects"
 authors = [
     {name = "Mike", email = "sortia@protonmail.com"}

{dbt_tui-0.1.0 → dbt_tui-0.2.0}/src/dbtui/backend/__init__.py

@@ -1,2 +1,3 @@
 from .project import DbtProject, DbtModelNotFoundException
 from .model import DbtModel
+from .metrics import LoadMetrics

dbt_tui-0.2.0/src/dbtui/backend/metrics.py

@@ -0,0 +1,75 @@
+"""
+Performance metrics for dbtui.
+
+Provides timing instrumentation for project loading and other operations.
+Metrics can be accessed after project load to diagnose performance issues.
+"""
+
+import time
+from dataclasses import dataclass, field
+from typing import Callable, TypeVar, ParamSpec
+
+P = ParamSpec('P')
+T = TypeVar('T')
+
+
+@dataclass
+class LoadMetrics:
+    """Metrics collected during project loading."""
+    model_count: int = 0
+    parse_dbt_project_yml_ms: float = 0.0
+    load_models_ms: float = 0.0
+    populate_graph_ms: float = 0.0
+    collect_property_claims_ms: float = 0.0
+    total_load_ms: float = 0.0
+
+    def __str__(self) -> str:
+        return (
+            f"Load metrics ({self.model_count} models):\n"
+            f"  parse_dbt_project.yml: {self.parse_dbt_project_yml_ms:.1f}ms\n"
+            f"  load_models: {self.load_models_ms:.1f}ms\n"
+            f"  populate_graph: {self.populate_graph_ms:.1f}ms\n"
+            f"  collect_property_claims: {self.collect_property_claims_ms:.1f}ms\n"
+            f"  total: {self.total_load_ms:.1f}ms"
+        )
+
+    def as_dict(self) -> dict:
+        """Return metrics as a dictionary for serialization."""
+        return {
+            'model_count': self.model_count,
+            'parse_dbt_project_yml_ms': self.parse_dbt_project_yml_ms,
+            'load_models_ms': self.load_models_ms,
+            'populate_graph_ms': self.populate_graph_ms,
+            'collect_property_claims_ms': self.collect_property_claims_ms,
+            'total_load_ms': self.total_load_ms,
+        }
+
+
+class Timer:
+    """Context manager for timing code blocks."""
+
+    def __init__(self):
+        self.start_time: float = 0.0
+        self.elapsed_ms: float = 0.0
+
+    def __enter__(self) -> 'Timer':
+        self.start_time = time.perf_counter()
+        return self
+
+    def __exit__(self, *args) -> None:
+        self.elapsed_ms = (time.perf_counter() - self.start_time) * 1000
+
+
+def timed(func: Callable[P, T]) -> Callable[P, tuple[T, float]]:
+    """
+    Decorator that returns both result and elapsed time in ms.
+
+    Usage:
+        result, elapsed_ms = timed(my_function)(arg1, arg2)
+    """
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> tuple[T, float]:
+        start = time.perf_counter()
+        result = func(*args, **kwargs)
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        return result, elapsed_ms
+    return wrapper

{dbt_tui-0.1.0 → dbt_tui-0.2.0}/src/dbtui/backend/project.py

@@ -16,11 +16,12 @@ InvalidProjectPathException
 
 from .model import DbtModel
 from .property_claim import PropertyClaimAggregate
-from .property_discovery import collect_model_claims
+from .property_discovery import collect_model_claims_cached, PropertyDiscoveryCache
+from .metrics import LoadMetrics, Timer
 
 
 class DbtProject(DbtProjectAbstract):
-    root_folder: Path 
+    root_folder: Path
     dbt_project_yml: Generator
     model_folder: str
     full_models_paths: list[Path]
@@ -32,6 +33,9 @@ class DbtProject(DbtProjectAbstract):
     models_by_name: dict[str, DbtModel]
     models_by_file_name: dict[str, DbtModel]
 
+    # performance metrics from last load
+    load_metrics: LoadMetrics | None
+
     def populate_graph(self):
         self.graph = DiGraph()
         for model in self.models:
@@ -60,10 +64,17 @@ class DbtProject(DbtProjectAbstract):
         This method creates a PropertyClaimAggregate for each model and populates
         it with claims from all sources (dbt_project.yml, schema.yml, model SQL).
         The aggregates handle precedence resolution lazily when accessed.
+
+        Uses PropertyDiscoveryCache to read and parse YAML files only once,
+        dramatically improving performance for projects with many models.
         """
+        # Initialize cache once for all models
+        cache = PropertyDiscoveryCache()
+        cache.initialize(self)
+
         for model in self.models:
             aggregate = PropertyClaimAggregate(model)
-            claims = collect_model_claims(model)
+            claims = collect_model_claims_cached(model, cache)
             aggregate.add_all(claims)
             model.property_claims = aggregate
     
@@ -90,12 +101,36 @@ class DbtProject(DbtProjectAbstract):
     def refresh(self):
         if not os.path.exists(self.root_folder):
             raise FileNotFoundError("Folder not found: %s" % self.root_folder)
+
+        total_timer = Timer()
+        self.load_metrics = LoadMetrics()
+
         try:
-            with open(self.root_folder / 'dbt_project.yml', 'r', encoding='utf-8') as f:
-                self.parse_dbt_project(f.read())
-            self.load_models()
-            self.populate_graph()
-            self.collect_property_claims()
+            with total_timer:
+                # Parse dbt_project.yml
+                with Timer() as t:
+                    with open(self.root_folder / 'dbt_project.yml', 'r', encoding='utf-8') as f:
+                        self.parse_dbt_project(f.read())
+                self.load_metrics.parse_dbt_project_yml_ms = t.elapsed_ms
+
+                # Load models
+                with Timer() as t:
+                    self.load_models()
+                self.load_metrics.load_models_ms = t.elapsed_ms
+
+                # Build graph
+                with Timer() as t:
+                    self.populate_graph()
+                self.load_metrics.populate_graph_ms = t.elapsed_ms
+
+                # Collect property claims
+                with Timer() as t:
+                    self.collect_property_claims()
+                self.load_metrics.collect_property_claims_ms = t.elapsed_ms
+
+            self.load_metrics.total_load_ms = total_timer.elapsed_ms
+            self.load_metrics.model_count = len(self.models)
+
         except FileNotFoundError:
             raise FileNotFoundError("dbt folder is present, but dbt_project.yml is not found: %s" % self.root_folder)
 
@@ -197,6 +232,7 @@ class DbtProject(DbtProjectAbstract):
                 )
         self.fall_back_to_filename = fall_back_to_filename
         self.root_folder = project_path
+        self.load_metrics = None
         self.refresh()
 
         

{dbt_tui-0.1.0 → dbt_tui-0.2.0}/src/dbtui/backend/property_discovery.py

@@ -5,6 +5,9 @@ This module provides functions to discover and collect property claims from:
 - dbt_project.yml (project-level configurations)
 - schema.yml files (model-specific properties and configs)
 - model SQL files (inline config() calls)
+
+Performance note: Use PropertyDiscoveryCache when collecting claims for multiple models
+to avoid re-reading and re-parsing YAML files.
 """
 
 from pathlib import Path
@@ -12,11 +15,124 @@ from typing import TYPE_CHECKING
 import yaml
 import re
 import logging
+from dataclasses import dataclass, field
 
 from .property_claim import PropertyClaim
 
 if TYPE_CHECKING:
     from .model import DbtModel
+    from .project import DbtProject
+
+
+@dataclass
+class PropertyDiscoveryCache:
+    """
+    Cache for parsed YAML files to avoid re-reading during property collection.
+
+    This dramatically improves performance when collecting properties for many models,
+    as each file is only read and parsed once.
+    """
+    project_path: Path = None
+    dbt_project_data: dict = field(default_factory=dict)
+    schema_files_by_dir: dict = field(default_factory=dict)
+    schema_data_by_path: dict = field(default_factory=dict)
+    models_by_name_in_schemas: dict = field(default_factory=dict)
+    _initialized: bool = False
+
+    def initialize(self, project: 'DbtProject') -> None:
+        """
+        Initialize the cache by reading all relevant files once.
+
+        Args:
+            project: The DbtProject to cache data for
+        """
+        if self._initialized:
+            return
+
+        self.project_path = project.root_folder
+
+        # 1. Parse dbt_project.yml once
+        project_file = self.project_path / "dbt_project.yml"
+        if project_file.exists():
+            try:
+                self.dbt_project_data = yaml.safe_load(project_file.read_text()) or {}
+            except Exception as e:
+                logging.warning(f"Failed to parse {project_file}: {e}")
+                self.dbt_project_data = {}
+
+        # 2. Find and parse all schema files in the project
+        self._discover_schema_files(project)
+
+        # 3. Build index of models in schema files
+        self._index_models_in_schemas()
+
+        self._initialized = True
+
+    def _discover_schema_files(self, project: 'DbtProject') -> None:
+        """Find all schema.yml files in the project and parse them."""
+        seen_paths: set[Path] = set()
+
+        for models_path in project.full_models_paths:
+            if not models_path.exists():
+                continue
+            for root, _, files in models_path.walk():
+                root_path = Path(root)
+                for ext in ("yml", "yaml"):
+                    for schema_file in root_path.glob(f"*.{ext}"):
+                        stem = schema_file.stem.lower()
+                        if stem in ("schema", "models", "_schema", "_models"):
+                            abs_path = schema_file.resolve()
+                            if abs_path in seen_paths:
+                                continue
+                            seen_paths.add(abs_path)
+
+                            # Track by directory for path-based lookup
+                            if root_path not in self.schema_files_by_dir:
+                                self.schema_files_by_dir[root_path] = []
+                            self.schema_files_by_dir[root_path].append(schema_file)
+
+                            # Parse and cache the content
+                            try:
+                                data = yaml.safe_load(schema_file.read_text()) or {}
+                                self.schema_data_by_path[schema_file] = data
+                            except Exception as e:
+                                logging.debug(f"Failed to parse {schema_file}: {e}")
+                                self.schema_data_by_path[schema_file] = {}
+
+    def _index_models_in_schemas(self) -> None:
+        """Build an index of model names to their schema file definitions."""
+        for schema_path, data in self.schema_data_by_path.items():
+            models_list = data.get("models", [])
+            if not isinstance(models_list, list):
+                continue
+            for model_def in models_list:
+                if not isinstance(model_def, dict):
+                    continue
+                model_name = model_def.get("name")
+                if model_name:
+                    if model_name not in self.models_by_name_in_schemas:
+                        self.models_by_name_in_schemas[model_name] = []
+                    self.models_by_name_in_schemas[model_name].append((schema_path, model_def))
+
+    def get_schema_files_for_model(self, model: 'DbtModel') -> list[Path]:
+        """Get schema files relevant to a model (in its directory and ancestors)."""
+        result: list[Path] = []
+        model_dir = model.file_path_full.parent.resolve()
+        project_dir = self.project_path.resolve()
+
+        current = model_dir
+        while current >= project_dir:
+            if current in self.schema_files_by_dir:
+                result.extend(self.schema_files_by_dir[current])
+            if current == project_dir:
+                break
+            current = current.parent
+
+        return result
+
+    def get_model_definitions(self, model_name: str) -> list[tuple[Path, dict]]:
+        """Get all schema definitions for a model by name."""
+        return self.models_by_name_in_schemas.get(model_name, [])
 
 
 def get_model_path_parts(
@@ -459,3 +575,176 @@ def get_effective_properties(
     claims = collect_model_claims(model)
     resolved = resolve_property_precedence(claims)
     return {name: claim.value for name, claim in resolved.items()}
+
+
+# ============================================================================
+# Cached versions for performance when processing multiple models
+# ============================================================================
+
+
+def collect_project_configs_cached(
+    model: 'DbtModel',
+    cache: PropertyDiscoveryCache,
+) -> list[PropertyClaim]:
+    """
+    Collect property claims from dbt_project.yml using cached data.
+
+    Args:
+        model: The DbtModel instance to collect configs for
+        cache: PropertyDiscoveryCache with pre-parsed dbt_project.yml
+
+    Returns:
+        List of PropertyClaim objects from dbt_project.yml
+    """
+    claims: list[PropertyClaim] = []
+    project_path = model.project.root_folder
+    model_path = model.file_path_full
+    project_file = project_path / "dbt_project.yml"
+
+    data = cache.dbt_project_data
+    if not data:
+        return claims
+
+    models = data.get("models", {})
+    if not models:
+        return claims
+
+    model_parts = get_model_path_parts(model_path, project_path)
+    package_name = data.get("name", "")
+
+    def walk(node, path_index: int, yaml_path: str, parts_matched: list[str]):
+        if not isinstance(node, dict):
+            return
+
+        for k, v in node.items():
+            if isinstance(k, str) and k.startswith("+"):
+                prop_name = k[1:]
+                effective = parts_matched == model_parts[:len(parts_matched)]
+                claims.append(
+                    PropertyClaim(
+                        source_type="dbt_project.yml",
+                        source_path=project_file,
+                        model=model,
+                        name=prop_name,
+                        value=v,
+                        yaml_path=yaml_path,
+                        effective=effective,
+                        kind="config",
+                    )
+                )
+
+        if path_index < len(model_parts):
+            next_key = model_parts[path_index]
+            if next_key in node:
+                walk(
+                    node[next_key],
+                    path_index + 1,
+                    f"{yaml_path}.{next_key}",
+                    parts_matched + [next_key]
+                )
+
+    if package_name in models:
+        walk(models[package_name], 0, f"models.{package_name}", [])
+
+    for k, v in models.items():
+        if isinstance(k, str) and k.startswith("+"):
+            prop_name = k[1:]
+            claims.append(
+                PropertyClaim(
+                    source_type="dbt_project.yml",
+                    source_path=project_file,
+                    model=model,
+                    name=prop_name,
+                    value=v,
+                    yaml_path="models",
+                    effective=True,
+                    kind="config",
+                )
+            )
+
+    return claims
+
+
+def collect_schema_properties_cached(
+    model: 'DbtModel',
+    cache: PropertyDiscoveryCache,
+) -> list[PropertyClaim]:
+    """
+    Collect property claims from schema.yml files using cached data.
+
+    Uses the pre-built model name index for fast lookup instead of
+    walking filesystem and parsing YAML files.
+
+    Args:
+        model: The DbtModel instance to collect properties for
+        cache: PropertyDiscoveryCache with pre-parsed schema files
+
+    Returns:
+        List of PropertyClaim objects from schema files
+    """
+    claims: list[PropertyClaim] = []
+
+    # Use the model name index for fast lookup
+    model_defs = cache.get_model_definitions(model.name)
+
+    for schema_path, model_def in model_defs:
+        for key, value in model_def.items():
+            if key == "name":
+                continue
+            elif key == "config":
+                if isinstance(value, dict):
+                    for ck, cv in value.items():
+                        claims.append(
+                            PropertyClaim(
+                                source_type="schema.yml",
+                                source_path=schema_path,
+                                model=model,
+                                name=ck,
+                                value=cv,
+                                kind="config",
+                            )
+                        )
+            else:
+                claims.append(
+                    PropertyClaim(
+                        source_type="schema.yml",
+                        source_path=schema_path,
+                        model=model,
+                        name=key,
+                        value=value,
+                        kind="property",
+                    )
+                )
+
+    return claims
+
+
+def collect_model_claims_cached(
+    model: 'DbtModel',
+    cache: PropertyDiscoveryCache,
+) -> list[PropertyClaim]:
+    """
+    Collect all property claims for a model using cached YAML data.
+
+    This is the cached version of collect_model_claims that uses pre-parsed
+    YAML files from PropertyDiscoveryCache for dramatically better performance.
+
+    Args:
+        model: The DbtModel instance to collect all properties for
+        cache: PropertyDiscoveryCache with pre-parsed YAML files
+
+    Returns:
+        List of all PropertyClaim objects for the model
+    """
+    claims: list[PropertyClaim] = []
+
+    # 1. Collect from dbt_project.yml (using cached data)
+    claims.extend(collect_project_configs_cached(model, cache))
+
+    # 2. Collect from schema.yml files (using cached data)
+    claims.extend(collect_schema_properties_cached(model, cache))
+
+    # 3. Collect from model SQL file (no caching needed - already parsed in model)
+    claims.extend(collect_sql_configs(model))
+
+    return claims

dbt_tui-0.2.0/src/dbtui/frontend/common/timing.py

@@ -0,0 +1,66 @@
+"""
+Timing utilities for frontend performance debugging.
+
+Set DBTUI_TIMING=1 environment variable to enable timing output.
+"""
+
+import os
+import time
+import logging
+from contextlib import contextmanager
+from functools import wraps
+
+# Check if timing is enabled via environment variable
+TIMING_ENABLED = os.environ.get('DBTUI_TIMING', '').lower() in ('1', 'true', 'yes')
+
+# Set up a dedicated logger for timing
+timing_logger = logging.getLogger('dbtui.timing')
+if TIMING_ENABLED:
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+    timing_logger.setLevel(logging.INFO)
+
+
+@contextmanager
+def timed_block(name: str):
+    """Context manager to time a block of code and log the result."""
+    start = time.perf_counter()
+    yield
+    elapsed_ms = (time.perf_counter() - start) * 1000
+    timing_logger.info(f"{name}: {elapsed_ms:.1f}ms")
+
+
+def timed(func):
+    """Decorator to time a function and log the result."""
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start = time.perf_counter()
+        result = func(*args, **kwargs)
+        elapsed_ms = (time.perf_counter() - start) * 1000
+        timing_logger.info(f"{func.__qualname__}: {elapsed_ms:.1f}ms")
+        return result
+    return wrapper
+
+
+class TimingContext:
+    """Accumulates timing for multiple operations."""
+
+    def __init__(self, name: str):
+        self.name = name
+        self.timings: dict[str, float] = {}
+
+    def record(self, step: str, elapsed_ms: float):
+        self.timings[step] = elapsed_ms
+
+    @contextmanager
+    def step(self, step_name: str):
+        start = time.perf_counter()
+        yield
+        self.timings[step_name] = (time.perf_counter() - start) * 1000
+
+    def log(self):
+        if not TIMING_ENABLED:
+            return
+        total = sum(self.timings.values())
+        timing_logger.info(f"{self.name} total: {total:.1f}ms")
+        for step, ms in self.timings.items():
+            timing_logger.info(f"  - {step}: {ms:.1f}ms")

{dbt_tui-0.1.0 → dbt_tui-0.2.0}/src/dbtui/frontend/main.py

@@ -14,6 +14,7 @@ from .new_model import NewModel
 
 from ..common import dbtuiCache, load_cache, save_cache, NonePathException
 from .common import DbtuiScreen, DbtProject, DbtModel
+from .common.timing import TimingContext
 import logging
 
 @dataclass
@@ -44,8 +45,12 @@ class dbtuiFrontend(App):
     screen_stack: list[DbtuiScreen]
     external_editor_command: reactive[str] = reactive('vi')
 
+    # For debounced save
+    _save_timer = None
+    _SAVE_DEBOUNCE_SECONDS = 0.5  # Save at most every 500ms
+
     def watch_external_editor_command(self, old_value: str, new_value: str):
-        self.save_context()
+        self.save_context_debounced()
 
     project: reactive[DbtProject|None] = reactive(None, always_update=True, init=True)
 
@@ -55,7 +60,7 @@ class dbtuiFrontend(App):
         return project
 
     def on_project_change(self, project: DbtProject|None):
-        self.save_context()
+        self.save_context_debounced()
         for screen in self.screen_stack:
             if not screen.id == '_default':
                 screen.on_project_change(project)
@@ -72,12 +77,20 @@ class dbtuiFrontend(App):
         return model
     
     def on_model_change(self, model: DbtModel|None):
-        self.save_context()
+        timing = TimingContext("on_model_change")
+
+        with timing.step("save_context_debounced"):
+            self.save_context_debounced()
+
         for screen in self.screen_stack:
             if not screen.id == '_default':
-                screen.on_model_change(model)
-        self.push_screen('model_view')
-        pass
+                with timing.step(f"screen.{screen.__class__.__name__}.on_model_change"):
+                    screen.on_model_change(model)
+
+        with timing.step("push_screen"):
+            self.push_screen('model_view')
+
+        timing.log()
     
     def watch_model(self, old_model: DbtModel|None, new_model: DbtModel|None):
         
@@ -116,11 +129,26 @@ class dbtuiFrontend(App):
         self.external_editor_command = cache.external_editor_command
         
     def save_context(self):
+        """Save context immediately (blocking)."""
         save_cache(
             project_path=self.project.root_folder if isinstance(self.project, DbtProject) else None,
             model_name=self.model.name if isinstance(self.model, DbtModel) else None,
             external_editor_command=self.external_editor_command,
         )
+
+    def save_context_debounced(self):
+        """Schedule a debounced save - coalesces rapid changes into one save."""
+        if self._save_timer is not None:
+            self._save_timer.stop()
+        self._save_timer = self.set_timer(
+            self._SAVE_DEBOUNCE_SECONDS,
+            self._do_debounced_save
+        )
+
+    def _do_debounced_save(self):
+        """Callback for debounced save timer."""
+        self._save_timer = None
+        self.save_context()
     
 
     def on_mount(self):

{dbt_tui-0.1.0 → dbt_tui-0.2.0}/src/dbtui/frontend/model_tree/model_tree.py

@@ -6,6 +6,7 @@ from textual.binding import Binding
 from textual.events import Key
 
 from ..common import DbtuiScreen, DbtModel, DbtProject
+from ..common.timing import TimingContext
 if TYPE_CHECKING:
     from ..main import dbtuiFrontend
 
@@ -73,20 +74,44 @@ class ModelTree(DbtuiScreen):
         if not model:
             self.app.push_screen('model_search')
             return
+
+        timing = TimingContext("ModelTree.on_model_change")
+
         # model_content
-        model_content = self.get_widget_by_id('model_content')
+        with timing.step("get_widget(model_content)"):
+            model_content = self.get_widget_by_id('model_content')
         assert isinstance(model_content, TextArea)
-        model_content.clear()
-        model_content.load_text(model.text)
+
+        with timing.step("model_content.clear"):
+            model_content.clear()
+
+        with timing.step("model.text (file read)"):
+            text = model.text
+
+        with timing.step("model_content.load_text"):
+            model_content.load_text(text)
+
         # parents
-        parents = self.get_widget_by_id('parents')
+        with timing.step("get_widget(parents)"):
+            parents = self.get_widget_by_id('parents')
         assert isinstance(parents, ParentsList)
-        parents.on_model_change(model)
+
+        with timing.step("parents.on_model_change"):
+            parents.on_model_change(model)
+
         # children
-        children = self.get_widget_by_id('children')
+        with timing.step("get_widget(children)"):
+            children = self.get_widget_by_id('children')
         assert isinstance(children, ChildrenList)
-        children.on_model_change(model)
-        self.recompose()
+
+        with timing.step("children.on_model_change"):
+            children.on_model_change(model)
+
+        # Note: recompose() removed - unnecessary since:
+        # - TextArea is updated via load_text()
+        # - ListViews are updated via clear() and append()
+
+        timing.log()
 
     def on_project_change(self, project: DbtProject | None):
         # When project changes, redirect to model search