sqlmesh 0.186.2.dev11__py3-none-any.whl → 0.186.2.dev13__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.186.2.dev11'
-__version_tuple__ = version_tuple = (0, 186, 2, 'dev11')
+__version__ = version = '0.186.2.dev13'
+__version_tuple__ = version_tuple = (0, 186, 2, 'dev13')

sqlmesh/core/loader.py

@@ -4,7 +4,6 @@ import abc
 import glob
 import itertools
 import linecache
-import logging
 import os
 import re
 import typing as t
@@ -12,6 +11,7 @@ from collections import Counter, defaultdict
 from dataclasses import dataclass
 from pathlib import Path
 from pydantic import ValidationError
+import concurrent.futures
 
 from sqlglot.errors import SqlglotError
 from sqlglot import exp
@@ -19,6 +19,7 @@ from sqlglot.helper import subclasses
 
 from sqlmesh.core import constants as c
 from sqlmesh.core.audit import Audit, ModelAudit, StandaloneAudit, load_multiple_audits
+from sqlmesh.core.console import Console
 from sqlmesh.core.dialect import parse
 from sqlmesh.core.environment import EnvironmentStatements
 from sqlmesh.core.linter.rule import Rule
@@ -28,7 +29,6 @@ from sqlmesh.core.metric import Metric, MetricMeta, expand_metrics, load_metric_
 from sqlmesh.core.model import (
     Model,
     ModelCache,
-    SeedModel,
     create_external_model,
     load_sql_based_models,
 )
@@ -41,6 +41,7 @@ from sqlmesh.utils.errors import ConfigError
 from sqlmesh.utils.jinja import JinjaMacroRegistry, MacroExtractor
 from sqlmesh.utils.metaprogramming import import_python_file
 from sqlmesh.utils.pydantic import validation_error_message
+from sqlmesh.utils.process import create_process_pool_executor
 from sqlmesh.utils.yaml import YAML, load as yaml_load
 
 
@@ -48,8 +49,6 @@ if t.TYPE_CHECKING:
     from sqlmesh.core.context import GenericContext
 
 
-logger = logging.getLogger(__name__)
-
 GATEWAY_PATTERN = re.compile(r"gateway:\s*([^\s]+)")
 
 
@@ -73,17 +72,113 @@ class CacheBase(abc.ABC):
         self, target_path: Path, loader: t.Callable[[], t.List[Model]]
     ) -> t.List[Model]:
         """Get or load all models from cache."""
+        pass
+
+    @abc.abstractmethod
+    def put(self, models: t.List[Model], path: Path) -> bool:
+        """Store models in the cache associated with the given path.
+
+        Args:
+            models: List of models to cache
+            path: File path to associate with the cached models
+
+        Returns:
+            True if the models were successfully cached,
+            False otherwise (empty list, not a list, unsupported model types)
+        """
+        pass
+
+    @abc.abstractmethod
+    def get(self, path: Path) -> t.List[Model]:
+        """Retrieve models from the cache for a given path.
+
+        Args:
+            path: File path to look up in the cache
+
+        Returns:
+            List of cached models associated with the path, an empty list if no cache entry exists
+        """
+        pass
+
+
+_defaults: t.Optional[t.Dict[str, t.Any]] = None
+_cache: t.Optional[CacheBase] = None
+_config_essentials: t.Optional[t.Dict[str, t.Any]] = None
+_selected_gateway: t.Optional[str] = None
+
+
+def _init_model_defaults(
+    config_essentials: t.Dict[str, t.Any],
+    selected_gateway: t.Optional[str],
+    model_loading_defaults: t.Optional[t.Dict[str, t.Any]] = None,
+    cache: t.Optional[CacheBase] = None,
+    console: t.Optional[Console] = None,
+) -> None:
+    global _defaults, _cache, _config_essentials, _selected_gateway
+    _defaults = model_loading_defaults
+    _cache = cache
+    _config_essentials = config_essentials
+    _selected_gateway = selected_gateway
+
+    # Set the console passed from the parent process
+    if console is not None:
+        from sqlmesh.core.console import set_console
+
+        set_console(console)
+
+
+def load_sql_models(path: Path) -> t.List[Model]:
+    assert _defaults
+    assert _cache
+
+    with open(path, "r", encoding="utf-8") as file:
+        expressions = parse(file.read(), default_dialect=_defaults["dialect"])
+    models = load_sql_based_models(expressions, path=Path(path).absolute(), **_defaults)
+
+    return [] if _cache.put(models, path) else models
+
+
+def get_variables(gateway_name: t.Optional[str] = None) -> t.Dict[str, t.Any]:
+    assert _config_essentials
+
+    gateway_name = gateway_name or _selected_gateway
+
+    try:
+        gateway = _config_essentials["gateways"].get(gateway_name)
+    except ConfigError:
+        from sqlmesh.core.console import get_console
+
+        get_console().log_warning(
+            f"Gateway '{gateway_name}' not found in project '{_config_essentials['project']}'."
+        )
+        gateway = None
+
+    return {
+        **_config_essentials["variables"],
+        **(gateway.variables if gateway else {}),
+        c.GATEWAY: gateway_name,
+    }
 
 
 class Loader(abc.ABC):
     """Abstract base class to load macros and models for a context"""
 
     def __init__(self, context: GenericContext, path: Path) -> None:
+        from sqlmesh.core.console import get_console
+
         self._path_mtimes: t.Dict[Path, float] = {}
         self.context = context
         self.config_path = path
         self.config = self.context.configs[self.config_path]
         self._variables_by_gateway: t.Dict[str, t.Dict[str, t.Any]] = {}
+        self._console = get_console()
+
+        self.config_essentials = {
+            "project": self.config.project,
+            "variables": self.config.variables,
+            "gateways": self.config.gateways,
+        }
+        _init_model_defaults(self.config_essentials, self.context.selected_gateway)
 
     def load(self) -> LoadedProject:
         """
@@ -243,19 +338,21 @@ class Loader(abc.ABC):
                         for row in YAML().load(file.read())
                     ]
             except Exception as ex:
-                self._raise_failed_to_load_model_error(path, ex)
-                raise
+                raise ConfigError(self._failed_to_load_model_error(path, ex))
 
         for path in paths_to_load:
             self._track_file(path)
 
             external_models = cache.get_or_load_models(path, lambda: _load(path))
+
             # external models with no explicit gateway defined form the base set
             for model in external_models:
                 if model.gateway is None:
                     if model.fqn in models:
-                        self._raise_failed_to_load_model_error(
-                            path, f"Duplicate external model name: '{model.name}'."
+                        raise ConfigError(
+                            self._failed_to_load_model_error(
+                                path, f"Duplicate external model name: '{model.name}'."
+                            )
                         )
                     models[model.fqn] = model
 
@@ -264,8 +361,10 @@ class Loader(abc.ABC):
                 for model in external_models:
                     if model.gateway == gateway:
                         if model.fqn in models and models[model.fqn].gateway == gateway:
-                            self._raise_failed_to_load_model_error(
-                                path, f"Duplicate external model name: '{model.name}'."
+                            raise ConfigError(
+                                self._failed_to_load_model_error(
+                                    path, f"Duplicate external model name: '{model.name}'."
+                                )
                             )
                         models.update({model.fqn: model})
 
@@ -352,33 +451,11 @@ class Loader(abc.ABC):
         """Project file to track for modifications"""
         self._path_mtimes[path] = path.stat().st_mtime
 
-    def _get_variables(self, gateway_name: t.Optional[str] = None) -> t.Dict[str, t.Any]:
-        gateway_name = gateway_name or self.context.selected_gateway
-
-        if gateway_name not in self._variables_by_gateway:
-            try:
-                gateway = self.config.get_gateway(gateway_name)
-            except ConfigError:
-                from sqlmesh.core.console import get_console
-
-                get_console().log_warning(
-                    f"Gateway '{gateway_name}' not found in project '{self.config.project}'."
-                )
-                gateway = None
-
-            self._variables_by_gateway[gateway_name] = {
-                **self.config.variables,
-                **(gateway.variables if gateway else {}),
-                c.GATEWAY: gateway_name,
-            }
-
-        return self._variables_by_gateway[gateway_name]
-
-    def _raise_failed_to_load_model_error(self, path: Path, error: t.Union[str, Exception]) -> None:
+    def _failed_to_load_model_error(self, path: Path, error: t.Union[str, Exception]) -> str:
         base_message = f"Failed to load model definition at '{path}':"
         if isinstance(error, ValidationError):
-            raise ConfigError(validation_error_message(error, base_message))
-        raise ConfigError(f"{base_message}\n  {error}")
+            return validation_error_message(error, base_message)
+        return f"{base_message}\n  {error}"
 
 
 class SqlMeshLoader(Loader):
@@ -442,14 +519,15 @@ class SqlMeshLoader(Loader):
         audits into a Dict and creates the dag
         """
         cache = SqlMeshLoader._Cache(self, self.config_path)
-        sql_models = self._load_sql_models(macros, jinja_macros, audits, signals, cache)
+
+        sql_models = self._load_sql_models(macros, jinja_macros, audits, signals, cache, gateway)
         external_models = self._load_external_models(audits, cache, gateway)
         python_models = self._load_python_models(macros, jinja_macros, audits, signals)
 
         all_model_names = list(sql_models) + list(external_models) + list(python_models)
         duplicates = [name for name, count in Counter(all_model_names).items() if count > 1]
         if duplicates:
-            raise ValueError(f"Duplicate model name(s) found: {', '.join(duplicates)}.")
+            raise ConfigError(f"Duplicate model name(s) found: {', '.join(duplicates)}.")
 
         return UniqueKeyDict("models", **sql_models, **external_models, **python_models)
 
@@ -460,9 +538,12 @@ class SqlMeshLoader(Loader):
         audits: UniqueKeyDict[str, ModelAudit],
         signals: UniqueKeyDict[str, signal],
         cache: CacheBase,
+        gateway: t.Optional[str],
     ) -> UniqueKeyDict[str, Model]:
         """Loads the sql models into a Dict"""
         models: UniqueKeyDict[str, Model] = UniqueKeyDict("models")
+        paths: t.Set[Path] = set()
+        cached_paths: UniqueKeyDict[Path, t.List[Model]] = UniqueKeyDict("cached_paths")
 
         for path in self._glob_paths(
             self.config_path / c.MODELS,
@@ -473,48 +554,63 @@ class SqlMeshLoader(Loader):
                 continue
 
             self._track_file(path)
+            paths.add(path)
+            if cached_models := cache.get(path):
+                cached_paths[path] = cached_models
 
-            def _load() -> t.List[Model]:
-                try:
-                    with open(path, "r", encoding="utf-8") as file:
-                        expressions = parse(
-                            file.read(), default_dialect=self.config.model_defaults.dialect
-                        )
-
-                    return load_sql_based_models(
-                        expressions,
-                        self._get_variables,
-                        defaults=self.config.model_defaults.dict(),
-                        macros=macros,
-                        jinja_macros=jinja_macros,
-                        audit_definitions=audits,
-                        default_audits=self.config.model_defaults.audits,
-                        path=Path(path).absolute(),
-                        module_path=self.config_path,
-                        dialect=self.config.model_defaults.dialect,
-                        time_column_format=self.config.time_column_format,
-                        physical_schema_mapping=self.config.physical_schema_mapping,
-                        project=self.config.project,
-                        default_catalog=self.context.default_catalog,
-                        infer_names=self.config.model_naming.infer_names,
-                        signal_definitions=signals,
-                        default_catalog_per_gateway=self.context.default_catalog_per_gateway,
-                    )
-                except Exception as ex:
-                    self._raise_failed_to_load_model_error(path, ex)
-                    raise
-
-            for model in cache.get_or_load_models(path, _load):
-                if model.fqn in models:
-                    self._raise_failed_to_load_model_error(
-                        path, f"Duplicate SQL model name: '{model.name}'."
-                    )
+        for path, cached_models in cached_paths.items():
+            paths.remove(path)
+            for model in cached_models:
                 if model.enabled:
                     models[model.fqn] = model
 
-                if isinstance(model, SeedModel):
-                    seed_path = model.seed_path
-                    self._track_file(seed_path)
+        if paths:
+            model_loading_defaults = dict(
+                get_variables=get_variables,
+                defaults=self.config.model_defaults.dict(),
+                macros=macros,
+                jinja_macros=jinja_macros,
+                audit_definitions=audits,
+                default_audits=self.config.model_defaults.audits,
+                module_path=self.config_path,
+                dialect=self.config.model_defaults.dialect,
+                time_column_format=self.config.time_column_format,
+                physical_schema_mapping=self.config.physical_schema_mapping,
+                project=self.config.project,
+                default_catalog=self.context.default_catalog,
+                infer_names=self.config.model_naming.infer_names,
+                signal_definitions=signals,
+                default_catalog_per_gateway=self.context.default_catalog_per_gateway,
+            )
+
+            with create_process_pool_executor(
+                initializer=_init_model_defaults,
+                initargs=(
+                    self.config_essentials,
+                    gateway,
+                    model_loading_defaults,
+                    cache,
+                    self._console,
+                ),
+                max_workers=c.MAX_FORK_WORKERS,
+            ) as pool:
+                futures_to_paths = {pool.submit(load_sql_models, path): path for path in paths}
+                for future in concurrent.futures.as_completed(futures_to_paths):
+                    path = futures_to_paths[future]
+                    try:
+                        loaded = future.result()
+                        for model in loaded or cache.get(path):
+                            if model.fqn in models:
+                                raise ConfigError(
+                                    self._failed_to_load_model_error(
+                                        path, f"Duplicate SQL model name: '{model.name}'."
+                                    )
+                                )
+                            elif model.enabled:
+                                model._path = path
+                                models[model.fqn] = model
+                    except Exception as ex:
+                        raise ConfigError(self._failed_to_load_model_error(path, ex))
 
         return models
 
@@ -548,7 +644,7 @@ class SqlMeshLoader(Loader):
                     registered |= new
                     for name in new:
                         for model in registry[name].models(
-                            self._get_variables,
+                            get_variables,
                             path=path,
                             module_path=self.config_path,
                             defaults=self.config.model_defaults.dict(),
@@ -566,7 +662,7 @@ class SqlMeshLoader(Loader):
                             if model.enabled:
                                 models[model.fqn] = model
                 except Exception as ex:
-                    self._raise_failed_to_load_model_error(path, ex)
+                    raise ConfigError(self._failed_to_load_model_error(path, ex))
 
         finally:
             model_registry._dialect = None
@@ -616,7 +712,7 @@ class SqlMeshLoader(Loader):
         """Loads all the model audits."""
         audits_by_name: UniqueKeyDict[str, Audit] = UniqueKeyDict("audits")
         audits_max_mtime: t.Optional[float] = None
-        variables = self._get_variables()
+        variables = get_variables()
 
         for path in self._glob_paths(
             self.config_path / c.AUDITS,
@@ -692,7 +788,7 @@ class SqlMeshLoader(Loader):
                 module_path=self.config_path,
                 jinja_macro_references=None,
                 macros=macros,
-                variables=self._get_variables(),
+                variables=get_variables(),
                 path=self.config_path,
             )
 
@@ -727,7 +823,7 @@ class SqlMeshLoader(Loader):
             gateway_line = GATEWAY_PATTERN.search(source)
             gateway = YAML().load(gateway_line.group(0))["gateway"] if gateway_line else None
 
-        contents = yaml_load(source, variables=self._get_variables(gateway))
+        contents = yaml_load(source, variables=get_variables(gateway))
 
         for test_name, value in contents.items():
             model_test_metadata[test_name] = ModelTestMetadata(
@@ -785,11 +881,30 @@ class SqlMeshLoader(Loader):
                 self._model_cache_entry_id(target_path),
                 loader=loader,
             )
+
             for model in models:
                 model._path = target_path
 
             return models
 
+        def put(self, models: t.List[Model], path: Path) -> bool:
+            return self._model_cache.put(
+                models,
+                self._cache_entry_name(path),
+                self._model_cache_entry_id(path),
+            )
+
+        def get(self, path: Path) -> t.List[Model]:
+            models = self._model_cache.get(
+                self._cache_entry_name(path),
+                self._model_cache_entry_id(path),
+            )
+
+            for model in models:
+                model._path = path
+
+            return models
+
         def _cache_entry_name(self, target_path: Path) -> str:
             return "__".join(target_path.relative_to(self.config_path).parts).replace(
                 target_path.suffix, ""

sqlmesh/core/model/cache.py

@@ -1,9 +1,7 @@
 from __future__ import annotations
 
 import logging
-import multiprocessing as mp
 import typing as t
-from concurrent.futures import ProcessPoolExecutor
 from pathlib import Path
 
 from sqlglot import exp
@@ -15,7 +13,7 @@ from sqlmesh.core import constants as c
 from sqlmesh.core.model.definition import ExternalModel, Model, SqlModel, _Model
 from sqlmesh.utils.cache import FileCache
 from sqlmesh.utils.hashing import crc32
-from sqlmesh.utils.windows import IS_WINDOWS
+from sqlmesh.utils.process import PoolExecutor, create_process_pool_executor
 
 from dataclasses import dataclass
 
@@ -46,12 +44,10 @@ class ModelCache:
         self, name: str, entry_id: str = "", *, loader: t.Callable[[], t.List[Model]]
     ) -> t.List[Model]:
         """Returns an existing cached model definition or loads and caches a new one.
-
         Args:
             name: The name of the entry.
             entry_id: The unique entry identifier. Used for cache invalidation.
             loader: Used to load a new model definition when no cached instance was found.
-
         Returns:
             The model definition.
         """
@@ -66,9 +62,22 @@ class ModelCache:
                 model.full_depends_on
 
             self._file_cache.put(name, entry_id, value=models)
-
         return models
 
+    def put(self, models: t.List[Model], name: str, entry_id: str = "") -> bool:
+        if models and isinstance(seq_get(models, 0), (SqlModel, ExternalModel)):
+            # make sure we preload full_depends_on
+            for model in models:
+                model.full_depends_on
+
+            self._file_cache.put(name, entry_id, value=models)
+            return True
+
+        return False
+
+    def get(self, name: str, entry_id: str = "") -> t.List[Model]:
+        return self._file_cache.get(name, entry_id) or []
+
 
 @dataclass
 class OptimizedQueryCacheEntry:
@@ -149,11 +158,8 @@ class OptimizedQueryCache:
         return f"{model.name}_{crc32(hash_data)}"
 
 
-def optimized_query_cache_pool(optimized_query_cache: OptimizedQueryCache) -> ProcessPoolExecutor:
-    # fork doesnt work on Windows. ref: https://docs.python.org/3/library/multiprocessing.html#multiprocessing-start-methods
-    context_type = "spawn" if IS_WINDOWS else "fork"
-    return ProcessPoolExecutor(
-        mp_context=mp.get_context(context_type),
+def optimized_query_cache_pool(optimized_query_cache: OptimizedQueryCache) -> PoolExecutor:
+    return create_process_pool_executor(
         initializer=_init_optimized_query_cache,
         initargs=(optimized_query_cache,),
         max_workers=c.MAX_FORK_WORKERS,

sqlmesh/core/model/schema.py

@@ -28,10 +28,7 @@ def update_model_schemas(
     schema = MappingSchema(normalize=False)
     optimized_query_cache: OptimizedQueryCache = OptimizedQueryCache(context_path / c.CACHE)
 
-    if c.MAX_FORK_WORKERS == 1:
-        _update_model_schemas_sequential(dag, models, schema, optimized_query_cache)
-    else:
-        _update_model_schemas_parallel(dag, models, schema, optimized_query_cache)
+    _update_model_schemas(dag, models, schema, optimized_query_cache)
 
 
 def _update_schema_with_model(schema: MappingSchema, model: Model) -> None:
@@ -49,25 +46,7 @@ def _update_schema_with_model(schema: MappingSchema, model: Model) -> None:
             raise
 
 
-def _update_model_schemas_sequential(
-    dag: DAG[str],
-    models: UniqueKeyDict[str, Model],
-    schema: MappingSchema,
-    optimized_query_cache: OptimizedQueryCache,
-) -> None:
-    for name in dag.sorted:
-        model = models.get(name)
-
-        # External models don't exist in the context, so we need to skip them
-        if not model:
-            continue
-
-        model.update_schema(schema)
-        optimized_query_cache.with_optimized_query(model)
-        _update_schema_with_model(schema, model)
-
-
-def _update_model_schemas_parallel(
+def _update_model_schemas(
     dag: DAG[str],
     models: UniqueKeyDict[str, Model],
     schema: MappingSchema,
@@ -107,12 +86,16 @@ def _update_model_schemas_parallel(
 
         while futures:
             for future in as_completed(futures):
-                futures.remove(future)
-                fqn, entry_name, data_hash, metadata_hash, mapping_schema = future.result()
-                model = models[fqn]
-                model._data_hash = data_hash
-                model._metadata_hash = metadata_hash
-                model.set_mapping_schema(mapping_schema)
-                optimized_query_cache.with_optimized_query(model, entry_name)
-                _update_schema_with_model(schema, model)
-                process_models(completed_model=model)
+                try:
+                    futures.remove(future)
+                    fqn, entry_name, data_hash, metadata_hash, mapping_schema = future.result()
+                    model = models[fqn]
+                    model._data_hash = data_hash
+                    model._metadata_hash = metadata_hash
+                    if model.mapping_schema != mapping_schema:
+                        model.set_mapping_schema(mapping_schema)
+                    optimized_query_cache.with_optimized_query(model, entry_name)
+                    _update_schema_with_model(schema, model)
+                    process_models(completed_model=model)
+                except Exception as ex:
+                    raise SchemaError(f"Failed to update model schemas\n\n{ex}")

sqlmesh/core/snapshot/cache.py

@@ -55,20 +55,19 @@ class SnapshotCache:
             for snapshot in loaded_snapshots:
                 snapshots[snapshot.snapshot_id] = snapshot
 
-        if c.MAX_FORK_WORKERS != 1:
-            with optimized_query_cache_pool(self._optimized_query_cache) as executor:
-                for key, entry_name in executor.map(
-                    load_optimized_query,
-                    (
-                        (snapshot.model, s_id)
-                        for s_id, snapshot in snapshots.items()
-                        if snapshot.is_model
-                    ),
-                ):
-                    if entry_name:
-                        self._optimized_query_cache.with_optimized_query(
-                            snapshots[key].model, entry_name
-                        )
+        with optimized_query_cache_pool(self._optimized_query_cache) as executor:
+            for key, entry_name in executor.map(
+                load_optimized_query,
+                (
+                    (snapshot.model, s_id)
+                    for s_id, snapshot in snapshots.items()
+                    if snapshot.is_model
+                ),
+            ):
+                if entry_name:
+                    self._optimized_query_cache.with_optimized_query(
+                        snapshots[key].model, entry_name
+                    )
 
         for snapshot in snapshots.values():
             self._update_node_hash_cache(snapshot)

sqlmesh/dbt/loader.py

@@ -355,6 +355,19 @@ class DbtLoader(Loader):
 
             return models
 
+        def put(self, models: t.List[Model], path: Path) -> bool:
+            return self._model_cache.put(
+                models,
+                self._cache_entry_name(path),
+                self._cache_entry_id(path),
+            )
+
+        def get(self, path: Path) -> t.List[Model]:
+            return self._model_cache.get(
+                self._cache_entry_name(path),
+                self._cache_entry_id(path),
+            )
+
         def _cache_entry_name(self, target_path: Path) -> str:
             try:
                 path_for_name = target_path.absolute().relative_to(

sqlmesh/utils/concurrency.py

@@ -105,16 +105,23 @@ class ConcurrentDAGExecutor(t.Generic[H]):
             self._finished_future.set_result(None)
             return
 
-        skipped_nodes = [node for node, deps in self._unprocessed_nodes.items() if parent in deps]
+        skipped_nodes = {node for node, deps in self._unprocessed_nodes.items() if parent in deps}
 
-        self._skipped_nodes.extend(skipped_nodes)
+        while skipped_nodes:
+            self._skipped_nodes.extend(skipped_nodes)
 
-        for skipped_node in skipped_nodes:
-            self._unprocessed_nodes_num -= 1
-            self._unprocessed_nodes.pop(skipped_node)
+            for skipped_node in skipped_nodes:
+                self._unprocessed_nodes_num -= 1
+                self._unprocessed_nodes.pop(skipped_node)
+
+            skipped_nodes = {
+                node
+                for node, deps in self._unprocessed_nodes.items()
+                if skipped_nodes.intersection(deps)
+            }
 
-        for skipped_node in skipped_nodes:
-            self._skip_next_nodes(skipped_node)
+        if not self._unprocessed_nodes_num:
+            self._finished_future.set_result(None)
 
     def _init_state(self) -> None:
         self._unprocessed_nodes = self.dag.graph

sqlmesh/utils/process.py

@@ -0,0 +1,73 @@
+# mypy: disable-error-code=no-untyped-def
+
+from concurrent.futures import Future, ProcessPoolExecutor
+import typing as t
+import multiprocessing as mp
+from sqlmesh.utils.windows import IS_WINDOWS
+
+
+class SynchronousPoolExecutor:
+    """A mock implementation of the ProcessPoolExecutor for synchronous use.
+
+    This executor runs functions synchronously in the same process, avoiding the issues
+    with forking in test environments or when forking isn't possible (non-posix).
+    """
+
+    def __init__(self, max_workers=None, mp_context=None, initializer=None, initargs=()):
+        if initializer is not None:
+            try:
+                initializer(*initargs)
+            except BaseException as ex:
+                raise RuntimeError(f"Exception in initializer: {ex}")
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.shutdown(wait=True)
+        return False
+
+    def shutdown(self, wait=True, cancel_futures=False):
+        """No-op method to match ProcessPoolExecutor API.
+
+        Since this executor runs synchronously, there are no background processes
+        or resources to shut down and all futures will have completed already.
+        """
+        pass
+
+    def submit(self, fn, *args, **kwargs):
+        """Execute the function synchronously and return a Future with the result."""
+        future = Future()
+        try:
+            result = fn(*args, **kwargs)
+            future.set_result(result)
+        except Exception as e:
+            future.set_exception(e)
+        return future
+
+    def map(self, fn, *iterables, timeout=None, chunksize=1):
+        """Synchronous implementation of ProcessPoolExecutor.map.
+
+        This executes the function for each set of inputs from the iterables in the
+        current process using Python's built-in map, rather than distributing work.
+        """
+        return map(fn, *iterables)
+
+
+PoolExecutor = t.Union[SynchronousPoolExecutor, ProcessPoolExecutor]
+
+
+def create_process_pool_executor(
+    initializer: t.Callable, initargs: t.Tuple, max_workers: t.Optional[int]
+) -> PoolExecutor:
+    if max_workers == 1 or IS_WINDOWS:
+        return SynchronousPoolExecutor(
+            initializer=initializer,
+            initargs=initargs,
+        )
+    return ProcessPoolExecutor(
+        mp_context=mp.get_context("fork"),
+        initializer=initializer,
+        initargs=initargs,
+        max_workers=max_workers,
+    )

{sqlmesh-0.186.2.dev11.dist-info → sqlmesh-0.186.2.dev13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.186.2.dev11
+Version: 0.186.2.dev13
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.186.2.dev11.dist-info → sqlmesh-0.186.2.dev13.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=69xw6bTmi4cW90M-sl3g6wLDgxVcLHcKg1q8GS3lqyA,7466
-sqlmesh/_version.py,sha256=xBlsO_cc3Rn15-v22URdHGhMCdLnw__XWcjdZeZXz4w,530
+sqlmesh/_version.py,sha256=vqg9_bdM0w1PriGacvlThb4jS8xV-36vEdbL-x7PuiI,530
 sqlmesh/magics.py,sha256=JHorAZNtezf123jmTyNSoUC9yHmv-08hMP205PJmmIo,39750
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -18,7 +18,7 @@ sqlmesh/core/context_diff.py,sha256=s9toP17d4tJfOpFH91LcjGK2CM64PVuB0MPkeFgF4hQ,
 sqlmesh/core/dialect.py,sha256=fPTV517IOS0nWMxYznNNQnXkQkFFrPOSKe8ghsQ0o8A,49759
 sqlmesh/core/environment.py,sha256=0TTQf4C4LXwf37fm-dfYv1qyG7vtZfQI5rizgm5g0j0,11442
 sqlmesh/core/lineage.py,sha256=zjB0Zfamo2Fja2r5SSZPMbrEKTXr1WjozZPVqvxdybI,3143
-sqlmesh/core/loader.py,sha256=5ty4qY4qQFQuinhtNRzB-l5yvU4TgdPPW_Y_9VrnS2U,32244
+sqlmesh/core/loader.py,sha256=exNCHmw06-CRBtOk-Hk5Ivw3gCowDWbxcvhMJVxpDd8,35887
 sqlmesh/core/macros.py,sha256=wSKkFDZ2eIvhPENIsQ_7j27zVMHIgT-2juRjGvDeEKM,59620
 sqlmesh/core/node.py,sha256=r4_1MRTPcU0MAlwZ5KzrykLT58B7dkKvHzPCsJmEKD8,14497
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
@@ -84,20 +84,20 @@ sqlmesh/core/metric/__init__.py,sha256=H1HmoD5IwN4YWe9iJXyueLYNmTQFZwok5nSWNJcZI
 sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw9x9M,7201
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
 sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu4,1644
-sqlmesh/core/model/cache.py,sha256=s-aEFuUmUKITaWHvYbOyRarsjsgoA6vfASK-f-MwJ9k,7257
+sqlmesh/core/model/cache.py,sha256=WJdn54xmA3NtX0dlX-DZnYlrWDhj4ycoq_4FD4MK2yg,7502
 sqlmesh/core/model/common.py,sha256=vbcXFhWE22-FeKYAVIlKWvTb-wkeC8XISjU_TIlak0c,16252
 sqlmesh/core/model/decorator.py,sha256=v_sVudNROGrtj5cZ3vgcOFnwPdzwoUGY6REAkQpIMQc,8927
 sqlmesh/core/model/definition.py,sha256=QBwGNL7Jqg8BGgpZfTtKciRJA7axkI5FShJJjnUDQ4w,108645
 sqlmesh/core/model/kind.py,sha256=i6_AMBarXPGPoMdF9kzBkyqS9tu__a2339-BoX7TgCM,36010
 sqlmesh/core/model/meta.py,sha256=osJgfSj6maIIdWAHzDVCkXXL--__gnKvqkzxM6Nuq6g,20779
-sqlmesh/core/model/schema.py,sha256=QzM1cOfukuK3L-cEE9MtxKeYec3x5JvutdZAnczJSiQ,3858
+sqlmesh/core/model/schema.py,sha256=OClbAeA8LpW-IiSiLT_TlDWkEqmIIvSk0GXq8SWI-GQ,3443
 sqlmesh/core/model/seed.py,sha256=ctUHH6CmeEnBVJyIAZznx0xJBm09W7VIZ-76Cfwfq28,5004
 sqlmesh/core/plan/__init__.py,sha256=ctpnu9mK3uTujs0It8qRMqcj9GdObXDnvSgn4rD2zqc,450
 sqlmesh/core/plan/builder.py,sha256=Fd5raxq6r6b_3mOZndiI2U7SmliVYevcrpxWNqRV_OE,38471
 sqlmesh/core/plan/definition.py,sha256=SI3zi2poPkKJKTZogYv60Q7eIOhksaAcFUcu-J2-XqI,13218
 sqlmesh/core/plan/evaluator.py,sha256=RAHJgQoV5Y5SIG_pV2O_Bn0VFlD1yzMC0UrUU6eaB_E,26842
 sqlmesh/core/snapshot/__init__.py,sha256=Z5HTetNCZGoKF1WAKXk4LWL9dc8EC8lwS2O4TtoBmt4,1317
-sqlmesh/core/snapshot/cache.py,sha256=MRYi_koDvp2yeQ5NDGoNFzhuJTw7Vh7mI0oJyeVbqio,4022
+sqlmesh/core/snapshot/cache.py,sha256=bgqCR2hyf6r2A_8QP1EnXFK25gDX37-Zg0YeMuETWxg,3934
 sqlmesh/core/snapshot/categorizer.py,sha256=qGG5870mu1jR2whKisSBH_GFbsyh96RnzJDgkN7DV9g,2482
 sqlmesh/core/snapshot/definition.py,sha256=PKzBLiQ1yIycGbmslEoQyJEwtfnJHsQuY_sl1xkBcvk,84081
 sqlmesh/core/snapshot/evaluator.py,sha256=Y01dkakdX-Jvz3t8DAzJPLVt3eL2DRqz7SabgZ7ME7g,89430
@@ -127,7 +127,7 @@ sqlmesh/dbt/builtin.py,sha256=fyzv4qR1isfnID1oWz-r46oDQvS90DN5mMUEWgapvN8,15153
 sqlmesh/dbt/column.py,sha256=Kvj_N3D_QegcoNQG7VqRiqy-FRfupud-6ilhlykBOgE,1873
 sqlmesh/dbt/common.py,sha256=g4PAL-sEdlnfb_zT-WW3xh-oamyCUoK8dUiDEMVwDWI,7804
 sqlmesh/dbt/context.py,sha256=dxClM-G1IeMbP0v5yxC_va8QNU-uKghW2dBlSZHJOk0,10331
-sqlmesh/dbt/loader.py,sha256=rpK6EUIZQdQ0igdgI7KNM9MIA6TGSTQbZQBxNej9Stc,15588
+sqlmesh/dbt/loader.py,sha256=dXnqs_MmbP5u1CzoCviQpA_i0XCaEvm_L3kJTWOmFJ8,16024
 sqlmesh/dbt/manifest.py,sha256=iKfUWiqydIW3YnX6hqWAzRc-TmhiXfVFS7e29XJaS6E,22906
 sqlmesh/dbt/model.py,sha256=-TqeldZAqW-4g84Xi_16DpUD3atImk_9OHMbATIKYb8,26328
 sqlmesh/dbt/package.py,sha256=WM5Ve6o0QPW3rZxhw1xbr3yX2619I5KtHOKK1qe5mfU,4291
@@ -251,7 +251,7 @@ sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 sqlmesh/utils/__init__.py,sha256=4vN2s-t0p0c9l4erdOFONl_j8StaW6Eljt_SBffMW80,10898
 sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
 sqlmesh/utils/cache.py,sha256=GITQ3sldwG4vlI0wKIDI7yOr1CND1EI3necAmeCEW8g,4743
-sqlmesh/utils/concurrency.py,sha256=HvK_AwVfy3Npzmsthmd73Z5tLhY-Anrb472IkaoRjOw,9041
+sqlmesh/utils/concurrency.py,sha256=EMgEecfPFN9_2Wjz4TUksJngECGse7fnKVcLswkmhAk,9274
 sqlmesh/utils/config.py,sha256=-SjIn7Z0HHnMtjPavQIvFcSaeg86XLsGd3sGIwOKrTE,1941
 sqlmesh/utils/connection_pool.py,sha256=To1zr9d5MQOZ5im09jLQKO4DFmfzH6s_q75cEa1y0PE,11390
 sqlmesh/utils/conversions.py,sha256=Ul8u6TzlKzFz_lVrUzFSzmJBKHUXOYe2e-xmkMXV084,406
@@ -265,11 +265,12 @@ sqlmesh/utils/jinja.py,sha256=6Iob2Pv88L2rCaHyl_V90b5GKsG9hI10Ua_FRWhQIl0,23628
 sqlmesh/utils/metaprogramming.py,sha256=i1rxWi-oxdh1VwKoBh5Qvy0GgHm9GaDZMvIrv3QXGt4,23083
 sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,835
 sqlmesh/utils/pandas.py,sha256=uKI-ygEtMcGDRzXdBu0RkGmMBkYxwu5O2Wabho841Ls,2196
+sqlmesh/utils/process.py,sha256=vpXcpJ1eIbivs7gZdwTKSJI7OS6kKBebnaBzz75ykvs,2428
 sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,11685
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.186.2.dev11.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.186.2.dev13.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=M50OuwjW63dRCS0Dz40Yecgvw9ecLFqizbTa6gv5Nd4,5472
 web/client/dist/assets/Audits-BPlxzqeG.js,sha256=TCx1qCjCGgmUSvH6HoPr5t8ztbRXMlOxrcW4o7hRjlA,1041
@@ -387,8 +388,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=Wd5YxeKbZB7AkrDvqFgvV9ZQlnKYOF1n5W_w2EHE_Zw,2085
-sqlmesh-0.186.2.dev11.dist-info/METADATA,sha256=IOFYQA2ApAqd-uK_CnG_aVdgziSCsIZibxpBmPWsL-o,25524
-sqlmesh-0.186.2.dev11.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-sqlmesh-0.186.2.dev11.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.186.2.dev11.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.186.2.dev11.dist-info/RECORD,,
+sqlmesh-0.186.2.dev13.dist-info/METADATA,sha256=QBfR5Kg6iu6zN8kIiC1cIN6mgeWn77raK94c3frUaq8,25524
+sqlmesh-0.186.2.dev13.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+sqlmesh-0.186.2.dev13.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.186.2.dev13.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.186.2.dev13.dist-info/RECORD,,