squirrels 0.5.0rc0__py3-none-any.whl → 0.5.1__py3-none-any.whl

squirrels/_manifest.py

@@ -1,11 +1,11 @@
 from functools import cached_property
-from typing import Any
+from typing import Literal, Any
 from urllib.parse import urlparse
 from sqlalchemy import Engine, create_engine
 from typing_extensions import Self
 from enum import Enum
 from pydantic import BaseModel, Field, field_validator, model_validator, ValidationInfo, ValidationError
-import yaml, time
+import yaml, time, re
 
 from . import _constants as c, _utils as u
 
@@ -16,10 +16,17 @@ class ProjectVarsConfig(BaseModel, extra="allow"):
     description: str = ""
     major_version: int
 
+    @field_validator("name")
+    @classmethod
+    def validate_name(cls, v: str) -> str:
+        if not re.fullmatch(r"[A-Za-z0-9_-]+", v):
+            raise ValueError("Project name must only contain alphanumeric characters, underscores, and dashes.")
+        return v
+
     @model_validator(mode="after")
     def finalize_label(self) -> Self:
         if self.label == "":
-            self.label = self.name
+            self.label = u.to_title_case(self.name)
         return self
 
 
@@ -39,10 +46,11 @@ class _ConfigWithNameBaseModel(BaseModel):
     name: str
 
 
-class ConnectionType(Enum):
+class ConnectionTypeEnum(Enum):
     SQLALCHEMY = "sqlalchemy"
     CONNECTORX = "connectorx"
     ADBC = "adbc"
+    DUCKDB = "duckdb"
 
 
 class ConnectionProperties(BaseModel):
@@ -54,7 +62,7 @@ class ConnectionProperties(BaseModel):
         uri: The URI for the connection
     """
     label: str | None = None
-    type: ConnectionType = Field(default=ConnectionType.SQLALCHEMY)
+    type: ConnectionTypeEnum = Field(default=ConnectionTypeEnum.SQLALCHEMY)
     uri: str
     sa_create_engine_args: dict[str, Any] = Field(default_factory=dict)
 
@@ -64,33 +72,39 @@ class ConnectionProperties(BaseModel):
         Creates and caches a SQLAlchemy engine if the connection type is sqlalchemy.
         Returns None for other connection types.
         """
-        if self.type == ConnectionType.SQLALCHEMY:
+        if self.type == ConnectionTypeEnum.SQLALCHEMY:
             return create_engine(self.uri, **self.sa_create_engine_args)
         else:
             raise ValueError(f'Connection type "{self.type}" does not support engine property')
 
     @cached_property
     def dialect(self) -> str:
-        if self.type == ConnectionType.SQLALCHEMY:
+        default_dialect = None
+        if self.type == ConnectionTypeEnum.SQLALCHEMY:
             dialect = self.engine.dialect.name
+        elif self.type == ConnectionTypeEnum.DUCKDB:
+            dialect = self.uri.split(':')[0]
+            default_dialect = 'duckdb'
         else:
             url = urlparse(self.uri)
             dialect = url.scheme
         
-        processed_dialect = next((d for d in ['sqlite', 'postgres', 'mysql'] if dialect.lower().startswith(d)), None)
+        processed_dialect = next((d for d in ['sqlite', 'postgres', 'mysql', 'duckdb'] if dialect.lower().startswith(d)), default_dialect)
         dialect = processed_dialect if processed_dialect is not None else dialect
         return dialect
     
     @cached_property
     def attach_uri_for_duckdb(self) -> str | None:
-        if self.type == ConnectionType.SQLALCHEMY:
+        if self.type == ConnectionTypeEnum.DUCKDB:
+            return self.uri
+        elif self.type == ConnectionTypeEnum.SQLALCHEMY:
             url = self.engine.url
             host = url.host
             port = url.port
             username = url.username
             password = url.password
             database = url.database
-            sqlite_database = database if database is not None else ""
+            database_as_file = database if database is not None else ""
         else:
             url = urlparse(self.uri)
             host = url.hostname
@@ -98,14 +112,18 @@ class ConnectionProperties(BaseModel):
             username = url.username
             password = url.password
             database = url.path.lstrip('/')
-            sqlite_database = self.uri.replace(f"{self.dialect}://", "")
+            database_as_file = self.uri.replace(f"{self.dialect}://", "")
         
-        if self.dialect == 'sqlite':
-            return sqlite_database
-        elif self.dialect in ('postgres', 'mysql'):
-            return f"dbname={database} user={username} password={password} host={host} port={port}"
+        if self.dialect in ('postgres', 'mysql'):
+            attach_uri = f"{self.dialect}:dbname={database} user={username} password={password} host={host} port={port}"
+        elif self.dialect == "sqlite":
+            attach_uri = f"{self.dialect}:{database_as_file}"
+        elif self.dialect == "duckdb":
+            attach_uri = database_as_file
         else:
-            return None
+            attach_uri = None
+        
+        return attach_uri
 
 
 class DbConnConfig(ConnectionProperties, _ConfigWithNameBaseModel):
@@ -114,6 +132,16 @@ class DbConnConfig(ConnectionProperties, _ConfigWithNameBaseModel):
         return self
 
 
+class DatasetConfigurablesConfig(BaseModel):
+    name: str
+    default: str
+
+
+class ConfigurablesConfig(DatasetConfigurablesConfig):
+    label: str = ""
+    description: str = ""
+
+
 class ParametersConfig(BaseModel):
     type: str
     factory: str
@@ -126,6 +154,20 @@ class PermissionScope(Enum):
     PRIVATE = 2
 
 
+class AuthenticationEnforcement(Enum):
+    REQUIRED = "required"
+    OPTIONAL = "optional"
+    DISABLED = "disabled"
+
+class AuthenticationType(Enum):
+    MANAGED = "managed"
+    EXTERNAL = "external"
+
+class AuthenticationConfig(BaseModel):
+    enforcement: AuthenticationEnforcement = AuthenticationEnforcement.OPTIONAL
+    type: AuthenticationType = AuthenticationType.MANAGED
+
+
 class AnalyticsOutputConfig(_ConfigWithNameBaseModel):
     label: str = ""
     description: str = ""
@@ -149,14 +191,9 @@ class AnalyticsOutputConfig(_ConfigWithNameBaseModel):
             raise ValueError(f'Scope "{value}" is invalid for dataset/dashboard "{name}". Scope must be one of {scope_list}') from e
 
 
-class DatasetTraitConfig(_ConfigWithNameBaseModel):
-    default: Any 
-
-
 class DatasetConfig(AnalyticsOutputConfig):
     model: str = ""
-    traits: dict = Field(default_factory=dict)
-    default_test_set: str = ""
+    configurables: list[DatasetConfigurablesConfig] = Field(default_factory=list)
     
     def __hash__(self) -> int:
         return hash("dataset_"+self.name)
@@ -168,23 +205,24 @@ class DatasetConfig(AnalyticsOutputConfig):
         return self
 
 
+class TestSetsUserConfig(BaseModel):
+    access_level: Literal["admin", "member", "guest"] = "guest"
+    custom_fields: dict[str, Any] = Field(default_factory=dict)
+
 class TestSetsConfig(_ConfigWithNameBaseModel):
-    datasets: list[str] | None = None
-    user_attributes: dict[str, Any] | None = None
+    user: TestSetsUserConfig = Field(default_factory=TestSetsUserConfig)
     parameters: dict[str, Any] = Field(default_factory=dict)
-
-    @property
-    def is_authenticated(self) -> bool:
-        return self.user_attributes is not None
+    configurables: dict[str, Any] = Field(default_factory=dict)
 
 
 class ManifestConfig(BaseModel):
     project_variables: ProjectVarsConfig
+    authentication: AuthenticationConfig = Field(default_factory=AuthenticationConfig)
     packages: list[PackageConfig] = Field(default_factory=list)
     connections: dict[str, DbConnConfig] = Field(default_factory=dict)
     parameters: list[ParametersConfig] = Field(default_factory=list)
+    configurables: dict[str, ConfigurablesConfig] = Field(default_factory=dict)
     selection_test_sets: dict[str, TestSetsConfig] = Field(default_factory=dict)
-    dataset_traits: dict[str, DatasetTraitConfig] = Field(default_factory=dict)
     datasets: dict[str, DatasetConfig] = Field(default_factory=dict)
     base_path: str = "."
     env_vars: dict[str, str] = Field(default_factory=dict)
@@ -199,13 +237,13 @@ class ManifestConfig(BaseModel):
             set_of_directories.add(package.directory)
         return packages
     
-    @field_validator("connections", "selection_test_sets", "dataset_traits", "datasets", mode="before")
+    @field_validator("connections", "selection_test_sets", "datasets", "configurables", mode="before")
     @classmethod
     def names_are_unique(cls, values: list[dict] | dict[str, dict], info: ValidationInfo) -> dict[str, dict]:
         if isinstance(values, list):
             values_as_dict = {}
             for obj in values:
-                name = obj["name"]
+                name = u.normalize_name(obj["name"])
                 if name in values_as_dict:
                     raise ValueError(f'In the {info.field_name} section, the name "{name}" was specified multiple times')
                 values_as_dict[name] = obj
@@ -220,45 +258,62 @@ class ManifestConfig(BaseModel):
         return self
     
     @model_validator(mode="after")
-    def validate_dataset_traits(self) -> Self:
-        for dataset_name, dataset in self.datasets.items():
-            # Validate that all trait keys in dataset.traits exist in dataset_traits
-            for trait_key in dataset.traits.keys():
-                if trait_key not in self.dataset_traits:
+    def validate_authentication_and_scopes(self) -> Self:
+        """
+        Enforce authentication rules:
+        - If authentication.is_required, no dataset may be PUBLIC.
+        """
+        if self.authentication.enforcement == AuthenticationEnforcement.REQUIRED:
+            invalid = [name for name, ds in self.datasets.items() if ds.scope == PermissionScope.PUBLIC]
+            if invalid:
+                raise ValueError(
+                    "Authentication is required, so datasets cannot be public. "
+                    f"Update the scope for datasets: {invalid}"
+                )
+        return self
+    
+    @model_validator(mode="after")
+    def validate_dataset_configurables(self) -> Self:
+        """
+        Validate that dataset configurables reference valid project-level configurables.
+        """
+        for dataset_name, dataset_cfg in self.datasets.items():
+            for cfg_override in dataset_cfg.configurables:
+                if cfg_override.name not in self.configurables:
                     raise ValueError(
-                        f'Dataset "{dataset_name}" references undefined trait "{trait_key}". '
-                        f'Traits must be defined with a default value in the dataset_traits section.'
+                        f'Dataset "{dataset_name}" references configurable "{cfg_override.name}" which is not defined '
+                        f'in the project configurables'
                     )
-            
-            # Set default values for any traits that are missing
-            for trait_name, trait_config in self.dataset_traits.items():
-                if trait_name not in dataset.traits:
-                    dataset.traits[trait_name] = trait_config.default
-                    
         return self
     
-    def get_default_test_set(self, dataset_name: str) -> TestSetsConfig:
+    def get_default_test_set(self) -> TestSetsConfig:
         """
         Raises KeyError if dataset name doesn't exist
         """
-        default_name_1 = self.datasets[dataset_name].default_test_set
-        default_name_2 = self.env_vars.get(c.SQRL_TEST_SETS_DEFAULT_NAME_USED, "default")
-        default_name = default_name_1 if default_name_1 else default_name_2
-        default_test_set = self.selection_test_sets.get(default_name, TestSetsConfig(name=default_name))
+        default_default_test_set = TestSetsConfig(name=c.DEFAULT_TEST_SET_NAME)
+        default_test_set = self.selection_test_sets.get(c.DEFAULT_TEST_SET_NAME, default_default_test_set)
         return default_test_set
     
-    def get_applicable_test_sets(self, dataset: str) -> list[str]:
-        applicable_test_sets = []
-        for test_set_name, test_set_config in self.selection_test_sets.items():
-            if test_set_config.datasets is None or dataset in test_set_config.datasets:
-                applicable_test_sets.append(test_set_name)
-        return applicable_test_sets
-    
-    def get_default_traits(self) -> dict[str, Any]:
-        default_traits = {}
-        for trait_name, trait_config in self.dataset_traits.items():
-            default_traits[trait_name] = trait_config.default
-        return default_traits
+    def get_default_configurables(self, dataset_name: str | None = None) -> dict[str, str]:
+        """
+        Return a dictionary of configurable name to its default value.
+        
+        If dataset_name is provided, merges project-level defaults with dataset-specific overrides.
+
+        Supports both list- and dict-shaped internal storage for configurables.
+        """
+        defaults: dict[str, str] = {}
+        for name, cfg in self.configurables.items():
+            defaults[name] = str(cfg.default)
+        
+        # Apply dataset-specific overrides if dataset_name is provided
+        if dataset_name is not None:
+            dataset_cfg = self.datasets.get(dataset_name)
+            if dataset_cfg:
+                for cfg_override in dataset_cfg.configurables:
+                    defaults[cfg_override.name] = cfg_override.default
+        
+        return defaults
 
 
 class ManifestIO:
@@ -269,7 +324,19 @@ class ManifestIO:
 
         raw_content = u.read_file(u.Path(base_path, c.MANIFEST_FILE))
         content = u.render_string(raw_content, base_path=base_path, env_vars=env_vars)
-        manifest_content = yaml.safe_load(content)
+        manifest_content: dict[str, Any] = yaml.safe_load(content)
+
+        auth_cfg: dict[str, Any] = manifest_content.get("authentication", {})
+        is_auth_required = bool(auth_cfg.get("is_required", False))
+
+        if is_auth_required:
+            # If authentication is required, assume PROTECTED when scope is not specified
+            # while explicitly forbidding PUBLIC (enforced in model validator)
+            datasets_raw = manifest_content.get("datasets", [])
+            for ds in datasets_raw:
+                if isinstance(ds, dict) and "scope" not in ds:
+                    ds["scope"] = "protected"
+        
         try:
             manifest_cfg = ManifestConfig(base_path=base_path, **manifest_content)
         except ValidationError as e:

squirrels/_model_builder.py

@@ -1,31 +1,26 @@
 from dataclasses import dataclass, field
-import asyncio, shutil, duckdb, time
+import duckdb, time
 
 from . import _utils as u, _connection_set as cs, _models as m
-from ._exceptions import InvalidInputError
 
 
 @dataclass
 class ModelBuilder:
-    _duckdb_venv_path: str
+    _datalake_db_path: str
     _conn_set: cs.ConnectionSet
     _static_models: dict[str, m.StaticModel]
     _conn_args: cs.ConnectionsArgs = field(default_factory=lambda: cs.ConnectionsArgs(".", {}, {}))
     _logger: u.Logger = field(default_factory=lambda: u.Logger(""))
     
-    def _attach_connections(self, duckdb_conn: duckdb.DuckDBPyConnection) -> dict[str, str]:
-        dialect_by_conn_name: dict[str, str] = {}
+    def _attach_connections(self, duckdb_conn: duckdb.DuckDBPyConnection) -> None:
         for conn_name, conn_props in self._conn_set.get_connections_as_dict().items():
             if not isinstance(conn_props, m.ConnectionProperties):
                 continue
-            dialect = conn_props.dialect
             attach_uri = conn_props.attach_uri_for_duckdb
             if attach_uri is None:
                 continue # skip unsupported dialects
-            attach_stmt = f"ATTACH IF NOT EXISTS '{attach_uri}' AS db_{conn_name} (TYPE {dialect}, READ_ONLY)"
+            attach_stmt = f"ATTACH IF NOT EXISTS '{attach_uri}' AS db_{conn_name} (READ_ONLY)"
             u.run_duckdb_stmt(self._logger, duckdb_conn, attach_stmt, redacted_values=[attach_uri])
-            dialect_by_conn_name[conn_name] = dialect
-        return dialect_by_conn_name
 
     async def _build_models(self, duckdb_conn: duckdb.DuckDBPyConnection, select: str | None, full_refresh: bool) -> None:
         """
@@ -50,62 +45,25 @@ class ModelBuilder:
         coroutines = []
         for model_name in terminal_nodes:
             model = self._static_models[model_name]
+            # await model.build_model(duckdb_conn, full_refresh)
             coro = model.build_model(duckdb_conn, full_refresh)
             coroutines.append(coro)
         await u.asyncio_gather(coroutines)
 
-    async def build(self, full_refresh: bool, select: str | None, stage_file: bool) -> None:
+    async def build(self, full_refresh: bool, select: str | None) -> None:
         start = time.time()
 
-        # Create target folder if it doesn't exist
-        duckdb_path = u.Path(self._duckdb_venv_path)
-        duckdb_path.parent.mkdir(parents=True, exist_ok=True)
-
-        # Delete any existing DuckDB file if full refresh is requested
-        duckdb_dev_path = u.Path(self._duckdb_venv_path + ".dev")
-        duckdb_stg_path = u.Path(self._duckdb_venv_path + ".stg")
-
-        # If the development copy is already in use, a concurrent build is not allowed
-        duckdb_dev_lock_path = u.Path(self._duckdb_venv_path + ".dev.lock")
-        if duckdb_dev_lock_path.exists():
-            raise InvalidInputError(60, "An existing build process is already running and a concurrent build is not allowed")
-        duckdb_dev_lock_path.touch(exist_ok=False)
-        
-        # Ensure the lock file is deleted even if an exception is raised
-        try:
-            # If not full refresh, create a development copy of the existing virtual data environment
-            if not full_refresh:
-                if duckdb_stg_path.exists():
-                    duckdb_stg_path.replace(duckdb_dev_path)
-                elif duckdb_path.exists():
-                    shutil.copy(duckdb_path, duckdb_dev_path)
-                
-            self._logger.log_activity_time("creating development copy of virtual data environment", start)
-        
-            # Connect to DuckDB file
-            duckdb_conn = u.create_duckdb_connection(duckdb_dev_path)
-            
-        except Exception:
-            duckdb_dev_lock_path.unlink()
-            raise
+        # Connect directly to DuckLake instead of attaching (supports concurrent connections)
+        duckdb_conn = u.create_duckdb_connection(self._datalake_db_path)
         
-        # Sometimes code after conn.close() doesn't run (as if the python process is killed but no error is raised)
-        # Using a new try block to ensure the lock file is removed before closing the connection
         try:
             # Attach connections
             self._attach_connections(duckdb_conn)
 
             # Construct build models
             await self._build_models(duckdb_conn, select, full_refresh)
-        
+
         finally:
-            duckdb_dev_lock_path.unlink()
             duckdb_conn.close()
 
-        # Rename duckdb_dev_path to duckdb_path (or duckdb_stg_path if stage_file is True)
-        if stage_file:
-            duckdb_dev_path.replace(duckdb_stg_path)
-        else:
-            duckdb_dev_path.replace(duckdb_path)
-
-        self._logger.log_activity_time("TOTAL TIME to build virtual data environment", start)
+        self._logger.log_activity_time("TOTAL TIME to build the Virtual Data Lake (VDL)", start)

squirrels/_model_configs.py

@@ -47,7 +47,7 @@ class QueryModelConfig(ModelConfig):
 
 
 class BuildModelConfig(QueryModelConfig):
-    materialization: str = Field(default="TABLE", description="The materialization of the model (ignored if Python model which is always a table)")
+    materialization: str = Field(default="VIEW", description="The materialization of the model (ignored if Python model which is always a table)")
 
     def get_sql_for_build(self, model_name: str, select_query: str) -> str:
         if self.materialization.upper() == "TABLE":
@@ -57,7 +57,7 @@ class BuildModelConfig(QueryModelConfig):
         else:
             raise ValueError(f"Invalid materialization: {self.materialization}")
         
-        create_prefix = f"CREATE OR REPLACE {materialization} {model_name} AS\n"
+        create_prefix = f"CREATE OR REPLACE {materialization} {model_name} AS\n\n"
         return create_prefix + select_query
 
 
@@ -70,5 +70,5 @@ class FederateModelConfig(QueryModelConfig):
 
     def get_sql_for_create(self, model_name: str, select_query: str) -> str:
         materialization = "TABLE" if self.eager else "VIEW"
-        create_prefix = f"CREATE {materialization} {model_name} AS\n"
+        create_prefix = f"CREATE {materialization} {model_name} AS\n\n"
         return create_prefix + select_query

squirrels/_model_queries.py

@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
 from typing import Callable, Generic, TypeVar, Any
 import polars as pl, pandas as pd
 
-from .arguments.run_time_args import BuildModelArgs
+from ._arguments.run_time_args import BuildModelArgs
 from ._model_configs import ModelConfig