TypeDAL 2.3.6__py3-none-any.whl → 3.0.0__py3-none-any.whl

typedal/__about__.py

@@ -5,4 +5,4 @@ This file contains the Version info for this package.
 # SPDX-FileCopyrightText: 2023-present Robin van der Noord <robinvandernoord@gmail.com>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "2.3.6"
+__version__ = "3.0.0"

typedal/caching.py

@@ -1,6 +1,7 @@
 """
 Helpers to facilitate db-based caching.
 """
+
 import contextlib
 import hashlib
 import json
@@ -12,6 +13,7 @@ import dill  # nosec
 from pydal.objects import Field, Rows, Set
 
 from .core import TypedField, TypedRows, TypedTable
+from .types import Query
 
 if typing.TYPE_CHECKING:  # pragma: no cover
     from .core import TypeDAL
@@ -171,7 +173,7 @@ def clear_expired() -> int:
     By default, expired items are only removed when trying to access them.
     """
     now = get_now()
-    return len(_TypedalCache.where(_TypedalCache.expires_at > now).delete())
+    return len(_TypedalCache.where(_TypedalCache.expires_at != None).where(_TypedalCache.expires_at < now).delete())
 
 
 def _remove_cache(s: Set, tablename: str) -> None:
@@ -269,3 +271,137 @@ def load_from_cache(key: str, db: "TypeDAL") -> Any | None:
         return _load_from_cache(key, db)
 
     return None  # pragma: no cover
+
+
+def humanize_bytes(size: int | float) -> str:
+    """
+    Turn a number of bytes into a human-readable version (e.g. 124 GB).
+    """
+    if not size:
+        return "0"
+
+    suffixes = ["B", "KB", "MB", "GB", "TB", "PB"]  # List of suffixes for different magnitudes
+    suffix_index = 0
+
+    while size > 1024 and suffix_index < len(suffixes) - 1:
+        suffix_index += 1
+        size /= 1024.0
+
+    return f"{size:.2f} {suffixes[suffix_index]}"
+
+
+def _expired_and_valid_query() -> tuple[str, str]:
+    expired_items = (
+        _TypedalCache.where(lambda row: (row.expires_at < get_now()) & (row.expires_at != None))
+        .select(_TypedalCache.id)
+        .to_sql()
+    )
+
+    valid_items = _TypedalCache.where(~_TypedalCache.id.belongs(expired_items)).select(_TypedalCache.id).to_sql()
+
+    return expired_items, valid_items
+
+
+T = typing.TypeVar("T")
+Stats = typing.TypedDict("Stats", {"total": T, "valid": T, "expired": T})
+
+RowStats = typing.TypedDict(
+    "RowStats",
+    {
+        "Dependent Cache Entries": int,
+    },
+)
+
+
+def _row_stats(db: "TypeDAL", table: str, query: Query) -> RowStats:
+    count_field = _TypedalCacheDependency.entry.count()
+    stats: TypedRows[_TypedalCacheDependency] = db(query & (_TypedalCacheDependency.table == table)).select(
+        _TypedalCacheDependency.entry, count_field, groupby=_TypedalCacheDependency.entry
+    )
+    return {
+        "Dependent Cache Entries": len(stats),
+    }
+
+
+def row_stats(db: "TypeDAL", table: str, row_id: str) -> Stats[RowStats]:
+    """
+    Collect caching stats for a specific table row (by ID).
+    """
+    expired_items, valid_items = _expired_and_valid_query()
+
+    query = _TypedalCacheDependency.idx == row_id
+
+    return {
+        "total": _row_stats(db, table, query),
+        "valid": _row_stats(db, table, _TypedalCacheDependency.entry.belongs(valid_items) & query),
+        "expired": _row_stats(db, table, _TypedalCacheDependency.entry.belongs(expired_items) & query),
+    }
+
+
+TableStats = typing.TypedDict(
+    "TableStats",
+    {
+        "Dependent Cache Entries": int,
+        "Associated Table IDs": int,
+    },
+)
+
+
+def _table_stats(db: "TypeDAL", table: str, query: Query) -> TableStats:
+    count_field = _TypedalCacheDependency.entry.count()
+    stats: TypedRows[_TypedalCacheDependency] = db(query & (_TypedalCacheDependency.table == table)).select(
+        _TypedalCacheDependency.entry, count_field, groupby=_TypedalCacheDependency.entry
+    )
+    return {
+        "Dependent Cache Entries": len(stats),
+        "Associated Table IDs": sum(stats.column(count_field)),
+    }
+
+
+def table_stats(db: "TypeDAL", table: str) -> Stats[TableStats]:
+    """
+    Collect caching stats for a table.
+    """
+    expired_items, valid_items = _expired_and_valid_query()
+
+    return {
+        "total": _table_stats(db, table, _TypedalCacheDependency.id > 0),
+        "valid": _table_stats(db, table, _TypedalCacheDependency.entry.belongs(valid_items)),
+        "expired": _table_stats(db, table, _TypedalCacheDependency.entry.belongs(expired_items)),
+    }
+
+
+GenericStats = typing.TypedDict(
+    "GenericStats",
+    {
+        "entries": int,
+        "dependencies": int,
+        "size": str,
+    },
+)
+
+
+def _calculate_stats(db: "TypeDAL", query: Query) -> GenericStats:
+    sum_len_field = _TypedalCache.data.len().sum()
+    size_row = db(query).select(sum_len_field).first()
+
+    size = size_row[sum_len_field] if size_row else 0  # type: ignore
+
+    return {
+        "entries": _TypedalCache.where(query).count(),
+        "dependencies": db(_TypedalCacheDependency.entry.belongs(query)).count(),
+        "size": humanize_bytes(size),
+    }
+
+
+def calculate_stats(db: "TypeDAL") -> Stats[GenericStats]:
+    """
+    Collect generic caching stats.
+    """
+    expired_items, valid_items = _expired_and_valid_query()
+
+    return {
+        "total": _calculate_stats(db, _TypedalCache.id > 0),
+        "valid": _calculate_stats(db, _TypedalCache.id.belongs(valid_items)),
+        "expired": _calculate_stats(db, _TypedalCache.id.belongs(expired_items)),
+    }

typedal/cli.py

@@ -1,6 +1,8 @@
 """
 Typer CLI for TypeDAL.
 """
+
+import fnmatch
 import sys
 import typing
 import warnings
@@ -12,6 +14,8 @@ from configuraptor import asdict
 from configuraptor.alias import is_alias
 from configuraptor.helpers import is_optional
 
+from .types import AnyDict
+
 try:
     import edwh_migrate
     import pydal2sql  # noqa: F401
@@ -19,6 +23,7 @@ try:
     import rich
     import tomlkit
     import typer
+    from tabulate import tabulate
 except ImportError as e:  # pragma: no cover
     # ImportWarning is hidden by default
     warnings.warn(
@@ -38,14 +43,16 @@ from pydal2sql.types import (
 from pydal2sql_core import core_alter, core_create
 from typing_extensions import Never
 
+from . import caching
 from .__about__ import __version__
 from .config import TypeDALConfig, _fill_defaults, load_config, transform
+from .core import TypeDAL
 
 app = typer.Typer(
     no_args_is_help=True,
 )
 
-questionary_types: dict[typing.Hashable, Optional[dict[str, typing.Any]]] = {
+questionary_types: dict[typing.Hashable, Optional[AnyDict]] = {
     str: {
         "type": "text",
         "validate": lambda text: True if len(text) > 0 else "Please enter a value",
@@ -92,7 +99,7 @@ T = typing.TypeVar("T")
 notfound = object()
 
 
-def _get_question(prop: str, annotation: typing.Type[T]) -> Optional[dict[str, typing.Any]]:  # pragma: no cover
+def _get_question(prop: str, annotation: typing.Type[T]) -> Optional[AnyDict]:  # pragma: no cover
     question = questionary_types.get(prop, notfound)
     if question is notfound:
         # None means skip the question, notfound means use the type default!
@@ -147,11 +154,7 @@ def setup(
 
     toml_contents = toml_path.read_text()
     # tomli has native Python types, tomlkit doesn't but preserves comments
-    toml_obj: dict[str, typing.Any] = tomli.loads(toml_contents)
-
-    if "[tool.typedal]" in toml_contents:
-        section = toml_obj["tool"]["typedal"]
-        config.update(**section, _overwrite=True)
+    toml_obj: AnyDict = tomli.loads(toml_contents)
 
     if "[tool.pydal2sql]" in toml_contents:
         mapping = {"": ""}  # <- placeholder
@@ -169,6 +172,10 @@ def setup(
 
         config.update(**extra_config)
 
+    if "[tool.typedal]" in toml_contents:
+        section = toml_obj["tool"]["typedal"]
+        config.update(**section, _overwrite=True)
+
     data = asdict(config, with_top_level_key=False)
     data["migrate"] = None  # determined based on existence of input/output file.
 
@@ -202,7 +209,7 @@ def setup(
         transform(data, prop)
 
     with toml_path.open("r") as f:
-        old_contents: dict[str, typing.Any] = tomlkit.load(f)
+        old_contents: AnyDict = tomlkit.load(f)
 
     if "tool" not in old_contents:
         old_contents["tool"] = {}
@@ -219,9 +226,10 @@ def setup(
     rich.print(f"[green]Wrote updated config to {toml_path}![/green]")
 
 
-@app.command()
+@app.command(name="migrations.generate")
 @with_exit_code(hide_tb=IS_DEBUG)
 def generate_migrations(
+    connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None,
     filename_before: OptionalArgument[str] = None,
     filename_after: OptionalArgument[str] = None,
     dialect: DBType_Option = None,
@@ -238,7 +246,7 @@ def generate_migrations(
     """
     # 1. choose CREATE or ALTER based on whether 'output' exists?
     # 2. pass right args based on 'config' to function chosen in 1.
-    generic_config = load_config()
+    generic_config = load_config(connection)
     pydal2sql_config = generic_config.to_pydal2sql()
     pydal2sql_config.update(
         magic=magic,
@@ -249,6 +257,7 @@ def generate_migrations(
         format=output_format,
         input=filename_before,
         output=output_file,
+        _skip_none=True,
     )
 
     if pydal2sql_config.output and Path(pydal2sql_config.output).exists():
@@ -288,9 +297,10 @@ def generate_migrations(
             )
 
 
-@app.command()
+@app.command(name="migrations.run")
 @with_exit_code(hide_tb=IS_DEBUG)
 def run_migrations(
+    connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None,
     migrations_file: OptionalArgument[str] = None,
     db_uri: Optional[str] = None,
     db_folder: Optional[str] = None,
@@ -310,7 +320,7 @@ def run_migrations(
     # 1. build migrate Config from TypeDAL config
     # 2. import right file
     # 3. `activate_migrations`
-    generic_config = load_config()
+    generic_config = load_config(connection)
     migrate_config = generic_config.to_migrate()
 
     migrate_config.update(
@@ -325,6 +335,7 @@ def run_migrations(
         create_flag_location=create_flag_location,
         db_folder=db_folder,
         migrations_file=migrations_file,
+        _skip_none=True,
     )
 
     if dry_run:
@@ -334,6 +345,220 @@ def run_migrations(
     return True
 
 
+def match_strings(patterns: list[str] | str, string_list: list[str]) -> list[str]:
+    """
+    Glob but on a list of strings.
+    """
+    if isinstance(patterns, str):
+        patterns = [patterns]
+
+    matches = []
+    for pattern in patterns:
+        matches.extend([s for s in string_list if fnmatch.fnmatch(s, pattern)])
+
+    return matches
+
+
+@app.command(name="migrations.fake")
+@with_exit_code(hide_tb=IS_DEBUG)
+def fake_migrations(
+    names: typing.Annotated[list[str], typer.Argument()] = None,
+    all: bool = False,  # noqa: A002
+    connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None,
+    migrations_file: Optional[str] = None,
+    db_uri: Optional[str] = None,
+    db_folder: Optional[str] = None,
+    migrate_table: Optional[str] = None,
+    dry_run: bool = False,
+) -> int:
+    """
+    Mark one or more migrations as completed in the database, without executing the SQL code.
+
+    glob is supported in 'names'
+    """
+    if not (names or all):
+        rich.print("Please provide one or more migration names, or pass --all to fake all.")
+        return 1
+
+    generic_config = load_config(connection)
+    migrate_config = generic_config.to_migrate()
+
+    migrate_config.update(
+        migrate_uri=db_uri,
+        migrate_table=migrate_table,
+        db_folder=db_folder,
+        migrations_file=migrations_file,
+        _skip_none=True,
+    )
+
+    migrations = edwh_migrate.list_migrations(migrate_config)
+
+    migration_names = list(migrations.keys())
+
+    to_fake = migration_names if all else match_strings(names or [], migration_names)
+
+    try:
+        db = edwh_migrate.setup_db(config=migrate_config)
+    except edwh_migrate.migrate.DatabaseNotYetInitialized:
+        db = edwh_migrate.setup_db(
+            config=migrate_config, migrate=True, migrate_enabled=True, remove_migrate_tablefile=True
+        )
+
+    previously_migrated = (
+        db(
+            db.ewh_implemented_features.name.belongs(to_fake)
+            & (db.ewh_implemented_features.installed == True)  # noqa E712
+        )
+        .select(db.ewh_implemented_features.name)
+        .column("name")
+    )
+
+    if dry_run:
+        rich.print("Would migrate these:", [_ for _ in to_fake if _ not in previously_migrated])
+        return 0
+
+    n = len(to_fake)
+    print(f"{len(previously_migrated)} / {n} were already installed.")
+
+    for name in to_fake:
+        if name in previously_migrated:
+            continue
+
+        edwh_migrate.mark_migration(db, name=name, installed=True)
+
+    db.commit()
+    rich.print(f"Faked {n} new migrations.")
+    return 0
+
+
+AnyNestedDict: typing.TypeAlias = dict[str, AnyDict]
+
+
+def tabulate_data(data: AnyNestedDict) -> None:
+    """
+    Print a nested dict of data in a nice, human-readable table.
+    """
+    flattened_data = []
+    for key, inner_dict in data.items():
+        temp_dict = {"": key}
+        temp_dict.update(inner_dict)
+        flattened_data.append(temp_dict)
+
+    # Display the tabulated data from the transposed dictionary
+    print(tabulate(flattened_data, headers="keys"))
+
+
+FormatOptions: typing.TypeAlias = typing.Literal["plaintext", "json", "yaml", "toml"]
+
+
+def get_output_format(fmt: FormatOptions) -> typing.Callable[[AnyNestedDict], None]:
+    """
+    This function takes a format option as input and \
+        returns a function that can be used to output data in the specified format.
+    """
+    match fmt:
+        case "plaintext":
+            output = tabulate_data
+        case "json":
+
+            def output(_data: AnyDict | AnyNestedDict) -> None:
+                import json
+
+                print(json.dumps(_data, indent=2))
+
+        case "yaml":
+
+            def output(_data: AnyDict | AnyNestedDict) -> None:
+                import yaml
+
+                print(yaml.dump(_data))
+
+        case "toml":
+
+            def output(_data: AnyDict | AnyNestedDict) -> None:
+                import tomli_w
+
+                print(tomli_w.dumps(_data))
+
+        case _:
+            options = typing.get_args(FormatOptions)
+            raise ValueError(f"Invalid format '{fmt}'. Please choose one of {options}.")
+
+    return output
+
+
+@app.command(name="cache.stats")
+@with_exit_code(hide_tb=IS_DEBUG)
+def cache_stats(
+    identifier: typing.Annotated[str, typer.Argument()] = "",
+    connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None,
+    fmt: typing.Annotated[
+        str, typer.Option("--format", "--fmt", "-f", help="plaintext (default) or json")
+    ] = "plaintext",
+) -> None:
+    """
+    Collect caching stats.
+
+    Examples:
+        typedal cache.stats
+        typedal cache.stats user
+        typedal cache.stats user.3
+    """
+    config = load_config(connection)
+    db = TypeDAL(config=config, migrate=False, fake_migrate=False)
+
+    output = get_output_format(typing.cast(FormatOptions, fmt))
+
+    data: AnyDict
+    parts = identifier.split(".")
+    match parts:
+        case [] | [""]:
+            # generic stats
+            data = caching.calculate_stats(db)  # type: ignore
+        case [table]:
+            # table stats
+            data = caching.table_stats(db, table)  # type: ignore
+        case [table, row_id]:
+            # row stats
+            data = caching.row_stats(db, table, row_id)  # type: ignore
+        case _:
+            raise ValueError("Please use the format `table` or `table.id` for this command.")
+
+    output(data)
+
+    # todo:
+    #  - sort by most dependencies
+    #  - sort by biggest data
+    #  - include size for table_stats, row_stats
+    #  - group by table
+
+
+@app.command(name="cache.clear")
+@with_exit_code(hide_tb=IS_DEBUG)
+def cache_clear(
+    connection: typing.Annotated[str, typer.Option("--connection", "-c")] = None,
+    purge: typing.Annotated[bool, typer.Option("--all", "--purge", "-p")] = False,
+) -> None:
+    """
+    Clear (expired) items from the cache.
+
+    Args:
+        connection (optional): [tool.typedal.<connection>]
+        purge (default: no): remove all items, not only expired
+    """
+    config = load_config(connection)
+    db = TypeDAL(config=config, migrate=False, fake_migrate=False)
+
+    if purge:
+        caching.clear_cache()
+        print("Emptied cache")
+    else:
+        n = caching.clear_expired()
+        print(f"Removed {n} expired from cache")
+
+    db.commit()
+
+
 def version_callback() -> Never:
     """
     --version requested!

typedal/config.py

@@ -1,6 +1,7 @@
 """
 TypeDAL can be configured by a combination of pyproject.toml (static), env (dynamic) and code (programmic).
 """
+
 import os
 import re
 import typing
@@ -9,11 +10,13 @@ from collections import defaultdict
 from pathlib import Path
 from typing import Any, Optional
 
-import black.files
 import tomli
 from configuraptor import TypedConfig, alias
+from configuraptor.helpers import find_pyproject_toml
 from dotenv import dotenv_values, find_dotenv
 
+from .types import AnyDict
+
 if typing.TYPE_CHECKING:  # pragma: no cover
     from edwh_migrate import Config as MigrateConfig
     from pydal2sql.typer_support import Config as P2SConfig
@@ -34,7 +37,7 @@ class TypeDALConfig(TypedConfig):
     connection: str = "default"
 
     # pydal2sql:
-    input: str = ""  # noqa: A003
+    input: str = ""
     output: str = ""
     noop: bool = False
     magic: bool = True
@@ -61,11 +64,7 @@ class TypeDALConfig(TypedConfig):
     db_type: str = alias("dialect")
     db_folder: str = alias("folder")
 
-    def __repr__(self) -> str:
-        """
-        Dump the config to a (fancy) string.
-        """
-        return f"<TypeDAL {self.__dict__}>"
+    # repr set by @beautify (by inheriting from TypedConfig)
 
     def to_pydal2sql(self) -> "P2SConfig":
         """
@@ -123,14 +122,8 @@ class TypeDALConfig(TypedConfig):
         )
 
 
-def find_pyproject_toml(directory: str | None = None) -> typing.Optional[str]:
-    """
-    Find the project's config toml, looks up until it finds the project root (black's logic).
-    """
-    return black.files.find_pyproject_toml((directory or os.getcwd(),))
-
 
-def _load_toml(path: str | bool | None = True) -> tuple[str, dict[str, Any]]:
+def _load_toml(path: str | bool | None = True) -> tuple[str, AnyDict]:
     """
     Path can be a file, a directory, a bool or None.
 
@@ -146,7 +139,7 @@ def _load_toml(path: str | bool | None = True) -> tuple[str, dict[str, Any]]:
     elif Path(str(path)).is_file():
         toml_path = str(path)
     else:
-        toml_path = find_pyproject_toml(str(path))
+        toml_path = find_pyproject_toml(path)
 
     if not toml_path:
         # nothing to load
@@ -156,13 +149,13 @@ def _load_toml(path: str | bool | None = True) -> tuple[str, dict[str, Any]]:
         with open(toml_path, "rb") as f:
             data = tomli.load(f)
 
-        return toml_path or "", typing.cast(dict[str, Any], data["tool"]["typedal"])
+        return str(toml_path) or "", typing.cast(AnyDict, data["tool"]["typedal"])
     except Exception as e:
         warnings.warn(f"Could not load typedal config toml: {e}", source=e)
-        return toml_path or "", {}
+        return str(toml_path) or "", {}
 
 
-def _load_dotenv(path: str | bool | None = True) -> tuple[str, dict[str, Any]]:
+def _load_dotenv(path: str | bool | None = True) -> tuple[str, AnyDict]:
     fallback_data = {k.lower().removeprefix("typedal_"): v for k, v in os.environ.items()}
     if path is False:
         dotenv_path = None
@@ -202,28 +195,28 @@ def get_db_for_alias(db_name: str) -> str:
     return DB_ALIASES.get(db_name, db_name)
 
 
-DEFAULTS: dict[str, Any | typing.Callable[[dict[str, Any]], Any]] = {
+DEFAULTS: dict[str, Any | typing.Callable[[AnyDict], Any]] = {
     "database": lambda data: data.get("db_uri") or "sqlite:memory",
-    "dialect": lambda data: get_db_for_alias(data["database"].split(":")[0])
-    if ":" in data["database"]
-    else data.get("db_type"),
+    "dialect": lambda data: (
+        get_db_for_alias(data["database"].split(":")[0]) if ":" in data["database"] else data.get("db_type")
+    ),
     "migrate": lambda data: not (data.get("input") or data.get("output")),
     "folder": lambda data: data.get("db_folder"),
-    "flag_location": lambda data: f"{db_folder}/flags"
-    if (db_folder := (data.get("folder") or data.get("db_folder")))
-    else "/flags",
+    "flag_location": lambda data: (
+        f"{db_folder}/flags" if (db_folder := (data.get("folder") or data.get("db_folder"))) else "/flags"
+    ),
     "pool_size": lambda data: 1 if data.get("dialect", "sqlite") == "sqlite" else 3,
 }
 
 
-def _fill_defaults(data: dict[str, Any], prop: str, fallback: Any = None) -> None:
+def _fill_defaults(data: AnyDict, prop: str, fallback: Any = None) -> None:
     default = DEFAULTS.get(prop, fallback)
     if callable(default):
         default = default(data)
     data[prop] = default
 
 
-def fill_defaults(data: dict[str, Any], prop: str) -> None:
+def fill_defaults(data: AnyDict, prop: str) -> None:
     """
     Fill missing property defaults with (calculated) sane defaults.
     """
@@ -231,14 +224,16 @@ def fill_defaults(data: dict[str, Any], prop: str) -> None:
         _fill_defaults(data, prop)
 
 
-TRANSFORMS: dict[str, typing.Callable[[dict[str, Any]], Any]] = {
-    "database": lambda data: data["database"]
-    if (":" in data["database"] or not data.get("dialect"))
-    else (data["dialect"] + "://" + data["database"])
+TRANSFORMS: dict[str, typing.Callable[[AnyDict], Any]] = {
+    "database": lambda data: (
+        data["database"]
+        if (":" in data["database"] or not data.get("dialect"))
+        else (data["dialect"] + "://" + data["database"])
+    )
 }
 
 
-def transform(data: dict[str, Any], prop: str) -> bool:
+def transform(data: AnyDict, prop: str) -> bool:
     """
     After the user has chosen a value, possibly transform it.
     """
@@ -278,7 +273,7 @@ def expand_posix_vars(posix_expr: str, context: dict[str, str]) -> str:
     return re.sub(pattern, replace_var, posix_expr)
 
 
-def expand_env_vars_into_toml_values(toml: dict[str, Any], env: dict[str, Any]) -> None:
+def expand_env_vars_into_toml_values(toml: AnyDict, env: AnyDict) -> None:
     """
     Recursively expands POSIX/Docker Compose-like environment variables in a TOML dictionary.
 
@@ -331,7 +326,10 @@ def expand_env_vars_into_toml_values(toml: dict[str, Any], env: dict[str, Any])
 
 
 def load_config(
-    _use_pyproject: bool | str | None = True, _use_env: bool | str | None = True, **fallback: Any
+    connection_name: Optional[str] = None,
+    _use_pyproject: bool | str | None = True,
+    _use_env: bool | str | None = True,
+    **fallback: Any,
 ) -> TypeDALConfig:
     """
     Combines multiple sources of config into one config instance.
@@ -345,8 +343,8 @@ def load_config(
 
     expand_env_vars_into_toml_values(toml, dotenv)
 
-    connection_name = dotenv.get("connection", "") or toml.get("default", "")
-    connection: dict[str, Any] = (toml.get(connection_name) if connection_name else toml) or {}
+    connection_name = connection_name or dotenv.get("connection", "") or toml.get("default", "")
+    connection: AnyDict = (toml.get(connection_name) if connection_name else toml) or {}
 
     combined = connection | dotenv | fallback
     combined = {k.replace("-", "_"): v for k, v in combined.items()}

typedal/core.py

@@ -1,6 +1,7 @@
 """
 Core functionality of TypeDAL.
 """
+
 import contextlib
 import csv
 import datetime as dt
@@ -44,6 +45,7 @@ from .types import (
     AfterDeleteCallable,
     AfterInsertCallable,
     AfterUpdateCallable,
+    AnyDict,
     BeforeDeleteCallable,
     BeforeInsertCallable,
     BeforeUpdateCallable,
@@ -352,8 +354,8 @@ class TypeDAL(pydal.DAL):  # type: ignore
         migrate_enabled: bool = True,
         fake_migrate_all: bool = False,
         decode_credentials: bool = False,
-        driver_args: Optional[dict[str, Any]] = None,
-        adapter_args: Optional[dict[str, Any]] = None,
+        driver_args: Optional[AnyDict] = None,
+        adapter_args: Optional[AnyDict] = None,
         attempts: int = 5,
         auto_import: bool = False,
         bigint_id: bool = False,
@@ -368,13 +370,15 @@ class TypeDAL(pydal.DAL):  # type: ignore
         enable_typedal_caching: bool = None,
         use_pyproject: bool | str = True,
         use_env: bool | str = True,
+        connection: Optional[str] = None,
+        config: Optional[TypeDALConfig] = None,
     ) -> None:
         """
         Adds some internal tables after calling pydal's default init.
 
         Set enable_typedal_caching to False to disable this behavior.
         """
-        config = load_config(_use_pyproject=use_pyproject, _use_env=use_env)
+        config = config or load_config(connection, _use_pyproject=use_pyproject, _use_env=use_env)
         config.update(
             database=uri,
             dialect=uri.split(":")[0] if uri and ":" in uri else None,
@@ -438,7 +442,7 @@ class TypeDAL(pydal.DAL):  # type: ignore
             # try again:
             return self.define(model, migrate=True, fake_migrate=True, redefine=True)
 
-    default_kwargs: typing.ClassVar[typing.Dict[str, Any]] = {
+    default_kwargs: typing.ClassVar[AnyDict] = {
         # fields are 'required' (notnull) by default:
         "notnull": True,
     }
@@ -747,7 +751,7 @@ class TableProtocol(typing.Protocol):  # pragma: no cover
     Make mypy happy.
     """
 
-    id: "TypedField[int]"  # noqa: A003
+    id: "TypedField[int]"
 
     def __getitem__(self, item: str) -> Field:
         """
@@ -847,7 +851,7 @@ class TableMeta(type):
         """
         return self(row)
 
-    def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]":  # noqa: A003
+    def all(self: typing.Type[T_MetaInstance]) -> "TypedRows[T_MetaInstance]":
         """
         Return all rows for this model.
         """
@@ -886,7 +890,7 @@ class TableMeta(type):
 
         return str(table._insert(**fields))
 
-    def bulk_insert(self: typing.Type[T_MetaInstance], items: list[dict[str, Any]]) -> "TypedRows[T_MetaInstance]":
+    def bulk_insert(self: typing.Type[T_MetaInstance], items: list[AnyDict]) -> "TypedRows[T_MetaInstance]":
         """
         Insert multiple rows, returns a TypedRows set of new instances.
         """
@@ -895,7 +899,7 @@ class TableMeta(type):
         return self.where(lambda row: row.id.belongs(result)).collect()
 
     def update_or_insert(
-        self: typing.Type[T_MetaInstance], query: T_Query | dict[str, Any] = DEFAULT, **values: Any
+        self: typing.Type[T_MetaInstance], query: T_Query | AnyDict = DEFAULT, **values: Any
     ) -> T_MetaInstance:
         """
         Update a row if query matches, else insert a new one.
@@ -1323,7 +1327,7 @@ class TypedTable(metaclass=TableMeta):
 
     _with: list[str]
 
-    id: "TypedField[int]"  # noqa: A003
+    id: "TypedField[int]"
 
     _before_insert: list[BeforeInsertCallable]
     _after_insert: list[AfterInsertCallable]
@@ -1471,7 +1475,7 @@ class TypedTable(metaclass=TableMeta):
     # underscore variants work for class instances (set up by _setup_instance_methods)
 
     @classmethod
-    def as_dict(cls, flat: bool = False, sanitize: bool = True) -> dict[str, Any]:
+    def as_dict(cls, flat: bool = False, sanitize: bool = True) -> AnyDict:
         """
         Dump the object to a plain dict.
 
@@ -1481,7 +1485,7 @@ class TypedTable(metaclass=TableMeta):
         """
         table = cls._ensure_table_defined()
         result = table.as_dict(flat, sanitize)
-        return typing.cast(dict[str, Any], result)
+        return typing.cast(AnyDict, result)
 
     @classmethod
     def as_json(cls, sanitize: bool = True, indent: Optional[int] = None, **kwargs: Any) -> str:
@@ -1521,7 +1525,7 @@ class TypedTable(metaclass=TableMeta):
 
     def _as_dict(
         self, datetime_to_str: bool = False, custom_types: typing.Iterable[type] | type | None = None
-    ) -> dict[str, Any]:
+    ) -> AnyDict:
         row = self._ensure_matching_row()
 
         result = row.as_dict(datetime_to_str=datetime_to_str, custom_types=custom_types)
@@ -1545,7 +1549,7 @@ class TypedTable(metaclass=TableMeta):
 
                 result[relationship] = data
 
-        return typing.cast(dict[str, Any], result)
+        return typing.cast(AnyDict, result)
 
     def _as_json(
         self,
@@ -1633,7 +1637,7 @@ class TypedTable(metaclass=TableMeta):
 
     # pickling:
 
-    def __getstate__(self) -> dict[str, Any]:
+    def __getstate__(self) -> AnyDict:
         """
         State to save when pickling.
 
@@ -1641,7 +1645,7 @@ class TypedTable(metaclass=TableMeta):
         Similar to as_dict but without changing the data of the relationships (dill does that recursively)
         """
         row = self._ensure_matching_row()
-        result: dict[str, Any] = row.as_dict()
+        result: AnyDict = row.as_dict()
 
         if _with := getattr(self, "_with", None):
             result["_with"] = _with
@@ -1653,7 +1657,7 @@ class TypedTable(metaclass=TableMeta):
         result["_row"] = self._row.as_json() if self._row else ""
         return result
 
-    def __setstate__(self, state: dict[str, Any]) -> None:
+    def __setstate__(self, state: AnyDict) -> None:
         """
         Used by dill when loading from a bytestring.
         """
@@ -1842,14 +1846,14 @@ class TypedRows(typing.Collection[T_MetaInstance], Rows):
         storage_to_dict: bool = False,
         datetime_to_str: bool = False,
         custom_types: list[type] = None,
-    ) -> dict[int, dict[str, Any]]:
+    ) -> dict[int, AnyDict]:
         """
         Get the data in a dict of dicts.
         """
         if any([key, compact, storage_to_dict, datetime_to_str, custom_types]):
             # functionality not guaranteed
             return typing.cast(
-                dict[int, dict[str, Any]],
+                dict[int, AnyDict],
                 super().as_dict(
                     key or "id",
                     compact,
@@ -1881,14 +1885,12 @@ class TypedRows(typing.Collection[T_MetaInstance], Rows):
         storage_to_dict: bool = False,
         datetime_to_str: bool = False,
         custom_types: list[type] = None,
-    ) -> list[dict[str, Any]]:
+    ) -> list[AnyDict]:
         """
         Get the data in a list of dicts.
         """
         if any([compact, storage_to_dict, datetime_to_str, custom_types]):
-            return typing.cast(
-                list[dict[str, Any]], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types)
-            )
+            return typing.cast(list[AnyDict], super().as_list(compact, storage_to_dict, datetime_to_str, custom_types))
 
         return [_.as_dict() for _ in self.records.values()]
 
@@ -1992,7 +1994,7 @@ class TypedRows(typing.Collection[T_MetaInstance], Rows):
         """
         return cls(rows, model, metadata=metadata)
 
-    def __getstate__(self) -> dict[str, Any]:
+    def __getstate__(self) -> AnyDict:
         """
         Used by dill to dump to bytes (exclude db connection etc).
         """
@@ -2003,7 +2005,7 @@ class TypedRows(typing.Collection[T_MetaInstance], Rows):
             "colnames": self.colnames,
         }
 
-    def __setstate__(self, state: dict[str, Any]) -> None:
+    def __setstate__(self, state: AnyDict) -> None:
         """
         Used by dill when loading from a bytestring.
         """
@@ -2031,7 +2033,7 @@ class QueryBuilder(typing.Generic[T_MetaInstance]):
     model: typing.Type[T_MetaInstance]
     query: Query
     select_args: list[Any]
-    select_kwargs: dict[str, Any]
+    select_kwargs: AnyDict
     relationships: dict[str, Relationship[Any]]
     metadata: Metadata
 
@@ -2040,7 +2042,7 @@ class QueryBuilder(typing.Generic[T_MetaInstance]):
         model: typing.Type[T_MetaInstance],
         add_query: Optional[Query] = None,
         select_args: Optional[list[Any]] = None,
-        select_kwargs: Optional[dict[str, Any]] = None,
+        select_kwargs: Optional[AnyDict] = None,
         relationships: dict[str, Relationship[Any]] = None,
         metadata: Metadata = None,
     ):
@@ -2090,7 +2092,7 @@ class QueryBuilder(typing.Generic[T_MetaInstance]):
         add_query: Optional[Query] = None,
         overwrite_query: Optional[Query] = None,
         select_args: Optional[list[Any]] = None,
-        select_kwargs: Optional[dict[str, Any]] = None,
+        select_kwargs: Optional[AnyDict] = None,
         relationships: dict[str, Relationship[Any]] = None,
         metadata: Metadata = None,
     ) -> "QueryBuilder[T_MetaInstance]":
@@ -2292,7 +2294,7 @@ class QueryBuilder(typing.Generic[T_MetaInstance]):
         db = self._get_db()
         return str(db(self.query)._update(**fields))
 
-    def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], dict[str, Any]]:
+    def _before_query(self, mut_metadata: Metadata, add_id: bool = True) -> tuple[Query, list[Any], AnyDict]:
         select_args = [self._select_arg_convert(_) for _ in self.select_args] or [self.model.ALL]
         select_kwargs = self.select_kwargs.copy()
         query = self.query
@@ -2352,6 +2354,17 @@ class QueryBuilder(typing.Generic[T_MetaInstance]):
 
         return load_from_cache(key, self._get_db())
 
+    def execute(self, add_id: bool = False) -> Rows:
+        """
+        Raw version of .collect which only executes the SQL, without performing any magic afterwards.
+        """
+        db = self._get_db()
+        metadata = typing.cast(Metadata, self.metadata.copy())
+
+        query, select_args, select_kwargs = self._before_query(metadata, add_id=add_id)
+
+        return db(query).select(*select_args, **select_kwargs)
+
     def collect(
         self, verbose: bool = False, _to: typing.Type["TypedRows[Any]"] = None, add_id: bool = True
     ) -> "TypedRows[T_MetaInstance]":
@@ -2400,7 +2413,7 @@ class QueryBuilder(typing.Generic[T_MetaInstance]):
         self,
         query: Query,
         select_args: list[Any],
-        select_kwargs: dict[str, Any],
+        select_kwargs: AnyDict,
         metadata: Metadata,
     ) -> tuple[Query, list[Any]]:
         db = self._get_db()
@@ -2718,7 +2731,7 @@ class PaginatedRows(TypedRows[T_MetaInstance]):
             "prev_page": pagination_data["current_page"] - 1 if has_prev_page else None,
         }
 
-    def next(self) -> Self:  # noqa: A003
+    def next(self) -> Self:
         """
         Get the next page.
         """
@@ -2754,7 +2767,7 @@ class TypedSet(pydal.objects.Set):  # type: ignore # pragma: no cover
     This class is not actually used, only 'cast' by TypeDAL.__call__
     """
 
-    def count(self, distinct: bool = None, cache: dict[str, Any] = None) -> int:
+    def count(self, distinct: bool = None, cache: AnyDict = None) -> int:
         """
         Count returns an int.
         """

typedal/for_py4web.py

@@ -1,13 +1,13 @@
 """
 ONLY USE IN COMBINATION WITH PY4WEB!
 """
-from typing import Any
 
 import threadsafevariable
 from py4web.core import ICECUBE
 from py4web.core import Fixture as _Fixture
 
 from .core import TypeDAL
+from .types import AnyDict
 from .web2py_py4web_shared import AuthUser
 
 
@@ -22,20 +22,20 @@ class DAL(TypeDAL, Fixture):  # pragma: no cover
     Fixture similar to the py4web pydal fixture, but for typedal.
     """
 
-    def on_request(self, _: dict[str, Any]) -> None:
+    def on_request(self, _: AnyDict) -> None:
         """
         Make sure there is a database connection when a request comes in.
         """
         self.get_connection_from_pool_or_new()
         threadsafevariable.ThreadSafeVariable.restore(ICECUBE)
 
-    def on_error(self, _: dict[str, Any]) -> None:
+    def on_error(self, _: AnyDict) -> None:
         """
         Rollback db on error.
         """
         self.recycle_connection_in_pool_or_close("rollback")
 
-    def on_success(self, _: dict[str, Any]) -> None:
+    def on_success(self, _: AnyDict) -> None:
         """
         Commit db on success.
         """

typedal/helpers.py

@@ -1,12 +1,15 @@
 """
 Helpers that work independently of core.
 """
+
 import io
 import types
 import typing
 from collections import ChainMap
 from typing import Any
 
+from .types import AnyDict
+
 T = typing.TypeVar("T")
 
 
@@ -29,7 +32,7 @@ def _all_annotations(cls: type) -> ChainMap[str, type]:
     return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))
 
 
-def all_dict(cls: type) -> dict[str, Any]:
+def all_dict(cls: type) -> AnyDict:
     """
     Get the internal data of a class and all it's parents.
     """

typedal/types.py

@@ -1,6 +1,7 @@
 """
 Stuff to make mypy happy.
 """
+
 import typing
 from datetime import datetime
 from typing import Any, Optional, TypedDict
@@ -15,6 +16,8 @@ from pydal.objects import Set as _Set
 from pydal.validators import Validator as _Validator
 from typing_extensions import NotRequired
 
+AnyDict: typing.TypeAlias = dict[str, Any]
+
 
 class Query(_Query):  # type: ignore
     """
@@ -116,7 +119,7 @@ class PaginateDict(TypedDict):
     Result of PaginatedRows.as_dict().
     """
 
-    data: dict[int, dict[str, Any]]
+    data: dict[int, AnyDict]
     pagination: Pagination
 
 
@@ -158,7 +161,7 @@ class Metadata(TypedDict):
 
     final_query: NotRequired[Query | str | None]
     final_args: NotRequired[list[Any]]
-    final_kwargs: NotRequired[dict[str, Any]]
+    final_kwargs: NotRequired[AnyDict]
     relationships: NotRequired[set[str]]
 
     sql: NotRequired[str]

{typedal-2.3.6.dist-info → typedal-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: TypeDAL
-Version: 2.3.6
+Version: 3.0.0
 Summary: Typing support for PyDAL
 Project-URL: Documentation, https://typedal.readthedocs.io/
 Project-URL: Issues, https://github.com/trialandsuccess/TypeDAL/issues
@@ -16,14 +16,15 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.10
 Requires-Dist: configurable-json
-Requires-Dist: configuraptor>=1.24.0
+Requires-Dist: configuraptor>=1.26.2
 Requires-Dist: dill
 Requires-Dist: pydal
 Provides-Extra: all
-Requires-Dist: edwh-migrate; extra == 'all'
+Requires-Dist: edwh-migrate>=0.8.0b1; extra == 'all'
 Requires-Dist: py4web; extra == 'all'
-Requires-Dist: pydal2sql[all]; extra == 'all'
+Requires-Dist: pydal2sql[all]>=1.1.3; extra == 'all'
 Requires-Dist: questionary; extra == 'all'
+Requires-Dist: tabulate; extra == 'all'
 Requires-Dist: tomlkit; extra == 'all'
 Requires-Dist: typer; extra == 'all'
 Provides-Extra: dev
@@ -33,10 +34,13 @@ Requires-Dist: mkdocs-dracula-theme; extra == 'dev'
 Requires-Dist: pytest-mypy-testing; extra == 'dev'
 Requires-Dist: python-semantic-release<8; extra == 'dev'
 Requires-Dist: su6[all]; extra == 'dev'
+Requires-Dist: types-pyyaml; extra == 'dev'
+Requires-Dist: types-tabulate; extra == 'dev'
 Provides-Extra: migrations
-Requires-Dist: edwh-migrate; extra == 'migrations'
-Requires-Dist: pydal2sql; extra == 'migrations'
+Requires-Dist: edwh-migrate>=0.8.0b1; extra == 'migrations'
+Requires-Dist: pydal2sql>=1.1.3; extra == 'migrations'
 Requires-Dist: questionary; extra == 'migrations'
+Requires-Dist: tabulate; extra == 'migrations'
 Requires-Dist: tomlkit; extra == 'migrations'
 Requires-Dist: typer; extra == 'migrations'
 Provides-Extra: py4web

typedal-3.0.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+typedal/__about__.py,sha256=q5Cn2Nax8QEhpWVeFUyO2PApUwXIgvp5_OADI5f0kMc,206
+typedal/__init__.py,sha256=QQpLiVl9w9hm2LBxey49Y_tCF_VB2bScVaS_mCjYy54,366
+typedal/caching.py,sha256=8UABVAhOlBpL96ykmqhxLaFYOe-XeAh7JoGh57OkxP8,11818
+typedal/cli.py,sha256=5-2U_pQOZNKHmhefiYtkd7g6B0DAXzjf4A1Jh7D37io,18427
+typedal/config.py,sha256=KDJXRsIQuFpSZy5XpSJiC_9WGLlmaOexACW0sWdCw54,11626
+typedal/core.py,sha256=qgJPvlcQYCujsjiiD6SOhWbIr1lxoUDpZUkMnK-mcDQ,95038
+typedal/fields.py,sha256=z2PD9vLWqBR_zXtiY0DthqTG4AeF3yxKoeuVfGXnSdg,5197
+typedal/for_py4web.py,sha256=d07b8hL_PvNDUS26Z5fDH2OxWb-IETBuAFPSzrRwm04,1285
+typedal/for_web2py.py,sha256=zvd5xC-SmuKc0JLDqT3hMIs6COaYnwTFXD_BIeC1vug,1832
+typedal/helpers.py,sha256=BFuGd-1tBA1-QS91C9PEvNY5z5KFHd3gTplxxDWdwSo,6509
+typedal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+typedal/types.py,sha256=1kGkNX6vfGg6ln84AG558C4Zx5ACRz-emrUTnuy-rRY,3410
+typedal/web2py_py4web_shared.py,sha256=cEbjkK0WOS9Q0nTyZuQaJWffeP4bjrL79Bx0xGy_UOs,1504
+typedal/serializers/as_json.py,sha256=ffo152W-sARYXym4BzwX709rrO2-QwKk2KunWY8RNl4,2229
+typedal-3.0.0.dist-info/METADATA,sha256=32JYzSrTHFkQomZ2v2iEAv0MSLx0VOnY-2w2pV36XPQ,7782
+typedal-3.0.0.dist-info/WHEEL,sha256=uNdcs2TADwSd5pVaP0Z_kcjcvvTUklh2S7bxZMF8Uj0,87
+typedal-3.0.0.dist-info/entry_points.txt,sha256=m1wqcc_10rHWPdlQ71zEkmJDADUAnZtn7Jac_6mbyUc,44
+typedal-3.0.0.dist-info/RECORD,,

{typedal-2.3.6.dist-info → typedal-3.0.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.17.1
+Generator: hatchling 1.22.4
 Root-Is-Purelib: true
 Tag: py3-none-any

typedal-2.3.6.dist-info/RECORD

@@ -1,18 +0,0 @@
-typedal/__about__.py,sha256=kbkWDzpCiYknqWjNq1J8RYaeb9wBLllZwZlNvQscZOk,206
-typedal/__init__.py,sha256=QQpLiVl9w9hm2LBxey49Y_tCF_VB2bScVaS_mCjYy54,366
-typedal/caching.py,sha256=cglkCphbg93Iy9-KHefQN9-JJxuA5-HjpzZAdw2BGvY,7709
-typedal/cli.py,sha256=F7M9D2pu2D-FoKZJrT3K9Wmbv5_ScUR8j5jNGoTgOfk,11800
-typedal/config.py,sha256=oDYSuPMm261hBoDBAXoq37Umf1Vw8rNx4BlP35WwCSY,11882
-typedal/core.py,sha256=hT4XLEMz_IzjTmWu2-scTP7VvrS0UwFSbG7A62OUnv0,94749
-typedal/fields.py,sha256=z2PD9vLWqBR_zXtiY0DthqTG4AeF3yxKoeuVfGXnSdg,5197
-typedal/for_py4web.py,sha256=dwh3cYThR6j2l_RDChcuSH3gXjv8FY-az3t5Lni5A7M,1302
-typedal/for_web2py.py,sha256=zvd5xC-SmuKc0JLDqT3hMIs6COaYnwTFXD_BIeC1vug,1832
-typedal/helpers.py,sha256=ZpHdwBMSANw-P9I5gs56Vf6GUbxGzFsIwbBvASKXX8s,6487
-typedal/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-typedal/types.py,sha256=5qm3PgS8DXGCu9ZTUWQiIi2XXD8gz4_4Csg_vZlu_yo,3379
-typedal/web2py_py4web_shared.py,sha256=cEbjkK0WOS9Q0nTyZuQaJWffeP4bjrL79Bx0xGy_UOs,1504
-typedal/serializers/as_json.py,sha256=ffo152W-sARYXym4BzwX709rrO2-QwKk2KunWY8RNl4,2229
-typedal-2.3.6.dist-info/METADATA,sha256=NVLY0EjdnmVKMpFMpPxnmyz9hEQN1ZrQEHccMRNtkYY,7573
-typedal-2.3.6.dist-info/WHEEL,sha256=KGYbc1zXlYddvwxnNty23BeaKzh7YuoSIvIMO4jEhvw,87
-typedal-2.3.6.dist-info/entry_points.txt,sha256=m1wqcc_10rHWPdlQ71zEkmJDADUAnZtn7Jac_6mbyUc,44
-typedal-2.3.6.dist-info/RECORD,,