dbly 0.3.0__tar.gz → 0.4.0__tar.gz

{dbly-0.3.0 → dbly-0.4.0}/CHANGELOG.md

@@ -7,6 +7,34 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [0.4.0] — 2026-07-23
+
+### Added
+
+- **Project configuration (`dbly.toml`).** An optional file at the repo root, for repos that
+  need more than a single connection profile:
+  - `object_root` — the subtree the declarative object files live under. The schema hint is
+    taken from the first path segment *below* this root, so a layout like
+    `pgsql/schema/<schema>/<obj>` maps to the right schema instead of the literal top folder.
+  - `environment` — default engine/dialect when a profile omits `environment=`.
+  - `[targets]` — named connection profiles, so `dbly plan --target dev` resolves to a
+    profile path.
+  - `ignore` — extra ignore patterns (gitwildmatch), merged with `.dbignore` (e.g. to skip a
+    handful of files a parser cannot read).
+
+  Absent config → unchanged behaviour (`object_root="."`, `--target` is a profile path).
+
+### Fixed
+
+- **Case-insensitive identifier matching on PostgreSQL.** Postgres folds unquoted identifiers
+  to lower case, so a DDL `CREATE TABLE FOO` lives as `foo`. `plan` (`table_exists`/
+  `get_columns`) and `check` looked the object up by the *declared* name and either planned a
+  spurious full `CREATE` or crashed with `NoSuchTableError`. Both now resolve the object
+  case-insensitively and reflect it by its real, live identity.
+- **`check` no longer aborts on a single unreflectable table.** A table whose columns cannot
+  be introspected is reported as advisory `unreadable` and the drift scan continues, instead
+  of failing the whole run.
+
 ## [0.3.0] — 2026-07-21
 
 ### Added

{dbly-0.3.0 → dbly-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbly
-Version: 0.3.0
+Version: 0.4.0
 Summary: State-based, cross-engine database deployment — git-driven, parser-assisted, SQL-first.
 Project-URL: Homepage, https://angrydata.info/dbly
 Project-URL: Repository, https://github.com/angrydat/dbly
@@ -93,6 +93,27 @@ changes the diff can't do safely (renaming a column, moving data), drop an order
 it touches defers to it for that deploy. On a fresh database, migrations are *baselined*
 (recorded, not run) since the object files already describe the end state.
 
+## Project configuration (optional)
+
+For anything beyond the defaults, drop a `dbly.toml` at the repo root. It's optional — without
+it, `--target` is a profile path and the schema is the first folder segment.
+
+```toml
+object_root = "pgsql/schema"     # objects live here; schema = first segment BELOW this root
+environment = "postgres"         # default engine when a profile omits environment=
+ignore = ["**/*_1252.sql"]       # extra ignore patterns, merged with .dbignore
+
+[targets]                        # dbly plan --target dev  →  resolves to the profile path
+dev  = "conn/dev.properties"
+beta = "conn/beta.properties"
+prod = "conn/prod.properties"
+```
+
+`object_root` is the key that lets dbly sit on an existing repo whose objects live deeper than
+the top level (e.g. `pgsql/schema/<schema>/<object>`): the schema is derived *relative to the
+root*, so `pgsql/schema/sales/customer.tbl` maps to schema `sales`, not `pgsql`. Put root-level
+keys (`object_root`, `environment`, `ignore`) **before** the `[targets]` table (TOML rule).
+
 ## Connect
 
 A connection profile (reuses the familiar `connection.properties` format):

{dbly-0.3.0 → dbly-0.4.0}/README.md

@@ -54,6 +54,27 @@ changes the diff can't do safely (renaming a column, moving data), drop an order
 it touches defers to it for that deploy. On a fresh database, migrations are *baselined*
 (recorded, not run) since the object files already describe the end state.
 
+## Project configuration (optional)
+
+For anything beyond the defaults, drop a `dbly.toml` at the repo root. It's optional — without
+it, `--target` is a profile path and the schema is the first folder segment.
+
+```toml
+object_root = "pgsql/schema"     # objects live here; schema = first segment BELOW this root
+environment = "postgres"         # default engine when a profile omits environment=
+ignore = ["**/*_1252.sql"]       # extra ignore patterns, merged with .dbignore
+
+[targets]                        # dbly plan --target dev  →  resolves to the profile path
+dev  = "conn/dev.properties"
+beta = "conn/beta.properties"
+prod = "conn/prod.properties"
+```
+
+`object_root` is the key that lets dbly sit on an existing repo whose objects live deeper than
+the top level (e.g. `pgsql/schema/<schema>/<object>`): the schema is derived *relative to the
+root*, so `pgsql/schema/sales/customer.tbl` maps to schema `sales`, not `pgsql`. Put root-level
+keys (`object_root`, `environment`, `ignore`) **before** the `[targets]` table (TOML rule).
+
 ## Connect
 
 A connection profile (reuses the familiar `connection.properties` format):

{dbly-0.3.0 → dbly-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dbly"
-version = "0.3.0"
+version = "0.4.0"
 description = "State-based, cross-engine database deployment — git-driven, parser-assisted, SQL-first."
 readme = "README.md"
 license = { text = "MIT" }

{dbly-0.3.0 → dbly-0.4.0}/src/dbly/__init__.py

@@ -4,4 +4,4 @@ git (what changed) + sqlglot (what it is) + live introspection (what's really th
 See CONCEPT.md for the design rationale.
 """
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"

{dbly-0.3.0 → dbly-0.4.0}/src/dbly/adapters/postgres.py

@@ -26,13 +26,49 @@ class PostgresAdapter(Adapter):
     transactional_ddl = True
     default_schema = "public"
 
+    def _resolve(self, schema: str | None, name: str) -> tuple[str, str] | None:
+        """Find a relation case-insensitively, returning its real (schema, name).
+
+        Postgres folds unquoted identifiers to lower case, so a DDL ``CREATE TABLE FOO`` lives
+        as ``foo``; the desired name (as written) need not match verbatim. Exact matches are
+        preferred over case-folded ones when both exist.
+        """
+        params = {"n": name}
+        schema_pred = ""
+        if schema is not None:
+            schema_pred = "AND lower(n.nspname) = lower(:s) "
+            params["s"] = schema
+        q = text(
+            "SELECT n.nspname, c.relname FROM pg_class c "
+            "JOIN pg_namespace n ON n.oid = c.relnamespace "
+            f"WHERE lower(c.relname) = lower(:n) {schema_pred}"
+            "AND c.relkind IN ('r','p','v','m') "
+            "ORDER BY (c.relname = :n) DESC LIMIT 1"
+        )
+        with self.engine.connect() as conn:
+            row = conn.execute(q, params).first()
+        return (row[0], row[1]) if row else None
+
     def table_exists(self, schema: str | None, name: str) -> bool:
-        insp = inspect(self.engine)
-        return insp.has_table(name, schema=schema)
+        params = {"n": name}
+        schema_pred = ""
+        if schema is not None:
+            schema_pred = "AND lower(n.nspname) = lower(:s) "
+            params["s"] = schema
+        with self.engine.connect() as conn:
+            return conn.execute(
+                text(
+                    "SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace "
+                    f"WHERE lower(c.relname) = lower(:n) {schema_pred}"
+                    "AND c.relkind IN ('r','p') LIMIT 1"
+                ),
+                params,
+            ).first() is not None
 
     def get_columns(self, schema: str | None, name: str) -> list[Column]:
         insp = inspect(self.engine)
-        cols = insp.get_columns(name, schema=schema)
+        rs, rn = self._resolve(schema, name) or (schema, name)
+        cols = insp.get_columns(rn, schema=rs)
         return [
             Column(
                 name=c["name"],

{dbly-0.3.0 → dbly-0.4.0}/src/dbly/cli.py

@@ -16,11 +16,12 @@ from rich.console import Console
 
 from dbly import __version__, drift, hooks, initializer, report
 from dbly.adapters import get_adapter
-from dbly.config import resolve_target
+from dbly.config import ConnectionConfig, load_profile, resolve_target
 from dbly.engine import detect_dialect
 from dbly.parsing import sqlglot_dialect
 from dbly.planner import build_plan
 from dbly.model import Plan, Severity
+from dbly.project import ProjectConfig, load_project
 from dbly.repo import WORKTREE, Repo
 
 app = typer.Typer(
@@ -48,6 +49,24 @@ def _main(
     pass
 
 
+def _open_repo(repo_path: Path, project: ProjectConfig) -> Repo:
+    return Repo(repo_path, object_root=project.object_root, extra_ignore=project.ignore)
+
+
+def _resolve_target(project: ProjectConfig, repo_path: Path, target: str) -> ConnectionConfig:
+    """Resolve ``--target``: a named target from ``dbly.toml`` [targets], else a profile path.
+
+    The project's ``environment`` fills in when the profile omits ``environment=``.
+    """
+    if target in project.targets:
+        cfg = load_profile(repo_path / project.targets[target])
+    else:
+        cfg = resolve_target(target)
+    if cfg.environment is None and project.environment:
+        cfg.environment = project.environment
+    return cfg
+
+
 def _decorations(repo_path: Path, plan: Plan) -> dict[str, str]:
     """Map each real ref in the plan to its git tag/branch names (for the plan header)."""
     try:
@@ -68,8 +87,9 @@ def _make_plan(
     repo_path: Path, target: str, from_ref: Optional[str], to_ref: str,
     *, worktree: bool = False,
 ) -> Plan:
-    repo = Repo(repo_path)
-    cfg = resolve_target(target)
+    project = load_project(repo_path)
+    repo = _open_repo(repo_path, project)
+    cfg = _resolve_target(project, repo_path, target)
     dialect = sqlglot_dialect(detect_dialect(cfg))
     adapter = get_adapter(cfg)
     try:
@@ -113,7 +133,7 @@ def plan(
         console.print(f"\n[dim]plan (YAML) written to {out}[/dim]")
     if sql:
         # state_table_ddl / record_deploy_sql are pure string builders — no DB connection.
-        adapter = get_adapter(resolve_target(target))
+        adapter = get_adapter(_resolve_target(load_project(repo_path), repo_path, target))
         try:
             script = report.plan_to_sql(
                 plan_obj,
@@ -165,8 +185,9 @@ def apply(
         console.print("[green]nothing to apply.[/green]")
         return
 
-    repo = Repo(repo_path)
-    cfg = resolve_target(target)
+    project = load_project(repo_path)
+    repo = _open_repo(repo_path, project)
+    cfg = _resolve_target(project, repo_path, target)
     adapter = get_adapter(cfg)
     try:
         _run_hooks(repo, "pre", py_interpreter)
@@ -204,12 +225,13 @@ def init(
 
     Greenfield only — brownfield (a handed-over database) skips this entirely.
     """
-    repo = Repo(repo_path)
+    project = load_project(repo_path)
+    repo = _open_repo(repo_path, project)
     scripts = initializer.discover_init_scripts(repo.root, init_dir)
     if not scripts:
         console.print(f"[yellow]no init scripts in {init_dir}/ — nothing to do.[/yellow]")
         return
-    adapter = get_adapter(resolve_target(init_target))
+    adapter = get_adapter(_resolve_target(project, repo_path, init_target))
     try:
         for s in scripts:
             adapter.run_init_script(s.read_text(encoding="utf-8"))
@@ -270,8 +292,9 @@ def check(
     ),
 ) -> None:
     """Detect drift: compare the desired state at <to> against the live database."""
-    repo = Repo(repo_path)
-    cfg = resolve_target(target)
+    project = load_project(repo_path)
+    repo = _open_repo(repo_path, project)
+    cfg = _resolve_target(project, repo_path, target)
     dialect = sqlglot_dialect(detect_dialect(cfg))
     adapter = get_adapter(cfg)
     try:
@@ -283,7 +306,7 @@ def check(
     finally:
         adapter.dispose()
 
-    if rep.clean:
+    if rep.clean and not rep.unreadable:
         console.print("[green]no drift — database matches the desired state.[/green]")
         return
 
@@ -298,7 +321,10 @@ def check(
         console.print(f"[yellow]changed[/yellow]   {kind.value} {oid}  (definition differs — advisory)")
     for kind, oid in rep.orphaned:
         console.print(f"[dim]orphaned[/dim]  {kind.value} {oid}  (in DB, not in repo)")
-    raise typer.Exit(code=1)
+    for kind, oid in rep.unreadable:
+        console.print(f"[dim]unreadable[/dim] {kind.value} {oid}  (columns could not be reflected — advisory)")
+    if not rep.clean:
+        raise typer.Exit(code=1)
 
 
 def _run_hooks(repo: Repo, phase: str, py_interpreter: str) -> None:

{dbly-0.3.0 → dbly-0.4.0}/src/dbly/drift.py

@@ -36,6 +36,7 @@ class DriftReport:
     orphaned: list[tuple[ObjectKind, ObjectId]] = field(default_factory=list)
     columns: list[ColumnDrift] = field(default_factory=list)
     definitions: list[tuple[ObjectKind, ObjectId]] = field(default_factory=list)
+    unreadable: list[tuple[ObjectKind, ObjectId]] = field(default_factory=list)  # reflection failed (advisory)
 
     @property
     def clean(self) -> bool:
@@ -83,7 +84,15 @@ def compute_drift(
 
     for key, obj in desired.items():
         if obj.kind is ObjectKind.TABLE and key in live:
-            actual = {c.key() for c in adapter.get_columns(obj.id.schema, obj.id.name)}
+            # Reflect columns by the LIVE object's identity — the engine may have folded the
+            # declared name's case (unquoted `FOO` → `foo` on Postgres), so the desired name
+            # need not exist verbatim. A table that cannot be reflected is skipped, not fatal.
+            live_id = live[key].id
+            try:
+                actual = {c.key() for c in adapter.get_columns(live_id.schema, live_id.name)}
+            except Exception:  # noqa: BLE001 — one odd table must not abort the whole check
+                report.unreadable.append((obj.kind, obj.id))
+                continue
             want = {c.key() for c in parsing.desired_columns(obj.sql, dialect=dialect)}
             added, removed = sorted(want - actual), sorted(actual - want)
             if added or removed:

dbly-0.4.0/src/dbly/project.py

@@ -0,0 +1,54 @@
+"""Project configuration — an optional ``dbly.toml`` at the repository root.
+
+The flag-driven workflow (``--target <profile>`` + ``.dbignore``) stays the default; a
+``dbly.toml`` adds the terraform-style project setup a real repo needs:
+
+* **object_root** — the subtree the declarative object files live under. The schema hint is the
+  first path segment *below* this root, so a repo laid out as ``pgsql/schema/<schema>/<obj>``
+  maps to the right schema instead of the literal top folder.
+* **environment** — the default engine/dialect when a profile omits ``environment=``.
+* **[targets]** — named connection profiles, so ``dbly plan --target dev`` resolves to a
+  profile path instead of spelling it out each time.
+* **ignore** — extra ignore patterns (gitwildmatch), merged with ``.dbignore`` — e.g. to
+  exclude a handful of files a parser cannot read.
+
+Absent config → :class:`ProjectConfig` defaults (``object_root="."``, no named targets), i.e.
+the previous behaviour unchanged.
+"""
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+
+CONFIG_NAME = "dbly.toml"
+
+
+@dataclass(slots=True)
+class ProjectConfig:
+    object_root: str = "."
+    environment: str | None = None
+    targets: dict[str, str] = field(default_factory=dict)
+    ignore: list[str] = field(default_factory=list)
+
+
+def load_project(repo_root: Path) -> ProjectConfig:
+    """Load ``<repo_root>/dbly.toml`` if present, else return defaults."""
+    path = repo_root / CONFIG_NAME
+    if not path.exists():
+        return ProjectConfig()
+    data = tomllib.loads(path.read_text(encoding="utf-8"))
+
+    targets = data.get("targets") or {}
+    if not isinstance(targets, dict):
+        raise ValueError(f"{CONFIG_NAME}: [targets] must be a table of name = \"profile-path\"")
+    ignore = data.get("ignore") or []
+    if not isinstance(ignore, list):
+        raise ValueError(f"{CONFIG_NAME}: ignore must be a list of patterns")
+
+    return ProjectConfig(
+        object_root=str(data.get("object_root", ".")),
+        environment=data.get("environment"),
+        targets={str(k): str(v) for k, v in targets.items()},
+        ignore=[str(p) for p in ignore],
+    )

{dbly-0.3.0 → dbly-0.4.0}/src/dbly/repo.py

@@ -27,11 +27,21 @@ class FileChange:
 
 
 class Repo:
-    def __init__(self, root: Path):
+    def __init__(
+        self,
+        root: Path,
+        *,
+        object_root: str | None = None,
+        extra_ignore: list[str] | None = None,
+    ):
         self.root = root.resolve()
         if not (self.root / ".git").exists():
             raise ValueError(f"not a git repository: {self.root}")
-        self._ignore = self._load_dbignore()
+        # the subtree object files live under; the schema hint is the first segment *below* it.
+        self.object_root = (
+            Path(object_root) if object_root and object_root not in (".", "") else None
+        )
+        self._ignore = self._load_dbignore(extra_ignore or [])
 
     def _git(self, *args: str) -> str:
         out = subprocess.run(
@@ -42,10 +52,10 @@ class Repo:
         )
         return out.stdout
 
-    def _load_dbignore(self) -> pathspec.PathSpec:
+    def _load_dbignore(self, extra: list[str]) -> pathspec.PathSpec:
         f = self.root / ".dbignore"
         lines = f.read_text(encoding="utf-8").splitlines() if f.exists() else []
-        return pathspec.PathSpec.from_lines("gitwildmatch", lines)
+        return pathspec.PathSpec.from_lines("gitwildmatch", [*lines, *extra])
 
     def is_ignored(self, rel: Path) -> bool:
         return self._ignore.match_file(rel.as_posix())
@@ -58,9 +68,19 @@ class Repo:
     def _is_migration(rel: Path) -> bool:
         return len(rel.parts) > 0 and rel.parts[0] == MIGRATIONS_DIR
 
+    def _under_object_root(self, rel: Path) -> bool:
+        if self.object_root is None:
+            return True
+        return rel == self.object_root or self.object_root in rel.parents
+
     def _is_object(self, rel: Path) -> bool:
-        """A deployable declarative object file (SQL, not a migration, not ignored)."""
-        return self._is_sql(rel) and not self._is_migration(rel) and not self.is_ignored(rel)
+        """A deployable declarative object file (SQL, under object_root, not a migration/ignored)."""
+        return (
+            self._is_sql(rel)
+            and not self._is_migration(rel)
+            and self._under_object_root(rel)
+            and not self.is_ignored(rel)
+        )
 
     def _untracked_files(self) -> list[Path]:
         """New files in the working tree not yet added to git (``git status`` "??")."""
@@ -169,8 +189,16 @@ class Repo:
     def schema_for(self, rel: Path) -> str | None:
         """Best-practice convention: the first path segment names the DB schema.
 
-        Only a *hint* — the parser overrides it when the DDL is schema-qualified.
-        Returns None when the file sits at the repo root.
+        With an ``object_root`` set, the segment is taken *relative to that root* — so
+        ``pgsql/schema/bas/foo.tbl`` under root ``pgsql/schema`` yields schema ``bas``.
+        Only a *hint* — the parser overrides it when the DDL is schema-qualified. Returns
+        None when the file sits directly at the (object) root.
         """
-        parts = rel.parts
+        r = rel
+        if self.object_root is not None:
+            try:
+                r = rel.relative_to(self.object_root)
+            except ValueError:
+                return None
+        parts = r.parts
         return parts[0] if len(parts) > 1 else None

{dbly-0.3.0 → dbly-0.4.0}/tests/test_integration.py

@@ -368,3 +368,50 @@ def test_repo_ref_names_decorates_tag_and_branch(tmp_path: Path):
     names = repo.ref_names(sha)
     assert "v0.1" in names
     assert any(n in ("main", "master") for n in names)  # the current branch points here too
+
+
+def test_object_root_scopes_discovery_and_schema(tmp_path: Path):
+    """object_root: schema = first segment BELOW the root; files outside it are ignored."""
+    from dbly import parsing
+    repo_root = tmp_path / "db"
+    (repo_root / "pgsql" / "schema" / "bas").mkdir(parents=True)
+    (repo_root / "ora").mkdir(parents=True)
+    _init_repo(repo_root)
+    # unqualified CREATE (schema comes from the folder), under pgsql/schema/bas
+    (repo_root / "pgsql" / "schema" / "bas" / "kunde.tbl").write_text(
+        "CREATE TABLE IF NOT EXISTS kunde (id INTEGER, name TEXT);", encoding="utf-8"
+    )
+    # an Oracle file outside object_root must NOT be discovered
+    (repo_root / "ora" / "irrelevant.tbl").write_text(
+        "CREATE TABLE junk (id NUMBER);", encoding="utf-8"
+    )
+    _commit(repo_root, "v1")
+
+    repo = Repo(repo_root, object_root="pgsql/schema")
+    files = repo.list_files("HEAD")
+    assert files == [Path("pgsql/schema/bas/kunde.tbl")]          # ora/ excluded
+    assert repo.schema_for(files[0]) == "bas"                      # segment below object_root
+
+    # parse with that hint → object id carries the real schema, not "pgsql"
+    sql = repo.read_at("HEAD", files[0])
+    obj = parsing.parse_file(sql, files[0], default_schema=repo.schema_for(files[0]),
+                             dialect="postgres")[0]
+    assert str(obj.id) == "bas.kunde"
+
+
+def test_extra_ignore_excludes_unparseable_files(tmp_path: Path):
+    repo_root = tmp_path / "db"
+    (repo_root / "schema" / "app").mkdir(parents=True)
+    _init_repo(repo_root)
+    (repo_root / "schema" / "app" / "good.tbl").write_text(
+        "CREATE TABLE app.good (id INTEGER);", encoding="utf-8"
+    )
+    (repo_root / "schema" / "app" / "bad.vw").write_text(
+        "CREATE VIEW app.bad AS SELECT 1;", encoding="utf-8"
+    )
+    _commit(repo_root, "v1")
+
+    repo = Repo(repo_root, object_root="schema", extra_ignore=["schema/app/bad.vw"])
+    files = repo.list_files("HEAD")
+    assert Path("schema/app/good.tbl") in files
+    assert Path("schema/app/bad.vw") not in files                 # ignored via extra_ignore

dbly-0.4.0/tests/test_project.py

@@ -0,0 +1,35 @@
+"""Tests for dbly.toml project configuration and object_root scoping."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from dbly.project import ProjectConfig, load_project
+
+
+def test_load_project_absent_returns_defaults(tmp_path: Path):
+    cfg = load_project(tmp_path)
+    assert cfg == ProjectConfig()
+    assert cfg.object_root == "." and cfg.targets == {} and cfg.ignore == []
+
+
+def test_load_project_parses_toml(tmp_path: Path):
+    (tmp_path / "dbly.toml").write_text(
+        """
+        object_root = "pgsql/schema"
+        environment = "postgres"
+        ignore = ["pgsql/schema/ggn/vw_bad.vw", "**/deploy-*.sql"]
+
+        [targets]
+        dev  = "test/dbly.dev.properties"
+        prod = "test/dbly.prod.properties"
+        """,
+        encoding="utf-8",
+    )
+    cfg = load_project(tmp_path)
+    assert cfg.object_root == "pgsql/schema"
+    assert cfg.environment == "postgres"
+    assert cfg.targets == {
+        "dev": "test/dbly.dev.properties",
+        "prod": "test/dbly.prod.properties",
+    }
+    assert "**/deploy-*.sql" in cfg.ignore