dpone 0.1.0__py3-none-any.whl

dpone/.env.example

@@ -0,0 +1,70 @@
+# Конфигурация окружения для dpone ETL Framework
+#
+# ENV_CODE определяет окружение (dev/prod/etc.)
+# и используется как vault_mount_point для всех подключений
+#
+# Скопируйте этот файл в .env и настройте под своё окружение:
+#   cp .env.example .env
+
+# =============================================================================
+# ОБЯЗАТЕЛЬНЫЕ ПЕРЕМЕННЫЕ
+# =============================================================================
+
+# Абсолютный путь к корню dpone
+# Используется для определения MANIFEST_DIR (etl-process-manifest/)
+# Пример: /opt/airflow/plugins/dpone
+DPONE_PROJECT_DIR=/opt/airflow/plugins/dpone
+
+# Код окружения (vault mount point)
+# Возможные значения: dev, prod, stage
+# По умолчанию: dev (если не указано)
+ENV_CODE=dev
+
+# =============================================================================
+# ОПЦИОНАЛЬНЫЕ ПЕРЕМЕННЫЕ
+# =============================================================================
+
+# Директория для временных файлов экспорта
+# По умолчанию: системная временная директория
+# DPONE_EXPORT_TMP_DIR=/tmp
+
+# Уровень сжатия для gzip экспорта (1-9)
+# По умолчанию: 1 (быстрое сжатие)
+# DPONE_EXPORT_GZIP_LEVEL=1
+
+# Размер буфера для экспорта (в байтах)
+# По умолчанию: 16 MB
+# DPONE_EXPORT_BUFFER_SIZE=16777216
+
+# Порог размера файла для загрузки через GCS (в MB)
+# Файлы больше этого размера загружаются через GCS, меньше - напрямую
+# По умолчанию: 1000 MB
+# DPONE_GCS_FILE_SIZE_THRESHOLD_MB=1000
+
+# Максимальное количество плохих записей для BigQuery Load Job
+# По умолчанию: 100
+# DPONE_BQ_MAX_BAD_RECORDS=100
+
+# Размер batch для snapshot операций
+# По умолчанию: 50000
+# DPONE_SNAPSHOT_BATCH_SIZE=50000
+
+# =============================================================================
+# ИСПОЛЬЗОВАНИЕ ENV_CODE
+# =============================================================================
+#
+# ENV_CODE автоматически используется как vault_mount_point для:
+# 1. source.vault_mount_point (если не переопределён в YAML)
+# 2. sink.vault_mount_point (если не переопределён в YAML)
+# 3. proxy.vault_mount_point (если не переопределён в YAML)
+# 4. state.vault_mount_point (всегда, блок state можно не указывать в YAML)
+#
+# Для dev/prod vault_path автоматически подставляется:
+# - dev:  gcp/tripster-dp-dev/bq/dp-gbq-etl-svc
+# - prod: gcp/tripster-dp-prod/bq/dp-gbq-etl-svc
+#
+# Для других окружений (staging, test) требуется явно указать:
+#   state:
+#     vault_path: gcp/tripster-dp-staging/bq/dp-gbq-etl-svc
+#
+# =============================================================================

dpone/__init__.py

@@ -0,0 +1,61 @@
+"""dpone package exports.
+
+Keep imports lazy to avoid importing heavy/optional dependencies on `import dpone`.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _distribution_version
+from typing import Any
+
+__all__ = [
+    "__version__",
+    "PostgresConnector",
+    "ETLProcess",
+    "ETLProcessConfig",
+    "RunContext",
+    "PostgresSink",
+    "PostgresSource",
+    "LoadConfig",
+    "LoadStrategy",
+    "ReconciliationManager",
+    "XMinStateQueries",
+    "RunStateQueries",
+    "ReconciliationQueries",
+]
+
+try:
+    __version__ = _distribution_version("dpone")
+except PackageNotFoundError:  # pragma: no cover - editable installs provide metadata in tests
+    __version__ = "0.0.0"
+
+# name -> "module:attribute"
+_EXPORTS: dict[str, str] = {
+    "PostgresConnector": "dpone.lib.connectors:PostgresConnector",
+    "ETLProcess": "dpone.core:ETLProcess",
+    "ETLProcessConfig": "dpone.core:ETLProcessConfig",
+    "RunContext": "dpone.core:RunContext",
+    "PostgresSink": "dpone.sink:PostgresSink",
+    "PostgresSource": "dpone.source:PostgresSource",
+    "LoadConfig": "dpone.config:LoadConfig",
+    "LoadStrategy": "dpone.config:LoadStrategy",
+    "ReconciliationManager": "dpone.reconciliation:ReconciliationManager",
+    "XMinStateQueries": "dpone.sql_helpers:XMinStateQueries",
+    "RunStateQueries": "dpone.sql_helpers:RunStateQueries",
+    "ReconciliationQueries": "dpone.sql_helpers:ReconciliationQueries",
+}
+
+
+def __getattr__(name: str) -> Any:
+    target = _EXPORTS.get(name)
+    if not target:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+    module_name, attr = target.split(":")
+    mod = import_module(module_name)
+
+    value = getattr(mod, attr)
+    globals()[name] = value  # кешируем, чтобы второй раз не импортировать
+    return value

dpone/adapters/fs_local.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from pathlib import Path
+
+from ..ports.filesystem import FileSystem
+
+
+class LocalFileSystem(FileSystem):
+    def read_text(self, path: Path, *, encoding: str = "utf-8") -> str:
+        return path.read_text(encoding=encoding)
+
+    def write_text(self, path: Path, data: str, *, encoding: str = "utf-8") -> None:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(data, encoding=encoding)
+
+    def exists(self, path: Path) -> bool:
+        return path.exists()
+
+    def glob(self, root: Path, pattern: str) -> Iterable[Path]:
+        return root.glob(pattern)

dpone/adapters/fs_memory.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..ports.filesystem import FileSystem
+
+
+@dataclass
+class InMemoryFileSystem(FileSystem):
+    """Tiny in-memory FS for unit tests."""
+
+    files: dict[str, str] = field(default_factory=dict)
+
+    def read_text(self, path: Path, *, encoding: str = "utf-8") -> str:
+        key = str(path)
+        if key not in self.files:
+            raise FileNotFoundError(key)
+        return self.files[key]
+
+    def write_text(self, path: Path, data: str, *, encoding: str = "utf-8") -> None:
+        self.files[str(path)] = data
+
+    def exists(self, path: Path) -> bool:
+        return str(path) in self.files
+
+    def glob(self, root: Path, pattern: str) -> Iterable[Path]:
+        # Minimal glob: not needed for now.
+        raise NotImplementedError("InMemoryFileSystem.glob is not implemented")

dpone/adapters/yaml_pyyaml.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from typing import Any
+
+import yaml
+
+from ..ports.yaml_codec import YamlCodec
+
+
+class PyYamlCodec(YamlCodec):
+    def load(self, text: str) -> Any:
+        return yaml.safe_load(text)
+
+    def dump(self, obj: Any) -> str:
+        # diff-friendly defaults
+        return yaml.safe_dump(obj, sort_keys=False, allow_unicode=True)

dpone/app/cli_reference_source.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+def build_root_parser():
+    """Return the canonical argparse tree for CLI reference generation.
+
+    Kept in the app layer so docs/services can depend on it without importing
+    dpone.cli.* directly.
+    """
+
+    from ..cli.parser import build_parser
+
+    return build_parser()

dpone/app/context.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+
+from ..adapters.fs_local import LocalFileSystem
+from ..adapters.yaml_pyyaml import PyYamlCodec
+from ..ports.filesystem import FileSystem
+from ..ports.yaml_codec import YamlCodec
+from .settings import Settings
+
+
+@dataclass(frozen=True)
+class AppContext:
+    """Composition root (DI container) for dpone CLI.
+
+    In dpone we intentionally avoid a heavy DI framework.
+    AppContext is the single place where we build and cache dependencies.
+
+    In next refactoring steps we will attach services here.
+    """
+
+    settings: Settings
+    logger: logging.Logger
+    fs: FileSystem
+    yaml: YamlCodec
+
+    @classmethod
+    def from_env(cls, *, logger: logging.Logger) -> AppContext:
+        settings = Settings.from_env()
+        fs = LocalFileSystem()
+        yaml = PyYamlCodec()
+        return cls(settings=settings, logger=logger, fs=fs, yaml=yaml)

dpone/app/logging.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import logging
+import os
+
+
+def setup_logging() -> logging.Logger:
+    """Configure dpone logger.
+
+    KISS:
+      - no external logging framework
+      - env-controlled level
+    """
+
+    level_raw = os.getenv("DPONE_LOG_LEVEL", "INFO").upper().strip()
+    level = getattr(logging, level_raw, logging.INFO)
+
+    logging.basicConfig(
+        level=level,
+        format="%(levelname)s: %(message)s",
+    )
+
+    return logging.getLogger("dpone")

dpone/app/settings.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Typed settings for dpone.
+
+    Goal:
+      - keep defaults predictable
+      - avoid module-level side effects
+      - keep CLI runnable in local dev
+
+    NOTE:
+      Runtime/Airflow may still rely on DPONE_PROJECT_DIR, but Settings should
+      degrade gracefully for local tooling (docs/metrics).
+    """
+
+    repo_root: Path
+    project_dir: Path
+    manifest_dir: Path
+    sources_registry_paths: tuple[Path, ...]
+
+    @staticmethod
+    def _detect_repo_root(start: Path) -> Path:
+        """Best-effort repo root detection.
+
+        We search upwards for `pyproject.toml`.
+        If not found, fallback to `start`.
+        """
+
+        cur = start.resolve()
+        for _ in range(10):
+            if (cur / "pyproject.toml").exists():
+                return cur
+            if cur.parent == cur:
+                break
+            cur = cur.parent
+        return start.resolve()
+
+    @classmethod
+    def from_env(cls, *, cwd: Path | None = None) -> Settings:
+        cwd = (cwd or Path.cwd()).resolve()
+        repo_root = cls._detect_repo_root(cwd)
+
+        project_dir_raw = os.getenv("DPONE_PROJECT_DIR", "").strip()
+        project_dir = Path(project_dir_raw).resolve() if project_dir_raw else repo_root
+
+        manifest_dir = project_dir / "etl-process-manifest"
+
+        raw_registry = os.getenv("DPONE_SOURCES_REGISTRY", "").strip()
+        paths: list[Path] = []
+        if raw_registry:
+            for part in [p.strip() for p in raw_registry.split(",") if p.strip()]:
+                p = Path(part)
+                if not p.is_absolute():
+                    p = project_dir / p
+                paths.append(p)
+
+        return cls(
+            repo_root=repo_root,
+            project_dir=project_dir,
+            manifest_dir=manifest_dir,
+            sources_registry_paths=tuple(paths),
+        )

dpone/cli/__init__.py

@@ -0,0 +1,7 @@
+"""dpone CLI.
+
+The CLI is primarily a UX/debugging aid:
+- list processes in a manifest (single or batch)
+- render the effective process config for a selector
+- validate naming/metadata conventions in CI
+"""

dpone/cli/legacy.py

@@ -0,0 +1,74 @@
+"""Deprecated compatibility shim for the old CLI module.
+
+`dpone.cli.legacy` used to contain a very large monolithic argparse-based CLI.
+All command implementations now live in:
+
+- ``dpone.cli.main`` / ``dpone.cli.parser``
+- ``dpone.commands.*``
+- ``dpone.services.*``
+- ``dpone.cli_render.*``
+
+This module stays only to preserve compatibility for code that still imports
+``dpone.cli.legacy.main`` or executes ``python -m dpone.cli.legacy``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import warnings
+from typing import Final
+
+from .main import main as _main
+from .parser import build_parser as _build_parser
+
+_DEPRECATION_MESSAGE: Final[str] = (
+    "`dpone.cli.legacy` is deprecated; use `dpone.cli.main` / `dpone` instead. "
+    "The old monolithic CLI module has been replaced with commands/services/renderers."
+)
+
+
+_REMOVED_EXPORTS: Final[dict[str, str]] = {
+    "run_legacy": "Use `dpone` / `dpone.cli.main.main(...)` or specific command modules.",
+    "_cmd_manifest_list": "Manifest commands now live in `dpone.commands.manifest.*`.",
+    "_cmd_manifest_render": "Manifest commands now live in `dpone.commands.manifest.*`.",
+    "_cmd_manifest_explain": "Manifest commands now live in `dpone.commands.manifest.*`.",
+    "_cmd_manifest_validate": "Manifest commands now live in `dpone.commands.manifest.*`.",
+    "_cmd_manifest_migrate": "Manifest commands now live in `dpone.commands.manifest.*`.",
+    "_cmd_manifest_verify": "Manifest commands now live in `dpone.commands.manifest.*`.",
+    "_cmd_dag_explain_edge": "DAG commands now live in `dpone.commands.dag.*`.",
+    "_cmd_dag_explain_edge_e2e": "DAG commands now live in `dpone.commands.dag.*`.",
+    "_cmd_dag_explain_node": "DAG commands now live in `dpone.commands.dag.*`.",
+    "_cmd_dag_explain_node_e2e": "DAG commands now live in `dpone.commands.dag.*`.",
+    "_cmd_dag_list_edges": "DAG commands now live in `dpone.commands.dag.*`.",
+    "_cmd_dag_subgraph": "DAG commands now live in `dpone.commands.dag.*`.",
+    "_cmd_dag_report": "DAG commands now live in `dpone.commands.dag.*`.",
+}
+
+
+def _warn() -> None:
+    warnings.warn(_DEPRECATION_MESSAGE, DeprecationWarning, stacklevel=2)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Return the canonical parser, but emit a deprecation warning."""
+
+    _warn()
+    return _build_parser()
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Run the canonical CLI through the deprecated legacy entrypoint."""
+
+    _warn()
+    _main(argv)
+
+
+def __getattr__(name: str):
+    if name in _REMOVED_EXPORTS:
+        raise AttributeError(f"`dpone.cli.legacy.{name}` was removed. {_REMOVED_EXPORTS[name]}")
+    raise AttributeError(name)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

dpone/cli/main.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import sys
+
+from ..app.context import AppContext
+from ..app.logging import setup_logging
+from ..contracts.errors import ETLConfigurationError
+from .parser import build_parser
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Run the canonical dpone CLI.
+
+    Parsing happens before AppContext construction so `--help` and parser errors
+    remain lightweight and do not touch filesystem/env-heavy setup.
+    """
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    cmd = getattr(args, "_command", None)
+    if cmd is None:
+        parser.print_help()
+        raise SystemExit(2)
+
+    logger = setup_logging()
+    ctx = AppContext.from_env(logger=logger)
+
+    try:
+        code = int(cmd.run(args, ctx))
+    except ETLConfigurationError as e:
+        logger.error(str(e))
+        raise SystemExit(2)
+    except KeyboardInterrupt:
+        logger.error("Interrupted")
+        raise SystemExit(130)
+
+    raise SystemExit(code)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

dpone/cli/parser.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import argparse
+
+from ..commands.registry import get_commands
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="dpone")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    for cmd in get_commands():
+        cmd.register(sub)
+
+    return parser

dpone/cli_render/__init__.py

@@ -0,0 +1,12 @@
+"""CLI renderers.
+
+Renderers are responsible for converting internal objects into human-friendly
+text output (tables, headings, YAML evidence blocks).
+
+Design goals:
+- keep command modules small (parse args -> compute -> render)
+- keep rendering logic reusable and testable
+- do not import runtime-heavy optional dependencies
+"""
+
+from __future__ import annotations

dpone/cli_render/dag/__init__.py

@@ -0,0 +1,3 @@
+"""DAG CLI renderers."""
+
+from __future__ import annotations

dpone/cli_render/dag/common.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from dpone.output import dumps_yaml
+from dpone.output_table import render_table
+
+HR = "-" * 80
+HR2 = "=" * 80
+
+
+def compact_yaml(value: Any) -> str:
+    """Make values diff-friendly in a single line.
+
+    Used by "end-to-end" explain commands where we want to show the exact
+    depends_on item but keep output compact.
+    """
+
+    if value is None:
+        return "null"
+    if isinstance(value, str | int | float | bool):
+        return json.dumps(value, ensure_ascii=False)
+
+    s = yaml.safe_dump(value, sort_keys=True, allow_unicode=True).strip()
+    # Collapse to one line
+    return " ".join(line.strip() for line in s.splitlines())
+
+
+def _node_line(prefix: str, key: str, value: Any) -> str:
+    return f"{prefix}{key}: {value}" if value is not None and value != "" else ""
+
+
+def render_dag_banner(*, root: Path, base_path: Path, task_count: int, extra: Sequence[str] | None = None) -> str:
+    lines = [
+        f"DAG root: {root}",
+        f"Base dir: {base_path}",
+        f"Tasks:    {task_count}",
+    ]
+    if extra:
+        for x in extra:
+            if x:
+                lines.append(str(x))
+    return "\n".join(lines)
+
+
+def render_node_block(title: str, node: Mapping[str, Any]) -> str:
+    """Render a node dict as printed by explain_* helpers."""
+
+    lines: list[str] = [f"\n{title}:"]
+    lines.append(f"  name:      {node.get('name')}")
+    lines.append(f"  ref:       {node.get('ref')}")
+
+    if node.get("task_group"):
+        lines.append(f"  task_group:{node.get('task_group')}")
+    if node.get("source"):
+        lines.append(f"  source:    {node.get('source')}")
+    if node.get("sink"):
+        lines.append(f"  sink:      {node.get('sink')}")
+
+    return "\n".join(lines)
+
+
+def render_task_block(title: str, task: Mapping[str, Any]) -> str:
+    # alias for older naming
+    return render_node_block(title, task)
+
+
+def render_warnings(warnings: Iterable[str]) -> str:
+    out = []
+    for w in warnings or ():
+        out.append(f"WARN: {w}")
+    return "\n".join(out)
+
+
+# ---------------- reasons ----------------
+
+
+def _reason_kind(reason: Any) -> str:
+    if isinstance(reason, dict):
+        return str(reason.get("kind") or "")
+    return str(getattr(reason, "kind", ""))
+
+
+def _reason_description(reason: Any) -> str:
+    if isinstance(reason, dict):
+        return str(reason.get("description") or "")
+    return str(getattr(reason, "description", ""))
+
+
+def _reason_evidence(reason: Any) -> dict:
+    if isinstance(reason, dict):
+        ev = reason.get("evidence")
+        return ev if isinstance(ev, dict) else {}
+    ev = getattr(reason, "evidence", None)
+    return ev if isinstance(ev, dict) else {}
+
+
+def render_reasons_section(
+    reasons: Sequence[Any],
+    *,
+    title: str = "Reasons",
+    include_evidence: bool = True,
+) -> str:
+    if not reasons:
+        return ""
+
+    lines: list[str] = [f"\n{title}:"]
+    rows = [[_reason_kind(r), _reason_description(r)] for r in reasons]
+    lines.append(render_table(["kind", "description"], rows))
+
+    if include_evidence:
+        for r in reasons:
+            ev = _reason_evidence(r)
+            if not ev:
+                continue
+            kind = _reason_kind(r)
+            lines.append(f"\n[{kind}] evidence:")
+            lines.append(dumps_yaml(ev, sort_keys=True).rstrip())
+
+    return "\n".join(lines)
+
+
+def render_reasons_compact(reasons: Sequence[Any]) -> str:
+    if not reasons:
+        return "(no attributed reasons)"
+    lines = [f"- {_reason_kind(r)}: {_reason_description(r)}" for r in reasons]
+    return "\n".join(lines)
+
+
+def render_table_section(title: str, headers: Sequence[str], rows: Sequence[Sequence[Any]]) -> str:
+    lines: list[str] = [f"\n{title}:"]
+    lines.append(render_table(list(headers), [list(r) for r in rows]))
+    return "\n".join(lines)
+
+
+def render_yaml_block(title: str, obj: Any) -> str:
+    lines: list[str] = [f"\n{title}:"]
+    lines.append(dumps_yaml(obj, sort_keys=True).rstrip())
+    return "\n".join(lines)