clearmetric-core 0.2.0__tar.gz

clearmetric_core-0.2.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: clearmetric-core
+Version: 0.2.0
+Summary: ClearMetric Core — local compiler, graph engine, and CLI foundation.
+Author: ClearMetric Labs
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/ClearMetric-Labs/ClearMetric-Core
+Project-URL: Source, https://github.com/ClearMetric-Labs/ClearMetric-Core
+Project-URL: Issues, https://github.com/ClearMetric-Labs/ClearMetric-Core/issues
+Keywords: catalog,lineage,metadata,graph,sql,dbt,powerbi,analytics
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.10.0
+Requires-Dist: sqlglot>=25.0.0
+Requires-Dist: pbi-parsers>=0.9.5
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: PyYAML>=6.0; extra == "dev"
+Provides-Extra: release
+Requires-Dist: build>=1.2.2; extra == "release"
+Requires-Dist: twine>=5.1.1; extra == "release"
+
+# clearmetric-core
+
+One PyPI package. All ClearMetric Core modules (`clearmetric.core`, `.lineage`,
+`.query`, `.powerbi`, `.cli`) ship inside this distribution.
+
+## Install
+
+```bash
+python -m pip install clearmetric-core
+```
+
+## CLI
+
+```bash
+cm compile ./manifest.json --dialect postgres
+cm impact orders.amount ./manifest.json --dialect postgres --upstream
+```
+
+If `cm` is occupied on your PATH:
+
+```bash
+python -m clearmetric.cli compile ./manifest.json --dialect postgres
+```
+
+## Modules
+
+| Module | Purpose |
+|--------|---------|
+| `clearmetric.core` | Artifact schema, canonical IDs, merge, cross-graph interop |
+| `clearmetric.lineage` | Project-level SQL lineage from dbt manifests and SQL folders |
+| `clearmetric.query` | Single-statement SQL structure mapping |
+| `clearmetric.powerbi` | PBIP file lineage and warehouse merge |
+| `clearmetric.cli` | `cm` command router |
+
+## Imports
+
+```python
+from clearmetric.core import (
+    CatalogArtifact,
+    Edge,
+    Evidence,
+    Node,
+    Warning,
+    load_table_alias_map,
+    merge,
+    resolve_table_match,
+)
+from clearmetric.lineage import build_catalog_artifact, trace_upstream
+```
+
+For local development:
+
+```bash
+python -m pip install -e ".[dev,release]"
+```
+
+## Contract
+
+The source of truth for the shared artifact contract is
+[`docs/contract.md`](docs/contract.md).

clearmetric_core-0.2.0/README.md

@@ -0,0 +1,60 @@
+# clearmetric-core
+
+One PyPI package. All ClearMetric Core modules (`clearmetric.core`, `.lineage`,
+`.query`, `.powerbi`, `.cli`) ship inside this distribution.
+
+## Install
+
+```bash
+python -m pip install clearmetric-core
+```
+
+## CLI
+
+```bash
+cm compile ./manifest.json --dialect postgres
+cm impact orders.amount ./manifest.json --dialect postgres --upstream
+```
+
+If `cm` is occupied on your PATH:
+
+```bash
+python -m clearmetric.cli compile ./manifest.json --dialect postgres
+```
+
+## Modules
+
+| Module | Purpose |
+|--------|---------|
+| `clearmetric.core` | Artifact schema, canonical IDs, merge, cross-graph interop |
+| `clearmetric.lineage` | Project-level SQL lineage from dbt manifests and SQL folders |
+| `clearmetric.query` | Single-statement SQL structure mapping |
+| `clearmetric.powerbi` | PBIP file lineage and warehouse merge |
+| `clearmetric.cli` | `cm` command router |
+
+## Imports
+
+```python
+from clearmetric.core import (
+    CatalogArtifact,
+    Edge,
+    Evidence,
+    Node,
+    Warning,
+    load_table_alias_map,
+    merge,
+    resolve_table_match,
+)
+from clearmetric.lineage import build_catalog_artifact, trace_upstream
+```
+
+For local development:
+
+```bash
+python -m pip install -e ".[dev,release]"
+```
+
+## Contract
+
+The source of truth for the shared artifact contract is
+[`docs/contract.md`](docs/contract.md).

clearmetric_core-0.2.0/pyproject.toml

@@ -0,0 +1,69 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "clearmetric-core"
+dynamic = ["version"]
+description = "ClearMetric Core — local compiler, graph engine, and CLI foundation."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+authors = [
+    {name = "ClearMetric Labs"},
+]
+keywords = [
+    "catalog",
+    "lineage",
+    "metadata",
+    "graph",
+    "sql",
+    "dbt",
+    "powerbi",
+    "analytics",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "pydantic>=2.10.0",
+    "sqlglot>=25.0.0",
+    "pbi-parsers>=0.9.5",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "PyYAML>=6.0",
+]
+release = [
+    "build>=1.2.2",
+    "twine>=5.1.1",
+]
+
+[project.scripts]
+cm = "clearmetric.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/ClearMetric-Labs/ClearMetric-Core"
+Source = "https://github.com/ClearMetric-Labs/ClearMetric-Core"
+Issues = "https://github.com/ClearMetric-Labs/ClearMetric-Core/issues"
+
+[tool.setuptools.dynamic]
+version = {attr = "clearmetric.core._version.__version__"}
+
+[tool.setuptools.package-dir]
+"" = "src"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["clearmetric.*"]
+namespaces = true

clearmetric_core-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

clearmetric_core-0.2.0/src/clearmetric/cli/__init__.py

@@ -0,0 +1,157 @@
+"""ClearMetric Core CLI — ``cm`` command router."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from clearmetric.core import __version__, render_json
+from clearmetric.lineage import (
+    build_catalog_artifact,
+    build_lineage_map,
+    trace_downstream,
+    trace_upstream,
+)
+from clearmetric.lineage.errors import LineageError
+from clearmetric.lineage.render.mermaid import render_traversal_mermaid
+from clearmetric.lineage.render.text import render_text, render_traversal_tree
+
+
+def _build_root_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="cm",
+        description="ClearMetric Core — local compiler, graph engine, and CLI.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"cm {__version__} (ClearMetric Core)",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    compile_parser = subparsers.add_parser(
+        "compile",
+        help="Compile project input into a catalog graph artifact (JSON).",
+    )
+    compile_parser.add_argument(
+        "project_input",
+        help="Path to a dbt manifest.json file or a folder of UTF-8 .sql files.",
+    )
+    compile_parser.add_argument(
+        "--dialect",
+        required=True,
+        help="sqlglot dialect name, for example postgres, snowflake, tsql, or bigquery.",
+    )
+    compile_parser.add_argument(
+        "--format",
+        choices=("json", "text"),
+        default="json",
+        help="Output format (default: json).",
+    )
+
+    impact_parser = subparsers.add_parser(
+        "impact",
+        help="Trace upstream or downstream column lineage for one selection.",
+    )
+    impact_parser.add_argument(
+        "selection",
+        help="Dataset column selection, for example orders.amount.",
+    )
+    impact_parser.add_argument(
+        "project_input",
+        help="Path to a dbt manifest.json file or a folder of UTF-8 .sql files.",
+    )
+    impact_parser.add_argument(
+        "--dialect",
+        required=True,
+        help="sqlglot dialect name, for example postgres, snowflake, tsql, or bigquery.",
+    )
+    traversal = impact_parser.add_mutually_exclusive_group(required=True)
+    traversal.add_argument(
+        "--upstream",
+        action="store_true",
+        help="Trace upstream lineage for the selection.",
+    )
+    traversal.add_argument(
+        "--downstream",
+        action="store_true",
+        help="Trace downstream impact for the selection.",
+    )
+    impact_parser.add_argument(
+        "--format",
+        choices=("text", "json", "mermaid"),
+        default="text",
+        help="Output format (default: text).",
+    )
+    return parser
+
+
+def _run_compile(args: argparse.Namespace) -> int:
+    if args.format == "json":
+        artifact = build_catalog_artifact(args.project_input, dialect=args.dialect)
+        print(json.dumps(render_json(artifact), indent=2, sort_keys=False))
+    else:
+        lineage_map = build_lineage_map(args.project_input, dialect=args.dialect)
+        print(render_text(lineage_map))
+    return 0
+
+
+def _run_impact(args: argparse.Namespace) -> int:
+    direction = "upstream" if args.upstream else "downstream"
+    if direction == "upstream":
+        result = trace_upstream(
+            args.project_input,
+            dialect=args.dialect,
+            selection=args.selection,
+        )
+    else:
+        result = trace_downstream(
+            args.project_input,
+            dialect=args.dialect,
+            selection=args.selection,
+        )
+
+    if args.format == "json":
+        print(json.dumps(result.model_dump(mode="json"), indent=2, sort_keys=False))
+        return 0
+
+    artifact = build_catalog_artifact(args.project_input, dialect=args.dialect)
+    if args.format == "mermaid":
+        print(
+            render_traversal_mermaid(
+                result.selection_id,
+                artifact,
+                direction=direction,
+            )
+        )
+        return 0
+
+    print(
+        render_traversal_tree(
+            result,
+            artifact,
+            direction=direction,
+        )
+    )
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_root_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        if args.command == "compile":
+            return _run_compile(args)
+        if args.command == "impact":
+            return _run_impact(args)
+    except LineageError as exc:
+        print(f"cm error: {exc}", file=sys.stderr)
+        return 1
+
+    print(f"cm: unknown command {args.command!r}", file=sys.stderr)
+    return 1
+
+
+__all__ = ["main"]

clearmetric_core-0.2.0/src/clearmetric/cli/__main__.py

@@ -0,0 +1,8 @@
+"""``python -m clearmetric.cli`` entry."""
+
+from __future__ import annotations
+
+from . import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

clearmetric_core-0.2.0/src/clearmetric/core/__init__.py

@@ -0,0 +1,78 @@
+"""Public package surface for clearmetric-core."""
+
+from __future__ import annotations
+
+from ._version import __version__
+from .aliases import load_table_alias_map
+from .errors import (
+    AliasMapError,
+    CanonicalIdError,
+    ClearMetricError,
+    MergeConflictError,
+)
+from .ids import (
+    asset_id,
+    column_id,
+    cte_id,
+    leaf_name,
+    measure_id,
+    model_id,
+    normalize_identifier,
+    normalize_identifier_part,
+    normalize_identifier_parts,
+    page_id,
+    report_id,
+    schema_name,
+    split_qualified_identifier,
+    table_id,
+    visual_id,
+)
+from .interop import (
+    AliasMap,
+    apply_alias_map,
+    normalize_fqn_for_matching,
+    resolve_table_match,
+    warehouse_table_fqn_candidates,
+    warehouse_table_fqn_candidates_from_name,
+)
+from .merge import merge
+from .models import CatalogArtifact, Edge, Evidence, MatchStatus, Node, Warning
+from .serialize import render_json
+
+__all__ = [
+    "__version__",
+    "AliasMap",
+    "AliasMapError",
+    "MatchStatus",
+    "apply_alias_map",
+    "asset_id",
+    "CatalogArtifact",
+    "ClearMetricError",
+    "CanonicalIdError",
+    "column_id",
+    "cte_id",
+    "Edge",
+    "Evidence",
+    "leaf_name",
+    "load_table_alias_map",
+    "measure_id",
+    "merge",
+    "normalize_fqn_for_matching",
+    "MergeConflictError",
+    "model_id",
+    "Node",
+    "page_id",
+    "normalize_identifier",
+    "normalize_identifier_part",
+    "normalize_identifier_parts",
+    "render_json",
+    "report_id",
+    "resolve_table_match",
+    "schema_name",
+    "split_qualified_identifier",
+    "table_id",
+    "visual_id",
+    "warehouse_table_fqn_candidates",
+    "warehouse_table_fqn_candidates_from_name",
+    "Warning",
+]

clearmetric_core-0.2.0/src/clearmetric/core/_version.py

@@ -0,0 +1,3 @@
+"""Package version."""
+
+__version__ = "0.2.0"

clearmetric_core-0.2.0/src/clearmetric/core/aliases.py

@@ -0,0 +1,91 @@
+"""Load versioned table alias files for cross-graph matching."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from .errors import AliasMapError
+from .interop import AliasMap, normalize_fqn_for_matching
+
+_SUPPORTED_VERSION = "1"
+
+
+def load_table_alias_map(path: str | Path) -> AliasMap:
+    """
+    Load a version-1 alias file into an ``AliasMap``.
+
+    Expected format::
+
+        version: 1
+        table_aliases:
+          salesmart.dbo.orders: orders
+    """
+    file_path = Path(path).expanduser().resolve()
+    if not file_path.is_file():
+        raise AliasMapError(f"Alias file does not exist: {file_path}")
+
+    version: str | None = None
+    aliases: AliasMap = {}
+    in_table_aliases = False
+
+    for line_number, raw_line in enumerate(
+        file_path.read_text(encoding="utf-8").splitlines(), start=1
+    ):
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+
+        if line.startswith("version:"):
+            version = line.split(":", 1)[1].strip()
+            if version != _SUPPORTED_VERSION:
+                raise AliasMapError(
+                    f"Unsupported alias file version {version!r} at {file_path}:{line_number}; "
+                    f"expected {_SUPPORTED_VERSION!r}."
+                )
+            continue
+
+        if line == "table_aliases:":
+            in_table_aliases = True
+            continue
+
+        if not in_table_aliases:
+            raise AliasMapError(
+                f"Unexpected content at {file_path}:{line_number}; "
+                "expected 'version:' and 'table_aliases:' sections."
+            )
+
+        if ":" not in line:
+            raise AliasMapError(
+                f"Invalid alias entry at {file_path}:{line_number}; "
+                "expected 'source: target' form."
+            )
+
+        source, target = line.split(":", 1)
+        source = source.strip()
+        target = target.strip()
+        if not source or not target:
+            raise AliasMapError(
+                f"Invalid alias entry at {file_path}:{line_number}; "
+                "source and target must be non-empty."
+            )
+
+        normalized_source = normalize_fqn_for_matching(source)
+        normalized_target = normalize_fqn_for_matching(target)
+        if (
+            normalized_source in aliases
+            and aliases[normalized_source] != normalized_target
+        ):
+            raise AliasMapError(
+                f"Duplicate alias key {normalized_source!r} with conflicting targets "
+                f"at {file_path}:{line_number}."
+            )
+        aliases[normalized_source] = normalized_target
+
+    if version is None:
+        raise AliasMapError(f"Missing 'version:' in alias file: {file_path}")
+    if not in_table_aliases:
+        raise AliasMapError(
+            f"Missing 'table_aliases:' section in alias file: {file_path}"
+        )
+
+    return aliases

clearmetric_core-0.2.0/src/clearmetric/core/errors.py

@@ -0,0 +1,19 @@
+"""Shared errors for clearmetric-core."""
+
+from __future__ import annotations
+
+
+class ClearMetricError(Exception):
+    """Base class for clearmetric-core failures."""
+
+
+class CanonicalIdError(ClearMetricError):
+    """Raised when an identifier cannot be normalized into a canonical ID."""
+
+
+class MergeConflictError(ClearMetricError):
+    """Raised when artifacts cannot be merged without losing information."""
+
+
+class AliasMapError(ClearMetricError):
+    """Raised when a table alias file is invalid or unsupported."""