datajunction 0.0.182__tar.gz → 0.0.184__tar.gz

{datajunction-0.0.182 → datajunction-0.0.184}/Makefile

@@ -1,5 +1,5 @@
 setup:
-	uv sync --extra mcp --group test --extra pandas
+	uv sync --extra mcp --group test
 check:
 	uv run pre-commit run --all-files
 

{datajunction-0.0.182 → datajunction-0.0.184}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datajunction
-Version: 0.0.182
+Version: 0.0.184
 Summary: DataJunction client library for connecting to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>
@@ -14,6 +14,7 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <4.0,>=3.10
 Requires-Dist: alive-progress>=3.1.2
 Requires-Dist: httpx>=0.27.0
+Requires-Dist: pandas>=2.0.2
 Requires-Dist: pytest-xdist>=3.5.0
 Requires-Dist: pyyaml>=6.0.1
 Requires-Dist: requests<3.0.0,>=2.28.2
@@ -22,8 +23,6 @@ Provides-Extra: mcp
 Requires-Dist: mcp>=1.0.0; extra == 'mcp'
 Requires-Dist: pydantic-settings>=2.10.1; extra == 'mcp'
 Requires-Dist: pydantic>=2.0; extra == 'mcp'
-Provides-Extra: pandas
-Requires-Dist: pandas>=2.0.2; extra == 'pandas'
 Description-Content-Type: text/markdown
 
 # DataJunction Python Client

{datajunction-0.0.182 → datajunction-0.0.184}/datajunction/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.182"
+__version__ = "0.0.184"

{datajunction-0.0.182 → datajunction-0.0.184}/datajunction/_internal.py

@@ -4,21 +4,11 @@
 import logging
 import os
 import platform
-import warnings
 from dataclasses import asdict, dataclass
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, TypedDict, Union
 from urllib.parse import urljoin
 
-try:
-    import pandas as pd
-except ImportError:  # pragma: no cover
-    warnings.warn(
-        (
-            "Optional dependency `pandas` not found, data retrieval"
-            "disabled. You can install pandas by running `pip install pandas`."
-        ),
-        ImportWarning,
-    )
+import pandas as pd
 import requests
 from requests.adapters import CaseInsensitiveDict, HTTPAdapter
 
@@ -271,7 +261,7 @@ class DJClient:
     @staticmethod
     def process_results(results) -> "pd.DataFrame":
         """
-        Return a pandas dataframe of the results if pandas is installed
+        Return a pandas dataframe of the results
         """
         if "results" in results and results["results"]:
             columns = results["results"][0]["columns"]

{datajunction-0.0.182 → datajunction-0.0.184}/datajunction/cli.py

@@ -5,7 +5,7 @@ import json
 import logging
 import os
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 from rich import box
 from rich.console import Console, Group
@@ -601,10 +601,15 @@ class DJCLI:
                 )
                 return
 
-            # Handle error responses
-            if isinstance(result, dict) and "message" in result:
-                console.print(f"[bold red]ERROR:[/bold red] {result['message']}")
-                return
+            # Handle error responses and raw dict responses.
+            # data() / node_data() should return a DataFrame via process_results,
+            # but if the raw query response dict is returned instead, try to
+            # process it here rather than crashing on result.columns.
+            if isinstance(result, dict):
+                if "message" in result:
+                    console.print(f"[bold red]ERROR:[/bold red] {result['message']}")
+                    return
+                result = self.builder_client.process_results(result)
 
             # result should be a DataFrame (already limited by API)
             if format == "json":
@@ -958,6 +963,27 @@ class DJCLI:
             default="GITHUB_TOKEN",
             help="Name of the env var holding the GitHub token (default: GITHUB_TOKEN)",
         )
+        codeowners_parser.add_argument(
+            "--default-owner",
+            type=str,
+            default=None,
+            help=(
+                "Owner for a leading `* <owner>` rule. Unmatched files and any "
+                "--exclude'd directories fall through to it (e.g. '@org/team')."
+            ),
+        )
+        codeowners_parser.add_argument(
+            "--exclude",
+            action="append",
+            default=None,
+            dest="exclude_dirs",
+            metavar="DIR",
+            help=(
+                "Directory (relative to <directory>) whose nodes get no per-file "
+                "owners — they fall through to --default-owner. Repeatable. Use for "
+                "machine-generated trees like 'nodes/generated'."
+            ),
+        )
 
         # `dj pull <namespace> <directory>`
         pull_parser = subparsers.add_parser(
@@ -1489,6 +1515,8 @@ class DJCLI:
                 output=args.output,
                 github_api_url=args.github_api_url,
                 github_token_env=args.github_token_env,
+                default_owner=args.default_owner,
+                exclude_dirs=args.exclude_dirs,
             )
             console = Console()
             console.print(
@@ -1620,52 +1648,115 @@ class DJCLI:
                 output_dir.mkdir(parents=True, exist_ok=True)
 
                 console.print(
-                    "[bold]📚 Installing DJ skill[/bold]\n",
+                    "[bold]📚 Installing DJ skills[/bold]\n",
                 )
 
-                # Load bundled skill from package
-                dir_name = "datajunction"
-                skill_dir = output_dir / dir_name
-                skill_file = skill_dir / "SKILL.md"
-
-                console.print(f"Installing [cyan]{dir_name}[/cyan]...")
-
-                # Find bundled skill file
                 from datajunction import __version__ as dj_version
                 from importlib.resources import files
 
-                try:
-                    # Read bundled skill markdown using importlib.resources (Python 3.9+)
-                    skill_file_path = files("datajunction").joinpath(
-                        "skills/datajunction.md",
-                    )
-                    bundled_skill = skill_file_path.read_text(encoding="utf-8")
+                # All bundled DJ skills. ``datajunction`` is the core concepts
+                # skill (always loaded); the others are audience-specific
+                # extensions that compose on top of it.
+                bundled_skills: list[dict[str, Any]] = [
+                    {
+                        "name": "datajunction",
+                        "filename": "datajunction.md",
+                        "description": "Core DataJunction concepts and vocabulary",
+                        "keywords": [
+                            "DataJunction",
+                            "DJ",
+                            "semantic layer",
+                            "dimension link",
+                            "star schema",
+                            "node types",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-query",
+                        "filename": "datajunction-query.md",
+                        "description": "Querying DJ metrics via MCP tools and APIs",
+                        "keywords": [
+                            "query metric",
+                            "generate SQL",
+                            "get metric data",
+                            "search_nodes",
+                            "build_metric_sql",
+                            "common dimensions",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-semantic-model",
+                        "filename": "datajunction-semantic-model.md",
+                        "description": "DJ semantic modeling: query-to-nodes decomposition, ratio decomposition, naming, ownership",
+                        "keywords": [
+                            "semantic modeling",
+                            "decompose query",
+                            "ratio metric",
+                            "derived metric",
+                            "base metric",
+                            "metric naming",
+                            "namespace organization",
+                            "node ownership",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-repo",
+                        "filename": "datajunction-repo.md",
+                        "description": "Authoring DJ nodes via YAML in a git-backed repository",
+                        "keywords": [
+                            "YAML nodes",
+                            "repo-backed namespace",
+                            "feature branch",
+                            "git workflow",
+                            "cube YAML",
+                            "metric YAML",
+                            "temporal partition",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-api",
+                        "filename": "datajunction-api.md",
+                        "description": "Direct REST API authoring of DJ nodes for exploration / prototyping",
+                        "keywords": [
+                            "DJ API",
+                            "REST API",
+                            "curl",
+                            "POST nodes",
+                            "API authoring",
+                            "prototyping",
+                        ],
+                    },
+                ]
+
+                missing: list[str] = []
+                for skill in bundled_skills:
+                    dir_name = skill["name"]
+                    skill_dir = output_dir / dir_name
+                    skill_file = skill_dir / "SKILL.md"
+
+                    console.print(f"Installing [cyan]{dir_name}[/cyan]...")
+
+                    try:
+                        skill_file_path = files("datajunction").joinpath(
+                            f"skills/{skill['filename']}",
+                        )
+                        bundled_skill = skill_file_path.read_text(encoding="utf-8")
+                    except FileNotFoundError:  # pragma: no cover
+                        missing.append(skill["filename"])
+                        continue
 
-                    # Create directory
                     skill_dir.mkdir(parents=True, exist_ok=True)
 
-                    # Write SKILL.md
                     with open(skill_file, "w") as f:
                         f.write(bundled_skill)
 
-                    # Write metadata
                     metadata_file = skill_dir / "metadata.json"
                     with open(metadata_file, "w") as f:
                         metadata = {
-                            "name": "datajunction",
+                            "name": skill["name"],
                             "version": dj_version,
-                            "description": "Comprehensive DataJunction semantic layer guide",
-                            "keywords": [
-                                "DataJunction",
-                                "DJ",
-                                "semantic layer",
-                                "dimension link",
-                                "metric",
-                                "SQL generation",
-                                "YAML nodes",
-                                "git workflow",
-                                "repo-backed namespace",
-                            ],
+                            "description": skill["description"],
+                            "keywords": skill["keywords"],
                             "metadata": {
                                 "source": "bundled",
                                 "dj_version": dj_version,
@@ -1681,13 +1772,14 @@ class DJCLI:
                         f"  [dim]└─ metadata.json (v{dj_version})[/dim]\n",
                     )
 
+                if missing:  # pragma: no cover
                     console.print(
-                        f"[bold green]✓ Skill installed to {output_dir}[/bold green]\n",
+                        f"[red]✗ Bundled skills not found: {', '.join(missing)}. "
+                        f"Please ensure datajunction is properly installed.[/red]",
                     )
-
-                except FileNotFoundError:  # pragma: no cover
+                else:
                     console.print(
-                        "[red]✗ Bundled skill not found. Please ensure datajunction is properly installed.[/red]",
+                        f"[bold green]✓ Skills installed to {output_dir}[/bold green]\n",
                     )
 
             # Install subagent if requested
@@ -1704,9 +1796,13 @@ name: dj
 description: >
   DataJunction semantic layer expert. Use proactively for any DataJunction
   or DJ work — querying metrics, exploring nodes and dimensions, building
-  SQL, understanding lineage, and semantic layer design.
+  SQL, understanding lineage, authoring metrics, and semantic layer design.
 skills:
   - datajunction
+  - datajunction-query
+  - datajunction-semantic-model
+  - datajunction-repo
+  - datajunction-api
 model: inherit
 ---
 """

{datajunction-0.0.182 → datajunction-0.0.184}/datajunction/deployment.py

@@ -304,6 +304,8 @@ class DeploymentService:
         output: str | Path = ".github/CODEOWNERS",
         github_api_url: str | None = None,
         github_token_env: str = "GITHUB_TOKEN",
+        default_owner: str | None = None,
+        exclude_dirs: list[str] | None = None,
     ) -> int:
         """
         Generate a CODEOWNERS file from the owners fields in DJ node YAML files.
@@ -317,9 +319,24 @@ class DeploymentService:
         email addresses in owners fields are resolved to GitHub usernames via the
         search API.  Unresolvable emails are emitted as-is with a warning comment.
 
-        Returns the number of entries written.
+        ``default_owner``, when set, emits a leading ``* <default_owner>`` rule so
+        unmatched files (and any excluded directories) fall through to it. Because
+        CODEOWNERS is last-match-wins, the per-file rules below it still take
+        precedence for the files they name.
+
+        ``exclude_dirs`` lists directories (relative to base_dir) whose nodes are
+        NOT given per-file owners — they fall through to ``default_owner`` instead.
+        This is for machine-generated trees (e.g. ``nodes/generated``) that have no
+        individual human owner and should be team-owned as a block.
+
+        Returns the number of per-file entries written (excludes the default rule).
         """
         base = Path(base_dir).resolve()
+        excluded_dirs = [(base / d).resolve() for d in (exclude_dirs or [])]
+
+        def _is_excluded(path: Path) -> bool:
+            return any(d == path or d in path.parents for d in excluded_dirs)
+
         _token = os.getenv(github_token_env)
         # Only resolve if both API URL and token are available
         lookup: tuple[str, str] | None = (
@@ -353,6 +370,8 @@ class DeploymentService:
         for path in sorted(base.rglob("*.yaml")):
             if path.name == "dj.yaml":
                 continue
+            if _is_excluded(path):
+                continue
             try:
                 node = DeploymentService.read_yaml_file(path)
             except Exception:  # skip unreadable / non-dict files
@@ -376,7 +395,12 @@ class DeploymentService:
             )
             for w in warnings:
                 lines.append(f"#   {w}")
-        lines += ["", *entries, ""]
+        lines.append("")
+        # Default rule first so per-file rules below win (last-match-wins).
+        if default_owner:
+            lines.append(f"* {default_owner}")
+            lines.append("")
+        lines += [*entries, ""]
         content = "\n".join(lines)
 
         output_path = Path(output)

datajunction-0.0.184/datajunction/skills/datajunction-api.md

@@ -0,0 +1,232 @@
+---
+name: datajunction-api
+description: |
+  Activate this skill when authoring DataJunction (DJ) nodes via the REST
+  API directly (curl, HTTP clients) — typically for exploration, ad-hoc
+  prototyping, or namespaces that aren't repo-backed. For modeling
+  decisions and the decomposition workflow, invoke `datajunction-semantic-model`.
+  For repo-backed YAML authoring (the production path), invoke
+  `datajunction-repo`. For concepts, invoke `datajunction`.
+  Keywords:
+  - DJ API, REST API, curl
+  - POST nodes/metric, POST nodes/dimension
+  - create metric, create dimension, create cube
+  - API approach, direct API changes
+  - prototyping, exploration
+user-invocable: false
+---
+
+# DataJunction Direct API Authoring
+
+Direct REST API authoring for DJ nodes. Use this skill for quick exploration, prototyping, or working in namespaces that don't use the repo-backed workflow.
+
+For the modeling work upstream of any authoring (decomposition, naming, ratio decomposition, etc.), see `datajunction-semantic-model`. For the production-path equivalent of these patterns in YAML, see `datajunction-repo`.
+
+## When to Use the API Approach
+
+**✅ MUST use repo workflow instead** (`datajunction-repo`) for:
+- Namespaces configured as repo-backed and read-only (`git_only: true`) — direct API changes are rejected
+
+**✅ Should use repo workflow** for:
+- Production changes (review required)
+- Multi-node changes (related metrics/dimensions)
+- Team environments (multiple contributors)
+- Audit-trail requirements
+- Complex refactoring
+
+**✅ API workflow is appropriate** for:
+- Quick exploration and prototyping
+- Ad-hoc analysis
+- Single-user, non-production namespaces
+- Temporary metrics
+- Non-production experiments
+- Only when the target namespace is NOT read-only repo-backed
+
+---
+
+## Checking if a Namespace Is Repo-Backed
+
+Before authoring via API, verify the target namespace allows it.
+
+**Best approach — `get_node_details` MCP tool** (`datajunction-query` skill):
+
+```
+get_node_details(name="finance.total_revenue")
+```
+
+The response will include git repository information:
+```
+Git Repository:
+  Repo: owner/dj-finance
+  Branch: main
+  Default Branch: main
+  → This namespace is repo-backed (use git workflow for changes)
+```
+
+**Alternative — REST API** (shows read-only status):
+```bash
+curl -b ~/.dj/cookies.txt -X GET $DJ_URL/namespaces/finance/git
+
+# Response:
+{
+  "github_repo_path": "owner/dj-finance",
+  "git_branch": "main",
+  "default_branch": "main",
+  "git_path": "nodes/",
+  "git_only": true    ← If true, namespace is read-only (API changes blocked)
+}
+```
+
+**Decision tree:**
+- **If git info is present AND `git_only: true`**: MUST use repo workflow (API changes will fail)
+- **If git info is present AND `git_only: false`**: Can use either workflow
+- **If git info is null**: Use API workflow (direct POST/PATCH)
+
+---
+
+## Creating a Metric
+
+### Metric Structure
+
+```sql
+SELECT <aggregation_expression> AS <metric_name>
+FROM <single_node>
+```
+
+Metrics select a **single expression** from a **single source, transform, or dimension node**. They cannot contain WHERE clauses — use CASE WHEN instead. See `datajunction-semantic-model` for the modeling rationale.
+
+### Metric Metadata Fields
+
+**Required:**
+- `name` — Fully qualified metric name (e.g., `finance.total_revenue`)
+- `query` — SQL aggregation expression
+
+**Recommended:**
+- `description` — Human-readable description
+- `metric_metadata.direction` — `higher_is_better` / `lower_is_better` / `neutral`
+- `metric_metadata.unit` — `dollar` / `unitless` (**⚠️ NOT `count`** — server rejects)
+- `mode` — `draft` / `published`
+- `required_dimensions` — Dimensions required for this metric to make sense
+- `owners` — List of email addresses (prefer team emails)
+
+### Examples
+
+**COUNT**:
+```bash
+curl -b ~/.dj/cookies.txt -X POST $DJ_URL/nodes/metric/ \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "finance.num_transactions",
+    "description": "Total number of transactions",
+    "query": "SELECT COUNT(transaction_id) AS num_transactions FROM finance.transactions",
+    "owners": ["data-platform-team@company.com"],
+    "mode": "published"
+  }'
+```
+
+**SUM**:
+```bash
+curl -b ~/.dj/cookies.txt -X POST $DJ_URL/nodes/metric/ \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "finance.total_revenue",
+    "description": "Total revenue from all transactions",
+    "query": "SELECT SUM(amount_usd) AS total_revenue FROM finance.transactions",
+    "metric_metadata": {
+      "direction": "higher_is_better",
+      "unit": "dollar"
+    },
+    "owners": ["finance-data-team@company.com"],
+    "mode": "published"
+  }'
+```
+
+**Conditional aggregation** (CASE WHEN, not WHERE):
+```bash
+curl -b ~/.dj/cookies.txt -X POST $DJ_URL/nodes/metric/ \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "finance.completed_revenue",
+    "description": "Revenue from completed non-refund transactions",
+    "query": "
+      SELECT SUM(
+        CASE
+          WHEN status = '\''completed'\'' AND refund_flag = false
+          THEN amount_usd
+          ELSE 0
+        END
+      ) AS completed_revenue
+      FROM finance.transactions
+    ",
+    "metric_metadata": {
+      "direction": "higher_is_better",
+      "unit": "dollar"
+    },
+    "owners": ["finance-data-team@company.com"],
+    "mode": "published"
+  }'
+```
+
+**Ratio over base metrics** (decompose first, then derive — see `datajunction-semantic-model`):
+```bash
+# Step 1: create the base metrics (one curl each)
+curl -X POST $DJ_URL/nodes/metric/ -d '{
+  "name": "finance.clicks",
+  "query": "SELECT COUNT_IF(event = '\''click'\'') FROM finance.events",
+  "owners": ["marketing@company.com"],
+  "mode": "published"
+}'
+
+curl -X POST $DJ_URL/nodes/metric/ -d '{
+  "name": "finance.impressions",
+  "query": "SELECT COUNT_IF(event = '\''impression'\'') FROM finance.events",
+  "owners": ["marketing@company.com"],
+  "mode": "published"
+}'
+
+# Step 2: derived ratio metric referencing the base metrics
+curl -X POST $DJ_URL/nodes/metric/ -d '{
+  "name": "finance.conversion_rate",
+  "description": "Click-through rate as percentage",
+  "query": "SELECT finance.clicks * 100.0 / NULLIF(finance.impressions, 0)",
+  "metric_metadata": {
+    "direction": "higher_is_better",
+    "unit": "unitless"
+  },
+  "owners": ["marketing@company.com"],
+  "mode": "published"
+}'
+```
+
+DJ automatically handles divide-by-zero, but `NULLIF()` is extra safety.
+
+---
+
+## Creating Other Node Types via API
+
+Same pattern as metrics — POST JSON to the appropriate endpoint:
+
+- `POST /nodes/source/` — source nodes (catalog/schema/table refs)
+- `POST /nodes/dimension/` — dimension nodes
+- `POST /nodes/transform/` — transform nodes
+- `POST /nodes/cube/` — cubes (metric + dimension combinations)
+
+For the YAML-equivalent shapes of each, see `datajunction-repo` — the field set is the same, just expressed in JSON instead of YAML.
+
+---
+
+## Updating and Deleting Nodes
+
+**Update** (PATCH):
+```bash
+curl -b ~/.dj/cookies.txt -X PATCH $DJ_URL/nodes/finance.total_revenue/ \
+  -H 'Content-Type: application/json' \
+  -d '{"description": "Updated description"}'
+```
+
+**Deactivate** (soft delete):
+```bash
+curl -b ~/.dj/cookies.txt -X DELETE $DJ_URL/nodes/finance.total_revenue/
+```
+
+Deactivated nodes can be revived. For hard-delete (irreversible), use the `dj` CLI: `dj delete-node finance.total_revenue --hard`.