datajunction 0.0.181__tar.gz → 0.0.183__tar.gz

{datajunction-0.0.181 → datajunction-0.0.183}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datajunction
-Version: 0.0.181
+Version: 0.0.183
 Summary: DataJunction client library for connecting to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

{datajunction-0.0.181 → datajunction-0.0.183}/datajunction/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.181"
+__version__ = "0.0.183"

{datajunction-0.0.181 → datajunction-0.0.183}/datajunction/cli.py

@@ -5,7 +5,7 @@ import json
 import logging
 import os
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 from rich import box
 from rich.console import Console, Group
@@ -1620,52 +1620,115 @@ class DJCLI:
                 output_dir.mkdir(parents=True, exist_ok=True)
 
                 console.print(
-                    "[bold]📚 Installing DJ skill[/bold]\n",
+                    "[bold]📚 Installing DJ skills[/bold]\n",
                 )
 
-                # Load bundled skill from package
-                dir_name = "datajunction"
-                skill_dir = output_dir / dir_name
-                skill_file = skill_dir / "SKILL.md"
-
-                console.print(f"Installing [cyan]{dir_name}[/cyan]...")
-
-                # Find bundled skill file
                 from datajunction import __version__ as dj_version
                 from importlib.resources import files
 
-                try:
-                    # Read bundled skill markdown using importlib.resources (Python 3.9+)
-                    skill_file_path = files("datajunction").joinpath(
-                        "skills/datajunction.md",
-                    )
-                    bundled_skill = skill_file_path.read_text(encoding="utf-8")
+                # All bundled DJ skills. ``datajunction`` is the core concepts
+                # skill (always loaded); the others are audience-specific
+                # extensions that compose on top of it.
+                bundled_skills: list[dict[str, Any]] = [
+                    {
+                        "name": "datajunction",
+                        "filename": "datajunction.md",
+                        "description": "Core DataJunction concepts and vocabulary",
+                        "keywords": [
+                            "DataJunction",
+                            "DJ",
+                            "semantic layer",
+                            "dimension link",
+                            "star schema",
+                            "node types",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-query",
+                        "filename": "datajunction-query.md",
+                        "description": "Querying DJ metrics via MCP tools and APIs",
+                        "keywords": [
+                            "query metric",
+                            "generate SQL",
+                            "get metric data",
+                            "search_nodes",
+                            "build_metric_sql",
+                            "common dimensions",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-semantic-model",
+                        "filename": "datajunction-semantic-model.md",
+                        "description": "DJ semantic modeling: query-to-nodes decomposition, ratio decomposition, naming, ownership",
+                        "keywords": [
+                            "semantic modeling",
+                            "decompose query",
+                            "ratio metric",
+                            "derived metric",
+                            "base metric",
+                            "metric naming",
+                            "namespace organization",
+                            "node ownership",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-repo",
+                        "filename": "datajunction-repo.md",
+                        "description": "Authoring DJ nodes via YAML in a git-backed repository",
+                        "keywords": [
+                            "YAML nodes",
+                            "repo-backed namespace",
+                            "feature branch",
+                            "git workflow",
+                            "cube YAML",
+                            "metric YAML",
+                            "temporal partition",
+                        ],
+                    },
+                    {
+                        "name": "datajunction-api",
+                        "filename": "datajunction-api.md",
+                        "description": "Direct REST API authoring of DJ nodes for exploration / prototyping",
+                        "keywords": [
+                            "DJ API",
+                            "REST API",
+                            "curl",
+                            "POST nodes",
+                            "API authoring",
+                            "prototyping",
+                        ],
+                    },
+                ]
+
+                missing: list[str] = []
+                for skill in bundled_skills:
+                    dir_name = skill["name"]
+                    skill_dir = output_dir / dir_name
+                    skill_file = skill_dir / "SKILL.md"
+
+                    console.print(f"Installing [cyan]{dir_name}[/cyan]...")
+
+                    try:
+                        skill_file_path = files("datajunction").joinpath(
+                            f"skills/{skill['filename']}",
+                        )
+                        bundled_skill = skill_file_path.read_text(encoding="utf-8")
+                    except FileNotFoundError:  # pragma: no cover
+                        missing.append(skill["filename"])
+                        continue
 
-                    # Create directory
                     skill_dir.mkdir(parents=True, exist_ok=True)
 
-                    # Write SKILL.md
                     with open(skill_file, "w") as f:
                         f.write(bundled_skill)
 
-                    # Write metadata
                     metadata_file = skill_dir / "metadata.json"
                     with open(metadata_file, "w") as f:
                         metadata = {
-                            "name": "datajunction",
+                            "name": skill["name"],
                             "version": dj_version,
-                            "description": "Comprehensive DataJunction semantic layer guide",
-                            "keywords": [
-                                "DataJunction",
-                                "DJ",
-                                "semantic layer",
-                                "dimension link",
-                                "metric",
-                                "SQL generation",
-                                "YAML nodes",
-                                "git workflow",
-                                "repo-backed namespace",
-                            ],
+                            "description": skill["description"],
+                            "keywords": skill["keywords"],
                             "metadata": {
                                 "source": "bundled",
                                 "dj_version": dj_version,
@@ -1681,13 +1744,14 @@ class DJCLI:
                         f"  [dim]└─ metadata.json (v{dj_version})[/dim]\n",
                     )
 
+                if missing:  # pragma: no cover
                     console.print(
-                        f"[bold green]✓ Skill installed to {output_dir}[/bold green]\n",
+                        f"[red]✗ Bundled skills not found: {', '.join(missing)}. "
+                        f"Please ensure datajunction is properly installed.[/red]",
                     )
-
-                except FileNotFoundError:  # pragma: no cover
+                else:
                     console.print(
-                        "[red]✗ Bundled skill not found. Please ensure datajunction is properly installed.[/red]",
+                        f"[bold green]✓ Skills installed to {output_dir}[/bold green]\n",
                     )
 
             # Install subagent if requested
@@ -1704,9 +1768,13 @@ name: dj
 description: >
   DataJunction semantic layer expert. Use proactively for any DataJunction
   or DJ work — querying metrics, exploring nodes and dimensions, building
-  SQL, understanding lineage, and semantic layer design.
+  SQL, understanding lineage, authoring metrics, and semantic layer design.
 skills:
   - datajunction
+  - datajunction-query
+  - datajunction-semantic-model
+  - datajunction-repo
+  - datajunction-api
 model: inherit
 ---
 """

datajunction-0.0.183/datajunction/skills/datajunction-api.md

@@ -0,0 +1,232 @@
+---
+name: datajunction-api
+description: |
+  Activate this skill when authoring DataJunction (DJ) nodes via the REST
+  API directly (curl, HTTP clients) — typically for exploration, ad-hoc
+  prototyping, or namespaces that aren't repo-backed. For modeling
+  decisions and the decomposition workflow, invoke `datajunction-semantic-model`.
+  For repo-backed YAML authoring (the production path), invoke
+  `datajunction-repo`. For concepts, invoke `datajunction`.
+  Keywords:
+  - DJ API, REST API, curl
+  - POST nodes/metric, POST nodes/dimension
+  - create metric, create dimension, create cube
+  - API approach, direct API changes
+  - prototyping, exploration
+user-invocable: false
+---
+
+# DataJunction Direct API Authoring
+
+Direct REST API authoring for DJ nodes. Use this skill for quick exploration, prototyping, or working in namespaces that don't use the repo-backed workflow.
+
+For the modeling work upstream of any authoring (decomposition, naming, ratio decomposition, etc.), see `datajunction-semantic-model`. For the production-path equivalent of these patterns in YAML, see `datajunction-repo`.
+
+## When to Use the API Approach
+
+**✅ MUST use repo workflow instead** (`datajunction-repo`) for:
+- Namespaces configured as repo-backed and read-only (`git_only: true`) — direct API changes are rejected
+
+**✅ Should use repo workflow** for:
+- Production changes (review required)
+- Multi-node changes (related metrics/dimensions)
+- Team environments (multiple contributors)
+- Audit-trail requirements
+- Complex refactoring
+
+**✅ API workflow is appropriate** for:
+- Quick exploration and prototyping
+- Ad-hoc analysis
+- Single-user, non-production namespaces
+- Temporary metrics
+- Non-production experiments
+- Only when the target namespace is NOT read-only repo-backed
+
+---
+
+## Checking if a Namespace Is Repo-Backed
+
+Before authoring via API, verify the target namespace allows it.
+
+**Best approach — `get_node_details` MCP tool** (`datajunction-query` skill):
+
+```
+get_node_details(name="finance.total_revenue")
+```
+
+The response will include git repository information:
+```
+Git Repository:
+  Repo: owner/dj-finance
+  Branch: main
+  Default Branch: main
+  → This namespace is repo-backed (use git workflow for changes)
+```
+
+**Alternative — REST API** (shows read-only status):
+```bash
+curl -b ~/.dj/cookies.txt -X GET $DJ_URL/namespaces/finance/git
+
+# Response:
+{
+  "github_repo_path": "owner/dj-finance",
+  "git_branch": "main",
+  "default_branch": "main",
+  "git_path": "nodes/",
+  "git_only": true    ← If true, namespace is read-only (API changes blocked)
+}
+```
+
+**Decision tree:**
+- **If git info is present AND `git_only: true`**: MUST use repo workflow (API changes will fail)
+- **If git info is present AND `git_only: false`**: Can use either workflow
+- **If git info is null**: Use API workflow (direct POST/PATCH)
+
+---
+
+## Creating a Metric
+
+### Metric Structure
+
+```sql
+SELECT <aggregation_expression> AS <metric_name>
+FROM <single_node>
+```
+
+Metrics select a **single expression** from a **single source, transform, or dimension node**. They cannot contain WHERE clauses — use CASE WHEN instead. See `datajunction-semantic-model` for the modeling rationale.
+
+### Metric Metadata Fields
+
+**Required:**
+- `name` — Fully qualified metric name (e.g., `finance.total_revenue`)
+- `query` — SQL aggregation expression
+
+**Recommended:**
+- `description` — Human-readable description
+- `metric_metadata.direction` — `higher_is_better` / `lower_is_better` / `neutral`
+- `metric_metadata.unit` — `dollar` / `unitless` (**⚠️ NOT `count`** — server rejects)
+- `mode` — `draft` / `published`
+- `required_dimensions` — Dimensions required for this metric to make sense
+- `owners` — List of email addresses (prefer team emails)
+
+### Examples
+
+**COUNT**:
+```bash
+curl -b ~/.dj/cookies.txt -X POST $DJ_URL/nodes/metric/ \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "finance.num_transactions",
+    "description": "Total number of transactions",
+    "query": "SELECT COUNT(transaction_id) AS num_transactions FROM finance.transactions",
+    "owners": ["data-platform-team@company.com"],
+    "mode": "published"
+  }'
+```
+
+**SUM**:
+```bash
+curl -b ~/.dj/cookies.txt -X POST $DJ_URL/nodes/metric/ \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "finance.total_revenue",
+    "description": "Total revenue from all transactions",
+    "query": "SELECT SUM(amount_usd) AS total_revenue FROM finance.transactions",
+    "metric_metadata": {
+      "direction": "higher_is_better",
+      "unit": "dollar"
+    },
+    "owners": ["finance-data-team@company.com"],
+    "mode": "published"
+  }'
+```
+
+**Conditional aggregation** (CASE WHEN, not WHERE):
+```bash
+curl -b ~/.dj/cookies.txt -X POST $DJ_URL/nodes/metric/ \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "name": "finance.completed_revenue",
+    "description": "Revenue from completed non-refund transactions",
+    "query": "
+      SELECT SUM(
+        CASE
+          WHEN status = '\''completed'\'' AND refund_flag = false
+          THEN amount_usd
+          ELSE 0
+        END
+      ) AS completed_revenue
+      FROM finance.transactions
+    ",
+    "metric_metadata": {
+      "direction": "higher_is_better",
+      "unit": "dollar"
+    },
+    "owners": ["finance-data-team@company.com"],
+    "mode": "published"
+  }'
+```
+
+**Ratio over base metrics** (decompose first, then derive — see `datajunction-semantic-model`):
+```bash
+# Step 1: create the base metrics (one curl each)
+curl -X POST $DJ_URL/nodes/metric/ -d '{
+  "name": "finance.clicks",
+  "query": "SELECT COUNT_IF(event = '\''click'\'') FROM finance.events",
+  "owners": ["marketing@company.com"],
+  "mode": "published"
+}'
+
+curl -X POST $DJ_URL/nodes/metric/ -d '{
+  "name": "finance.impressions",
+  "query": "SELECT COUNT_IF(event = '\''impression'\'') FROM finance.events",
+  "owners": ["marketing@company.com"],
+  "mode": "published"
+}'
+
+# Step 2: derived ratio metric referencing the base metrics
+curl -X POST $DJ_URL/nodes/metric/ -d '{
+  "name": "finance.conversion_rate",
+  "description": "Click-through rate as percentage",
+  "query": "SELECT finance.clicks * 100.0 / NULLIF(finance.impressions, 0)",
+  "metric_metadata": {
+    "direction": "higher_is_better",
+    "unit": "unitless"
+  },
+  "owners": ["marketing@company.com"],
+  "mode": "published"
+}'
+```
+
+DJ automatically handles divide-by-zero, but `NULLIF()` is extra safety.
+
+---
+
+## Creating Other Node Types via API
+
+Same pattern as metrics — POST JSON to the appropriate endpoint:
+
+- `POST /nodes/source/` — source nodes (catalog/schema/table refs)
+- `POST /nodes/dimension/` — dimension nodes
+- `POST /nodes/transform/` — transform nodes
+- `POST /nodes/cube/` — cubes (metric + dimension combinations)
+
+For the YAML-equivalent shapes of each, see `datajunction-repo` — the field set is the same, just expressed in JSON instead of YAML.
+
+---
+
+## Updating and Deleting Nodes
+
+**Update** (PATCH):
+```bash
+curl -b ~/.dj/cookies.txt -X PATCH $DJ_URL/nodes/finance.total_revenue/ \
+  -H 'Content-Type: application/json' \
+  -d '{"description": "Updated description"}'
+```
+
+**Deactivate** (soft delete):
+```bash
+curl -b ~/.dj/cookies.txt -X DELETE $DJ_URL/nodes/finance.total_revenue/
+```
+
+Deactivated nodes can be revived. For hard-delete (irreversible), use the `dj` CLI: `dj delete-node finance.total_revenue --hard`.

datajunction-0.0.183/datajunction/skills/datajunction-query.md

@@ -0,0 +1,248 @@
+---
+name: datajunction-query
+description: |
+  Activate this skill for querying DataJunction (DJ) — finding nodes,
+  generating SQL, fetching metric data, exploring lineage, visualizing
+  results — via the DJ UI, MCP tools, or REST/GraphQL APIs. For core DJ
+  concepts and vocabulary, invoke `datajunction`. For modeling decisions
+  (what shape something should take), invoke `datajunction-semantic-model`.
+  For authoring nodes, invoke `datajunction-repo` (YAML) or
+  `datajunction-api` (REST).
+  Keywords:
+  - query metric, query metrics
+  - generate SQL, build metric SQL
+  - get metric data, fetch metric
+  - available dimensions, common dimensions
+  - search_nodes, get_node_details, get_node_lineage
+  - get_common, build_metric_sql, get_metric_data
+  - visualize metrics
+  - MCP tools, DJ API, GraphQL
+  - DJ UI, web UI, browse
+user-invocable: false
+---
+
+# DataJunction Query
+
+Consumer-side workflow for DJ: find existing nodes, build SQL to query them, fetch data, visualize results. For the underlying concepts (node types, dimension links, star schema), invoke the `datajunction` skill.
+
+## DJ UI (Web)
+
+For interactive exploration — browsing namespaces, inspecting a node's lineage / SQL / available dimensions, building queries by clicking dimensions on/off — the DJ web UI is usually the fastest path. It's hosted at the same URL as the DJ server (e.g. `https://your-dj-server.example.com/`).
+
+**Use the UI when:**
+- The user wants to *look around* — browse what exists, read a node's description, explore lineage visually
+- They want to build a query interactively and copy the SQL out
+- They're sharing a node or query with a teammate (UI URLs are shareable links)
+
+**Use the MCP tools / API (below) when:**
+- You need programmatic access — pulling node names into a script, generating SQL as part of a workflow, fetching data into a notebook
+- The interaction is "give me this answer," not "let me explore"
+
+Suggest opening the UI when the user's question is exploratory and you don't yet know the right node name. Suggest MCP tools when you have a name in hand and need data, lineage, or generated SQL.
+
+## Discovery & Exploration (Use MCP Tools)
+
+**When you need to explore the DJ semantic layer, use these MCP tools:**
+
+### Find Available Nodes
+
+**Use MCP tool**: `search_nodes`
+- Search for metrics, dimensions, cubes by name or namespace
+- Filter by node type
+- Returns list of matching nodes
+
+**Example**: `search_nodes(query="revenue", node_type="metric", namespace="finance")`
+
+### Get Node Details
+
+**Use MCP tool**: `get_node_details`
+- Get comprehensive information about a specific node
+- Returns: SQL definition, description, available dimensions, lineage, tags
+- Input: Full node name (e.g., "finance.total_revenue")
+
+**Example**: `get_node_details(name="finance.total_revenue")`
+
+**What you get**:
+- Description and SQL definition
+- All available dimensions (via dimension links)
+- Metric metadata (unit, direction, required dimensions)
+- Upstream dependencies
+- Tags and collections
+
+### Check Common Dimensions
+
+**Use MCP tool**: `get_common`
+- Find dimensions shared across multiple metrics, or metrics shared across multiple dimensions (bidirectional)
+- Essential before querying multiple metrics together
+- Returns only entries shared by ALL of the specified inputs
+
+**Example**: `get_common(metrics=["finance.total_revenue", "growth.daily_active_users"])`
+
+**When to use**: Always check this before building queries with multiple metrics!
+
+### Get Node Lineage
+
+**Use MCP tool**: `get_node_lineage`
+- Get upstream dependencies (what data sources this uses)
+- Get downstream dependencies (what will break if you change this)
+- Direction: "upstream", "downstream", or "both"
+
+**Example**: `get_node_lineage(node_name="finance.total_revenue", direction="both")`
+
+---
+
+## Querying & SQL Generation (Use MCP Tools)
+
+**When you need to query metrics or generate SQL, use these MCP tools:**
+
+### Build Metric SQL
+
+**Use MCP tool**: `build_metric_sql`
+- Generate executable SQL for querying metrics
+- Supports filters, dimensions, ordering, limits
+- Returns SQL query and metadata
+
+**Example**:
+```
+build_metric_sql(
+  metrics=["finance.total_revenue"],
+  dimensions=["core.date.date"],
+  filters=["core.date.date >= '2024-01-01'"],
+  orderby=["core.date.date ASC"],
+  limit=100,
+  dialect="trino",
+)
+```
+
+**Returns**:
+- Generated SQL for specified engine
+- Output columns with types
+- Cube name (if materialized cube used)
+
+**Performance**: always pass a filter on a date/time dimension. When the upstream cube has a temporal partition declared on that dimension, DJ pushes the filter down to the partition column, limiting the data scanned. Without a time filter, queries scan the full underlying table.
+
+(Don't pass `include_temporal_filters=True` here — that's a parameter on `get_query_plan` for inspecting partition templates in materialization-plan SQL, not for live queries.)
+
+### Get Metric Data
+
+**Use MCP tool**: `get_metric_data`
+- Execute query and get actual data
+- Returns query results as rows
+- **Scan-cost guardrail**: materialized cubes always run; ad-hoc queries
+  (e.g. against Trino) run only if DJ's scan estimate is under the safety
+  threshold. Over-threshold queries are refused up front and the estimate
+  is surfaced so you can narrow the query (tighter date range, fewer
+  dimensions, lower limit) and retry.
+
+**Example**:
+```
+get_metric_data(
+  metrics=["finance.total_revenue", "finance.transaction_count"],
+  dimensions=["core.date.date", "core.region.region_name"],
+  filters=["core.date.date >= '2024-01-01'"],
+  orderby=["core.date.date ASC"],
+  limit=1000
+)
+```
+
+**Best practices**:
+- Always set a reasonable `limit`
+- Use specific date range filters — they drive partition pushdown and
+  keep the scan estimate below the guardrail threshold
+- Check common dimensions first for multi-metric queries
+- If you're not sure how heavy a query will be, run `get_query_plan`
+  first (see below) — it returns the scan estimate without executing
+
+### Visualize Metrics
+
+**Use MCP tool**: `visualize_metrics`
+- Query metrics and generate ASCII chart visualization
+- Creates terminal-friendly charts (line, bar, scatter)
+- Same scan-cost guardrails as `get_metric_data`
+
+**Example**:
+```
+visualize_metrics(
+  metrics=["finance.total_revenue"],
+  dimensions=["core.date.date"],
+  filters=["core.date.date >= '2024-01-01'"],
+  orderby=["core.date.date ASC"],
+  limit=90,
+  chart_type="line"
+)
+```
+
+**Chart types**:
+- `line`: Time series (default)
+- `bar`: Categorical comparisons
+- `scatter`: Correlation analysis
+
+### Pre-Flight: Get Query Plan
+
+**Use MCP tool**: `get_query_plan`
+- Returns how DJ decomposes the requested metrics into grain groups and
+  components, AND the scan estimate, without executing anything
+- Use it before `get_metric_data` / `visualize_metrics` if you suspect
+  the query might be expensive — the same scan estimate that drives the
+  refusal guardrail is surfaced here, so you can iterate on filters
+  until the estimate is reasonable
+
+**Example**:
+```
+get_query_plan(
+  metrics=["finance.total_revenue"],
+  dimensions=["core.date.date"],
+  filters=["core.date.date >= '2024-01-01'"],
+)
+```
+
+**When to use**:
+- "Will this query work?" — check before fetching
+- "Why is this fetch refused?" — re-run the same args here to see the
+  estimate
+- "What grain group will this go through?" — useful for understanding
+  multi-metric queries
+
+---
+
+## API Reference (Educational Context)
+
+The MCP tools call the DJ REST and GraphQL APIs under the hood. Here's what they're doing:
+
+### REST API Endpoints
+
+**Node discovery**:
+```bash
+GET /nodes?node_type=metric&namespace=finance
+GET /nodes/{node_name}
+```
+
+**SQL generation** (⚠️ Always use V3):
+```bash
+GET /sql/metrics/v3    # Generate query SQL
+GET /sql/measures/v3   # Generate pre-aggregation SQL
+```
+
+**Dimension compatibility**:
+```bash
+GET /metrics/common/dimensions
+```
+
+### GraphQL API
+
+Available at `/graphql`:
+
+```graphql
+query {
+  nodes(nodeType: METRIC, namespace: "finance") {
+    name
+    description
+    dimensions { name type }
+  }
+
+  commonDimensions(nodes: ["finance.revenue", "growth.users"]) {
+    name
+    type
+  }
+}
+```