typedef-data-intelligence 0.2.0__tar.gz

typedef_data_intelligence-0.2.0/.dockerignore

@@ -0,0 +1,101 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+.venv/
+
+# Node.js
+node_modules/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.npm
+.yarn-integrity
+
+# Next.js
+.next/
+out/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+.nox/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Git
+.git/
+.gitignore
+
+# Environment
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+# Logs
+*.log
+logs/
+
+# Database files
+*.db
+*.duckdb
+*.sqlite
+*.sqlite3
+lineage_store/
+*.kuzu
+
+# Development artifacts
+tickets/
+reports/
+agent_memory/
+captured_events/
+
+# Docker
+Dockerfile*
+docker-compose*.yml
+.dockerignore
+
+# Documentation
+docs/
+*.md
+README*
+
+# Temporary files
+*.tmp
+*.temp

typedef_data_intelligence-0.2.0/.env.example

@@ -0,0 +1,191 @@
+# =============================================================================
+# TypeDef Data Intelligence - Environment Configuration
+# =============================================================================
+# This file provides example values for all environment variables used by the
+# typedef data intelligence platform. Copy this file to .env and fill in your
+# actual values.
+#
+# Configuration files reference these variables using ${VAR_NAME} syntax.
+#
+# IMPORTANT: Never commit the actual .env file - it contains secrets!
+# =============================================================================
+
+# -----------------------------------------------------------------------------
+# LLM API Keys
+# -----------------------------------------------------------------------------
+# Required for semantic analysis and agent operations.
+# At minimum, set ANTHROPIC_API_KEY (used by default for agents).
+
+# Anthropic Claude (primary - used by agents)
+ANTHROPIC_API_KEY=sk-ant-...
+
+# OpenAI (optional - for GPT models in semantic analysis)
+OPENAI_API_KEY=sk-proj-...
+
+# Google API Key (optional - for Gemini models)
+GOOGLE_API_KEY=...
+
+# OpenRouter (optional - for multi-model access, used by semantic analysis)
+OPENROUTER_API_KEY=sk-or-v1-...
+
+# Pydantic AI Gateway (optional, used for agents in the future)
+PYDANTIC_AI_GATEWAY_API_KEY=paig_...
+
+# -----------------------------------------------------------------------------
+# AWS Bedrock Configuration (Optional)
+# -----------------------------------------------------------------------------
+# Use AWS Bedrock as an alternative LLM provider for PydanticAI agents.
+# Bedrock uses IAM authentication - credentials are resolved via boto3's
+# default credential chain:
+#   1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+#   2. AWS credentials file (~/.aws/credentials)
+#   3. IAM role (when running on AWS infrastructure like EC2, ECS, Lambda)
+#
+# Model format in config: bedrock:anthropic.claude-3-sonnet-20240229-v1:0
+# With explicit region: bedrock:us-east-1/anthropic.claude-3-sonnet-20240229-v1:0
+
+# AWS Region for Bedrock (defaults to us-east-1 if not set)
+AWS_DEFAULT_REGION=us-west-2
+
+# AWS credentials (optional if using IAM roles or ~/.aws/credentials)
+# AWS_ACCESS_KEY_ID=AKIA...
+# AWS_SECRET_ACCESS_KEY=...
+
+# AWS Session Token (for temporary credentials only)
+# AWS_SESSION_TOKEN=...
+
+# AWS Profile (if using named profiles in ~/.aws/credentials)
+# AWS_PROFILE=default
+
+# -----------------------------------------------------------------------------
+# dbt Project Configuration
+# -----------------------------------------------------------------------------
+# These variables define which dbt project to analyze and where it's located.
+#
+# The platform supports external git-managed dbt projects via environment vars.
+# Example projects: medallion-reference, mattermost-data-warehouse
+#
+# Usage: just medallion-quickstart (handles git clone + setup automatically)
+
+# Parent directory containing your dbt projects
+GITHUB_DBT_PROJECT_LOCAL_PATH=/Users/you/dev/dbt-workspace
+
+# =============================================================================
+# OPTION 1: Medallion Reference Test Project (simpler setup)
+# =============================================================================
+# A demo SaaS finance analytics project for testing
+GITHUB_DBT_PROJECT_REPO_NAME=medallion-reference
+DBT_PROJECT_ROOT=.
+
+# =============================================================================
+# OPTION 2: Mattermost Analytics (production-scale example)
+# =============================================================================
+# Uncomment these and comment out the medallion settings above
+# GITHUB_DBT_PROJECT_REPO_NAME=mattermost-data-warehouse
+# DBT_PROJECT_NAME_OVERRIDE=mattermost-analytics
+# DBT_PROJECT_ROOT=transform/mattermost-analytics
+# DBT_PROFILES_DIR=profile
+
+# -----------------------------------------------------------------------------
+# Snowflake Configuration
+# -----------------------------------------------------------------------------
+# Data warehouse credentials for:
+# - Querying semantic models
+# - Profiling tables/columns
+# - Running data quality checks
+# - Agent SQL execution
+
+SNOWFLAKE_ACCOUNT=your-account.region
+SNOWFLAKE_USER=your-username
+SNOWFLAKE_ROLE=ANALYST
+SNOWFLAKE_WAREHOUSE=COMPUTE_WH
+SNOWFLAKE_DATABASE=ANALYTICS
+SNOWFLAKE_SCHEMA=MARTS
+
+# Path to private key file for key-pair authentication
+# Generate: openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out key.p8 -nocrypt
+SNOWFLAKE_PRIVATE_KEY_PATH=/Users/you/.dbt/snowflake_key.p8
+
+# -----------------------------------------------------------------------------
+# Mattermost-specific Snowflake Configuration (Optional)
+# -----------------------------------------------------------------------------
+# Additional databases/schemas for mattermost-analytics project
+# SNOWFLAKE_TRANSFORM_DATABASE=ANALYTICS
+# SNOWFLAKE_TRANSFORM_SCHEMA=MATTERMOST_ANALYTICS
+# SNOWFLAKE_TRANSFORM_ROLE=ENGINEER
+# MATTERMOST_RAW_DB=RAW
+# MATTERMOST_ANALYTICS_DB=ANALYTICS
+
+# -----------------------------------------------------------------------------
+# User Context
+# -----------------------------------------------------------------------------
+# Default user/org for agent operations
+DEFAULT_USER_ID=your-username
+DEFAULT_ORG_ID=your-org
+
+# -----------------------------------------------------------------------------
+# Linear Configuration (Optional)
+# -----------------------------------------------------------------------------
+# Ticket management for agent collaboration.
+# If not set, falls back to filesystem-based tickets.
+#
+# Setup:
+# 1. Create Linear team: https://linear.app/
+# 2. Get team ID from URL or API
+# 3. Generate API keys: Settings -> API -> Personal API Keys
+
+LINEAR_TEAM_ID=your-team-uuid
+LINEAR_ANALYST_API_KEY=lin_api_...
+LINEAR_DATA_ENGINEER_API_KEY=lin_api_...
+
+# -----------------------------------------------------------------------------
+# GitHub Configuration (Optional)
+# -----------------------------------------------------------------------------
+# For data-engineer agent git operations
+# If unset, agent uses your local git credentials
+# GITHUB_TOKEN=ghp_...
+
+# -----------------------------------------------------------------------------
+# Observability (Optional)
+# -----------------------------------------------------------------------------
+# Logfire token for Pydantic AI observability
+# Get from: https://logfire.pydantic.dev/
+LOGFIRE_TOKEN=pylf_v1_...
+
+# -----------------------------------------------------------------------------
+# Slack Integration (Optional)
+# -----------------------------------------------------------------------------
+# For Slack bot notifications
+SLACK_SIGNING_SECRET=your-signing-secret
+SLACK_BOT_TOKEN=xoxb-...
+
+# AG UI base URL (for agent UI)
+AG_UI_BASE_URL=http://localhost:8000
+
+# -----------------------------------------------------------------------------
+# OpenLineage Configuration (Optional)
+# -----------------------------------------------------------------------------
+# For collecting runtime lineage from dbt runs.
+# Enables tracking actual execution patterns vs static DAG.
+
+# OpenLineage collection endpoint (if using lineage serve)
+# OPENLINEAGE_URL=http://localhost:8080/api/v1/lineage
+
+# Namespace for grouping lineage events
+# OPENLINEAGE_NAMESPACE=dbt://demo_finance
+
+# Set to false to enable collection
+# OPENLINEAGE_DISABLED=false
+
+# -----------------------------------------------------------------------------
+# Configuration File Selection
+# -----------------------------------------------------------------------------
+# The platform uses different config files for different modes:
+#
+# config.ingest.yml  - For loading dbt projects (just medallion-full)
+# config.cli.yml     - For CLI agent operations (lineage-agent-anthropic)
+# config.api.yml     - For web API + UI (just dev-webui)
+#
+# Each file can reference these environment variables using ${VAR_NAME} syntax.
+# See the individual config files for specific variable usage.
+# -----------------------------------------------------------------------------

typedef_data_intelligence-0.2.0/.gitignore

@@ -0,0 +1,54 @@
+# Generated reports and charts (from agent visualization tools)
+/charts/
+/reports/
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+venv/
+ENV/
+env/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Database files
+*.db
+*.duckdb
+*.duckdb.wal
+*.kuzu
+
+# Logs
+*.log
+
+# Environment variables
+.env
+.env.local

typedef_data_intelligence-0.2.0/.justfile_help.txt

@@ -0,0 +1,60 @@
+# CLI Agents Quick Reference
+
+## From typedef_data_intelligence/ directory:
+
+Interactive Mode:
+  just cli-analyst                 # Start analyst agent (REPL)
+  just cli-engineer                # Start data engineering agent
+  just cli-quality                 # Start data quality agent
+
+Single Question:
+  just ask-analyst "question"      # Ask analyst
+  just ask-engineer "question"     # Ask engineer
+  just ask-quality "question"      # Ask quality agent
+
+## From dbt_projects/ directory:
+
+Interactive Mode:
+  just analyst                     # Start analyst agent
+  just engineer                    # Start engineer agent
+  just quality                     # Start quality agent
+
+Single Question:
+  just ask "question"              # Ask analyst
+  just ask-eng "question"          # Ask engineer
+  just ask-qa "question"           # Ask quality agent
+
+## Configuration:
+
+Default config (config.example.yml):
+  just cli-analyst
+
+Custom config:
+  just cli-analyst CONFIG=myconfig.yml
+
+Environment variables:
+  export UNIFIED_CONFIG=/path/to/config.yml
+  export CLI_USER_ID=your_user@example.com
+  export CLI_ORG_ID=your_org
+
+## Example Questions:
+
+Analyst:
+  "What semantic views are available?"
+  "Show me total ARR by month for 2024"
+  "Create a chart of revenue trends"
+
+Engineer:
+  "What models depend on dim_customers?"
+  "Show me the grain of fct_arr_reporting_monthly"
+  "What tables are in the marts schema?"
+
+Quality:
+  "Find any failed dbt runs"
+  "Show error patterns from the last week"
+  "Check for null values in critical columns"
+
+## More Info:
+
+  just cli-info                    # Show detailed usage
+  cat docs/webui/cli-testing.md    # Read full documentation

typedef_data_intelligence-0.2.0/CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+
+## [0.2.0](https://github.com/typedef-ai/data-intelligence/compare/v0.1.0...v0.2.0) (2026-01-13)
+
+
+### Features
+
+* Add Data Engineer tools/prompts to Pydantic CLI Agent, improved stability of logfire ([#32](https://github.com/typedef-ai/data-intelligence/issues/32)) ([08efbb1](https://github.com/typedef-ai/data-intelligence/commit/08efbb1fead1fd7727f641d42c85ec4086e8ca85))
+* add rich progress tracking to ingest pipeline ([#47](https://github.com/typedef-ai/data-intelligence/issues/47)) ([91adc20](https://github.com/typedef-ai/data-intelligence/commit/91adc200a4d8f1e71756a31cf7024913545e973f))
+* Add smart autoscroll control in TUI and enforce feature branch workflow for DE copilot ([#81](https://github.com/typedef-ai/data-intelligence/issues/81)) ([42e787d](https://github.com/typedef-ai/data-intelligence/commit/42e787db12d90585d9fc85559f4ebf464f76f385))
+* Add support for bedrock models ([#58](https://github.com/typedef-ai/data-intelligence/issues/58)) ([7711bcc](https://github.com/typedef-ai/data-intelligence/commit/7711bcc37b33c13e940d275c171c07f63cdf6a62))
+* add the daemon ticket based experience to the tui ([#70](https://github.com/typedef-ai/data-intelligence/issues/70)) ([fc87df8](https://github.com/typedef-ai/data-intelligence/commit/fc87df83100ad4f86189c6fc36d4dcb7f212169c))
+* add TUI setup wizard with Textual (prototype) ([#49](https://github.com/typedef-ai/data-intelligence/issues/49)) ([6147e8c](https://github.com/typedef-ai/data-intelligence/commit/6147e8c5d5340144719d2e690f4b49a60065eb68))
+* adding a webui for data analyst ([#2](https://github.com/typedef-ai/data-intelligence/issues/2)) ([9096a18](https://github.com/typedef-ai/data-intelligence/commit/9096a186c56087c101ec46c1a82ccebd90103593))
+* dev docker compose ([#4](https://github.com/typedef-ai/data-intelligence/issues/4)) ([97e1d73](https://github.com/typedef-ai/data-intelligence/commit/97e1d73aa6de637eebe5f245e41ad71ad597ba97))
+* falkordb full text search, snowflake db visibility filtering ([#46](https://github.com/typedef-ai/data-intelligence/issues/46)) ([5110928](https://github.com/typedef-ai/data-intelligence/commit/5110928ae576303fbe25456ae718e93d136213f5))
+* full clustering implementation ([#10](https://github.com/typedef-ai/data-intelligence/issues/10)) ([71bebad](https://github.com/typedef-ai/data-intelligence/commit/71bebad07d0060dc0a07224472bffd0c0db4db88))
+* Improve demo workflow with benchmark branch preservation and session management cleanup ([#64](https://github.com/typedef-ai/data-intelligence/issues/64)) ([df015f8](https://github.com/typedef-ai/data-intelligence/commit/df015f8613d532a14b7eada0b7c7c05c7719edf9))
+* Mattermost clones working plus tasks for scenarios ([#43](https://github.com/typedef-ai/data-intelligence/issues/43)) ([fd77b49](https://github.com/typedef-ai/data-intelligence/commit/fd77b49b4ebd5f2905f9c227c0479a20f5c09d98))
+* prototype benchmark for data-intelligence ([#26](https://github.com/typedef-ai/data-intelligence/issues/26)) ([de2871a](https://github.com/typedef-ai/data-intelligence/commit/de2871afdd78d586fbcd3da86d82d9bf1522d616))
+* separate logical and physical concepts ([#20](https://github.com/typedef-ai/data-intelligence/issues/20)) ([3a9bd05](https://github.com/typedef-ai/data-intelligence/commit/3a9bd05cd6b07328d5098893574a1ab7477a17ef))
+
+
+### Bug Fixes
+
+* **backend/docker:** misc path/entrypoint fixes to get it running ([#22](https://github.com/typedef-ai/data-intelligence/issues/22)) ([aa47c3f](https://github.com/typedef-ai/data-intelligence/commit/aa47c3fba30895bde9f107d3f9bea4014fb2a8b1))
+* daemon only grabs labeled tickets ([#18](https://github.com/typedef-ai/data-intelligence/issues/18)) ([748f6c3](https://github.com/typedef-ai/data-intelligence/commit/748f6c332cb5fac490f4c6645a2bdab7d4749e07))
+* if the dbt project gets cloned with git, enable git in the config ([#73](https://github.com/typedef-ai/data-intelligence/issues/73)) ([6e99162](https://github.com/typedef-ai/data-intelligence/commit/6e9916232ab569c4385cbf54d6651c1312517c92))
+* limit the daemon to only process tickets assigned to data-engineer-agent ([#74](https://github.com/typedef-ai/data-intelligence/issues/74)) ([ac94267](https://github.com/typedef-ai/data-intelligence/commit/ac94267c4965b040977cdf3933f39ace67cd06e1))
+* minor fixes to get medallion + frontend to work ([#59](https://github.com/typedef-ai/data-intelligence/issues/59)) ([1444285](https://github.com/typedef-ai/data-intelligence/commit/1444285ba030a4a2f242a4494e102f106d81d868))
+* readme ([#1](https://github.com/typedef-ai/data-intelligence/issues/1)) ([172767a](https://github.com/typedef-ai/data-intelligence/commit/172767a78d33c963e5a09685b34f5016bd7a9714))

typedef_data_intelligence-0.2.0/Dockerfile

@@ -0,0 +1,60 @@
+# Production Dockerfile for FastAPI Backend
+# Multi-stage build for optimized image size
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS builder
+
+WORKDIR /workspace
+
+# Copy workspace configuration
+COPY pyproject.toml uv.lock ./
+COPY typedef_data_intelligence/pyproject.toml ./typedef_data_intelligence/
+
+# Copy source code
+COPY typedef_data_intelligence/src ./typedef_data_intelligence/src
+
+# Install minimal runtime dependencies
+RUN apt-get update && apt-get install -y \
+  curl git build-essential
+
+# Install dependencies with frozen lockfile (reproducible builds)
+# Use FalkorDB backend for production
+# --all-packages ensures workspace members are installed
+RUN uv sync --frozen --package typedef-data-intelligence
+
+# ============================================================================
+# Runtime Stage
+# ============================================================================
+FROM python:3.12-slim
+
+# Install minimal runtime dependencies
+RUN apt-get update && apt-get install -y \
+  curl \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /workspace
+
+# Copy virtual environment from builder
+COPY --from=builder /workspace/.venv /workspace/.venv
+
+# Copy source code
+COPY --from=builder /workspace/typedef_data_intelligence /workspace/typedef_data_intelligence
+
+# Set environment
+ENV PATH="/workspace/.venv/bin:$PATH"
+ENV PYTHONPATH="/workspace/typedef_data_intelligence/src"
+
+# Create non-root user for security
+RUN useradd -m -u 1000 appuser && \
+  chown -R appuser:appuser /workspace
+
+USER appuser
+
+# Expose FastAPI port
+EXPOSE 8000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+  CMD curl -f http://localhost:8000/health || exit 1
+
+# Run backend server
+# Config file should be mounted at runtime via volume
+CMD ["python", "-m", "lineage.api.pydantic"]

typedef_data_intelligence-0.2.0/Dockerfile.dev

@@ -0,0 +1,35 @@
+# Development Dockerfile for FastAPI Backend
+# This runs in dev mode with hot reloading - no production optimization
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+  curl git build-essential \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /workspace/typedef_data_intelligence
+
+# Copy package metadata and lock file
+COPY pyproject.toml ./
+# Copy source code so uv sync can install the package
+# (This will be overridden by volume mount at runtime for hot reload)
+COPY src ./src
+
+# Install package and dependencies with FalkorDB backend
+RUN uv sync
+
+# Set environment
+ENV PATH="/workspace/typedef_data_intelligence/.venv/bin:$PATH"
+ENV PYTHONPATH="/workspace/typedef_data_intelligence/src"
+
+# Copy entrypoint script
+COPY docker-entrypoint.dev.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Expose FastAPI port
+EXPOSE 8000
+
+# Run backend server in development mode with hot reload
+# Note: Source code is mounted via volumes at runtime (see docker-compose.yml)
+# The entrypoint script reinstalls the package in editable mode after volumes are mounted
+ENTRYPOINT ["/entrypoint.sh"]

typedef_data_intelligence-0.2.0/Justfile

@@ -0,0 +1,65 @@
+set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
+
+# Default config file
+CONFIG_FILE := "config.example.yml"
+
+# Default recipe - show available commands
+default:
+    @just --list
+
+# ============================================================================
+# Backend Management
+# ============================================================================
+
+# Show current backend configuration
+config-info:
+    @echo "Configuration System"
+    @echo "==================="
+    @echo ""
+    @echo "Unified config file approach - all settings in one YAML file"
+    @echo ""
+    @echo "Example configs:"
+    @echo "  - config.example.yml       (FalkorDB + DuckDB)"
+    @echo "  - config.production.example.yml  (Neo4j + Snowflake)"
+    @echo "  - config.postgres.example.yml    (PostgreSQL AGE + DuckDB)"
+    @echo ""
+    @echo "Config structure:"
+    @echo "  lineage:          # Graph database config"
+    @echo "    backend: falkordb|falkordblite|neo4j|postgres-age|arcadedb"
+    @echo "    population:     # Data loading settings"
+    @echo "      model: google/gemini-2.5-flash-lite"
+    @echo "      max_workers: 64"
+    @echo "  data:             # Data warehouse config"
+    @echo "    backend: duckdb|snowflake"
+    @echo "  agent:            # Agent runtime config"
+    @echo "    model: anthropic:claude-sonnet-4-5-20250929"
+    @echo ""
+    @echo "Environment variables:"
+    @echo "  Use \$${VAR_NAME} syntax in config.yml for sensitive data"
+    @echo "  Example: password: \$${NEO4J_PASSWORD}"
+
+
+# ============================================================================
+# Development
+# ============================================================================
+
+# Run tests
+test:
+    uv run pytest
+
+# Run tests with coverage
+test-cov:
+    uv run pytest --cov=src/lineage
+
+# Format code
+fmt:
+    uv run ruff format .
+
+# Lint code
+lint:
+    uv run ruff check .
+
+# Install dependencies
+install:
+    uv sync --all-extras
+