ultimate-pi 0.1.0 → 0.1.3

package/firecrawl/docker-compose.yaml

@@ -0,0 +1,201 @@
+name: firecrawl
+
+# Firecrawl self-hosted — pre-built image variant.
+# Uses GHCR images so no local source checkout is needed.
+# To build locally instead, flip build: ↔ image: lines in each service.
+
+x-common-service: &common-service
+  image: ghcr.io/firecrawl/firecrawl
+  restart: unless-stopped
+  ulimits:
+    nofile:
+      soft: 65535
+      hard: 65535
+  networks:
+    - backend
+  extra_hosts:
+    - "host.docker.internal:host-gateway"
+  logging:
+    driver: "json-file"
+    options:
+      max-size: "10m"
+      max-file: "3"
+      compress: "true"
+
+x-common-env: &common-env
+  REDIS_URL: ${REDIS_URL:-redis://redis:6379}
+  REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
+  PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}
+  POSTGRES_USER: ${POSTGRES_USER:-postgres}
+  POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-postgres}"
+  POSTGRES_DB: ${POSTGRES_DB:-postgres}
+  POSTGRES_HOST: ${POSTGRES_HOST:-nuq-postgres}
+  POSTGRES_PORT: ${POSTGRES_PORT:-5432}
+  USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION:-false}
+  NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE:-8}
+  CRAWL_CONCURRENT_REQUESTS: ${CRAWL_CONCURRENT_REQUESTS:-10}
+  MAX_CONCURRENT_JOBS: ${MAX_CONCURRENT_JOBS:-5}
+  BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-5}
+  OPENAI_API_KEY: ${OPENAI_API_KEY}
+  OPENAI_BASE_URL: ${OPENAI_BASE_URL}
+  MODEL_NAME: ${MODEL_NAME}
+  MODEL_EMBEDDING_NAME: ${MODEL_EMBEDDING_NAME} 
+  OLLAMA_BASE_URL: ${OLLAMA_BASE_URL} 
+  AUTUMN_SECRET_KEY: ${AUTUMN_SECRET_KEY}
+  SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
+  BULL_AUTH_KEY: ${BULL_AUTH_KEY}
+  TEST_API_KEY: ${TEST_API_KEY}
+  SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
+  SUPABASE_URL: ${SUPABASE_URL}
+  SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
+  SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+  LOGGING_LEVEL: ${LOGGING_LEVEL}
+  PROXY_SERVER: ${PROXY_SERVER}
+  PROXY_USERNAME: ${PROXY_USERNAME}
+  PROXY_PASSWORD: ${PROXY_PASSWORD}
+  SEARXNG_ENDPOINT: ${SEARXNG_ENDPOINT}
+  SEARXNG_ENGINES: ${SEARXNG_ENGINES}
+  SEARXNG_CATEGORIES: ${SEARXNG_CATEGORIES}
+
+services:
+  playwright-service:
+    image: ghcr.io/firecrawl/playwright-service:latest
+    restart: unless-stopped
+    environment:
+      PORT: 3000
+      PROXY_SERVER: ${PROXY_SERVER}
+      PROXY_USERNAME: ${PROXY_USERNAME}
+      PROXY_PASSWORD: ${PROXY_PASSWORD}
+      ALLOW_LOCAL_WEBHOOKS: ${ALLOW_LOCAL_WEBHOOKS}
+      BLOCK_MEDIA: ${BLOCK_MEDIA}
+      MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}
+    networks:
+      - backend
+    cpus: 2.0
+    mem_limit: 4G
+    memswap_limit: 4G
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+        compress: "true"
+    tmpfs:
+      - /tmp/.cache:noexec,nosuid,size=1g
+
+  api:
+    <<: *common-service
+    environment:
+      <<: *common-env
+      HOST: "0.0.0.0"
+      PORT: ${INTERNAL_PORT:-3002}
+      EXTRACT_WORKER_PORT: ${EXTRACT_WORKER_PORT:-3004}
+      WORKER_PORT: ${WORKER_PORT:-3005}
+      NUQ_RABBITMQ_URL: amqp://rabbitmq:5672
+      HARNESS_STARTUP_TIMEOUT_MS: ${HARNESS_STARTUP_TIMEOUT_MS:-60000}
+      ENV: local
+    command: node dist/src/harness.js --start-docker
+    depends_on:
+      redis:
+        condition: service_started
+      playwright-service:
+        condition: service_started
+      rabbitmq:
+        condition: service_healthy
+    ports:
+      - "${PORT:-3002}:${INTERNAL_PORT:-3002}"
+    cpus: 4.0
+    mem_limit: 8G
+    memswap_limit: 8G
+
+  redis:
+    image: redis:alpine
+    restart: unless-stopped
+    networks:
+      - backend
+    command: redis-server --bind 0.0.0.0
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "5m"
+        max-file: "2"
+        compress: "true"
+    
+  rabbitmq:
+    image: rabbitmq:3-management
+    restart: unless-stopped
+    networks:
+      - backend
+    command: rabbitmq-server
+    healthcheck:
+      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "5m"
+        max-file: "2"
+        compress: "true"
+  
+  nuq-postgres:
+    image: ghcr.io/firecrawl/nuq-postgres:latest
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+      POSTGRES_DB: ${POSTGRES_DB:-postgres}
+    networks:
+      - backend
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+        compress: "true"
+
+  # SearXNG metasearch engine — used by Firecrawl /search API when SEARXNG_ENDPOINT is set
+  searxng-valkey:
+    image: docker.io/valkey/valkey:9-alpine
+    command: valkey-server --save 30 1 --loglevel warning
+    restart: unless-stopped
+    networks:
+      - backend
+    volumes:
+      - searxng-valkey-data:/data
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "5m"
+        max-file: "2"
+        compress: "true"
+
+  searxng:
+    image: docker.io/searxng/searxng:latest
+    restart: unless-stopped
+    ports:
+      - "${SEARXNG_EXTERNAL_PORT:-8080}:8080"
+    networks:
+      - backend
+    env_file: ./searxng/searxng.env
+    volumes:
+      - ./searxng/settings.yml:/etc/searxng/settings.yml
+      - searxng-data:/var/cache/searxng
+    depends_on:
+      - searxng-valkey
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"
+        compress: "true"
+
+volumes:
+  searxng-valkey-data:
+  searxng-data:
+
+networks:
+  backend:
+    driver: bridge

package/firecrawl/searxng/searxng.env

@@ -0,0 +1,3 @@
+SEARXNG_PORT=8080
+SEARXNG_BIND_ADDRESS=0.0.0.0
+SEARXNG_SECRET=fcaacc9a734a7f85634f48aa23bfde183ef646fd0c51054f95de6fd73226b04e

package/firecrawl/searxng/settings.yml

@@ -0,0 +1,85 @@
+# SearXNG settings for Firecrawl self-hosted integration
+# Docs: https://docs.searxng.org/admin/settings/settings.html
+
+general:
+  debug: false
+  instance_name: "Firecrawl Search"
+  enable_metrics: false
+
+brand:
+  docs_url: https://docs.searxng.org/
+  public_instances: https://searx.space
+  wiki_url: https://github.com/searxng/searxng/wiki
+  issue_url: https://github.com/searxng/searxng/issues
+
+search:
+  safe_search: 0
+  autocomplete: ""
+  formats:
+    - html
+    - json
+  ban_time_on_fail: 5
+  max_ban_time_on_fail: 120
+
+server:
+  port: 8080
+  bind_address: "0.0.0.0"
+  base_url: false
+  limiter: false
+  secret_key: "fcaacc9a734a7f85634f48aa23bfde183ef646fd0c51054f95de6fd73226b04e"
+  image_proxy: false
+  method: "GET"
+
+valkey:
+  url: valkey://searxng-valkey:6379/0
+
+ui:
+  default_theme: simple
+  default_locale: ""
+
+outgoing:
+  request_timeout: 10.0
+  pool_connections: 100
+  pool_maxsize: 20
+  enable_http2: true
+
+plugins:
+  searx.plugins.calculator.SXNGPlugin:
+    active: true
+  searx.plugins.hash_plugin.SXNGPlugin:
+    active: true
+  searx.plugins.self_info.SXNGPlugin:
+    active: true
+  searx.plugins.unit_converter.SXNGPlugin:
+    active: true
+  searx.plugins.tracker_url_remover.SXNGPlugin:
+    active: true
+
+categories_as_tabs:
+  general:
+  images:
+  videos:
+  news:
+
+engines:
+  - name: duckduckgo
+    engine: duckduckgo
+    shortcut: ddg
+  - name: google
+    engine: google
+    shortcut: go
+  - name: brave
+    engine: brave
+    shortcut: br
+  - name: startpage
+    engine: startpage
+    shortcut: sp
+  - name: wikipedia
+    engine: wikipedia
+    shortcut: wp
+
+doi_resolvers:
+  oadoi.org: "https://oadoi.org/"
+  doi.org: "https://doi.org/"
+
+default_doi_resolver: "oadoi.org"

package/lefthook.yml

@@ -0,0 +1,8 @@
+pre-commit:
+  commands:
+    lint:
+      glob: "*.{ts,js,json,md}"
+      run: npx @biomejs/biome check --no-errors-on-unmatched --staged
+    typecheck:
+      glob: "*.ts"
+      run: npm run check:ts

package/package.json

@@ -1,17 +1,56 @@
 {
-  "name": "ultimate-pi",
-  "version": "0.1.0",
-  "description": "Custom pi package starter with caveman and impeccable.style skills",
-  "keywords": [
-    "pi-package"
-  ],
-  "license": "MIT",
-  "pi": {
-    "extensions": [
-      "./extensions"
-    ],
-    "skills": [
-      "./.agents/skills"
-    ]
-  }
-}
+	"name": "ultimate-pi",
+	"version": "0.1.3",
+	"description": "Ultimate AI coding harness for pi.dev — extensible skills, Obsidian wiki knowledge layer, compressed context, deterministic output",
+	"keywords": [
+		"pi-package",
+		"pi-mono",
+		"pi",
+		"ai-harness",
+		"coding-agent",
+		"obsidian-wiki",
+		"knowledge-base",
+		"context-compression",
+		"agent-skills",
+		"cursor-sdk",
+		"firecrawl",
+		"lean-ctx",
+		"vcc"
+	],
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/aryaniyaps/ultimate-pi"
+	},
+	"pi": {
+		"extensions": [
+			"./.pi/extensions",
+			"./.pi/providers"
+		],
+		"skills": [
+			"./.agents/skills"
+		]
+	},
+	"scripts": {
+		"check:ts": "tsc --noEmit --target ES2022 --moduleResolution nodenext --module nodenext --skipLibCheck .pi/extensions/dotenv-loader.ts",
+		"lint": "biome check",
+		"lint:fix": "biome check --fix",
+		"format": "biome format --write",
+		"format:check": "biome format",
+		"prepare": "lefthook install",
+		"test": "node --experimental-strip-types --test",
+		"test:integration": "npx -y tsx --test test/cursor-sdk-provider.integration.test.ts"
+	},
+	"devDependencies": {
+		"@biomejs/biome": "2.0.6",
+		"@mariozechner/pi-coding-agent": "0.70.2",
+		"@sinclair/typebox": "^0.34.49",
+		"lefthook": "1.11.13",
+		"typescript": "^6.0.3"
+	},
+	"dependencies": {
+		"@cursor/sdk": "^1.0.12",
+		"@sting8k/pi-vcc": "^0.3.12",
+		"asciify-image": "^0.1.10"
+	}
+}

package/vault/AGENTS.md

@@ -0,0 +1,37 @@
+# Codebase Map & Architecture Wiki
+
+Mode: GitHub / Repository (Mode B)
+Purpose: Maintain a codebase map, architecture wiki, and software design decisions.
+Owner: aryaniyaps
+Created: 2026-04-28
+
+## Structure
+
+```
+vault/
+├── .raw/              # source documents: code dumps, issue exports
+├── wiki/
+│   ├── modules/       # one note per major module / package / service
+│   ├── components/    # reusable UI or functional components
+│   ├── decisions/     # Architecture Decision Records (ADRs)
+│   ├── dependencies/  # external deps, versions, risk assessment
+│   ├── flows/         # data flows, request paths, auth flows
+│   └── meta/          # dashboards, lint reports, conventions
+└── CLAUDE.md          # schema and instructions (this file)
+```
+
+## Conventions
+
+- All notes use YAML frontmatter: type, status, created, updated, tags (minimum)
+- Wikilinks use [[Note Name]] format: filenames are unique, no paths needed
+- .raw/ contains source documents: never modify them
+- wiki/index.md is the master catalog: update on every ingest
+- wiki/log.md is append-only: never edit past entries
+- New log entries go at the TOP of the file
+
+## Operations
+
+- Ingest: drop source in .raw/, say "ingest [filename]"
+- Query: ask any question: Claude reads index first, then drills in
+- Lint: say "lint the wiki" to run a health check
+- Archive: move cold sources to .archive/ to keep .raw/ clean

package/vault/wiki/_templates/comparison.md

@@ -0,0 +1,39 @@
+---
+type: comparison
+title: "<% tp.file.title %>"
+subjects:
+  - "[[Subject A]]"
+  - "[[Subject B]]"
+dimensions:
+  - "dimension 1"
+  - "dimension 2"
+verdict: "Replace with one-line conclusion."
+created: <% tp.date.now("YYYY-MM-DD") %>
+updated: <% tp.date.now("YYYY-MM-DD") %>
+tags:
+  - comparison
+status: seed
+related: []
+sources: []
+---
+
+# <% tp.file.title %>
+
+## Overview
+
+Replace with: why these two things are being compared and what question this answers.
+
+## Comparison
+
+| Dimension | Subject A | Subject B |
+|-----------|-----------|-----------|
+| | | |
+| | | |
+
+## Verdict
+
+Replace with: one clear conclusion — which is better for what use case.
+
+## Sources
+
+-

package/vault/wiki/_templates/concept.md

@@ -0,0 +1,40 @@
+---
+type: concept
+title: "<% tp.file.title %>"
+complexity: intermediate
+domain: ""
+aliases: []
+created: <% tp.date.now("YYYY-MM-DD") %>
+updated: <% tp.date.now("YYYY-MM-DD") %>
+tags:
+  - concept
+status: seed
+related: []
+sources: []
+---
+
+# <% tp.file.title %>
+
+## Definition
+
+[What this concept is. Declarative, present tense. One clear paragraph.]
+
+## How It Works
+
+[Mechanism or explanation]
+
+## Why It Matters
+
+[Significance in this domain]
+
+## Examples
+
+-
+
+## Connections
+
+-
+
+## Sources
+
+-

package/vault/wiki/_templates/decision.md

@@ -0,0 +1,21 @@
+---
+type: decision
+status: active
+priority: 3
+date: ""
+owner: ""
+due_date: ""
+context: ""
+tags: [decision]
+created: ""
+updated: ""
+created: 2026-04-30
+updated: 2026-04-30
+---
+# {{title}}
+
+## Context
+## Options Considered
+## Decision
+## Rationale
+## Consequences

package/vault/wiki/_templates/entity.md

@@ -0,0 +1,32 @@
+---
+type: entity
+title: "<% tp.file.title %>"
+entity_type: person
+role: ""
+first_mentioned: "[[]]"
+created: <% tp.date.now("YYYY-MM-DD") %>
+updated: <% tp.date.now("YYYY-MM-DD") %>
+tags:
+  - entity
+status: seed
+related: []
+sources: []
+---
+
+# <% tp.file.title %>
+
+## Overview
+
+[Who or what this is. One paragraph.]
+
+## Key Facts
+
+-
+
+## Connections
+
+-
+
+## Sources
+
+-

package/vault/wiki/_templates/flow.md

@@ -0,0 +1,14 @@
+---
+type: flow
+status: active
+tags: [flow]
+created: ""
+updated: ""
+created: 2026-04-30
+updated: 2026-04-30
+---
+# {{title}}
+
+## Description
+## Sequence
+## Components Involved

package/vault/wiki/_templates/module.md

@@ -0,0 +1,18 @@
+---
+type: module
+path: ""
+status: active
+language: ""
+purpose: ""
+maintainer: ""
+last_updated: ""
+linked_issues: []
+depends_on: []
+used_by: []
+tags: [module]
+created: ""
+updated: ""
+created: 2026-04-30
+updated: 2026-04-30
+---
+# {{title}}

package/vault/wiki/_templates/question.md

@@ -0,0 +1,31 @@
+---
+type: question
+title: "<% tp.file.title %>"
+question: ""
+answer_quality: draft
+created: <% tp.date.now("YYYY-MM-DD") %>
+updated: <% tp.date.now("YYYY-MM-DD") %>
+tags:
+  - question
+status: developing
+related: []
+sources: []
+---
+
+# <% tp.file.title %>
+
+**Question:** [restate the original query]
+
+## Answer
+
+[The synthesized answer, with citations to specific wiki pages]
+
+(Source: [[]])
+
+## Confidence
+
+[draft | solid | definitive] — [why]
+
+## Related Questions
+
+-

package/vault/wiki/_templates/source.md

@@ -0,0 +1,39 @@
+---
+type: source
+title: "<% tp.file.title %>"
+source_type: article
+author: ""
+date_published: <% tp.date.now("YYYY-MM-DD") %>
+url: ""
+confidence: medium
+key_claims:
+  - ""
+created: <% tp.date.now("YYYY-MM-DD") %>
+updated: <% tp.date.now("YYYY-MM-DD") %>
+tags:
+  - source
+status: seed
+related: []
+sources: []
+---
+
+# <% tp.file.title %>
+
+## Summary
+
+[2-3 sentence summary of the source]
+
+## Key Claims
+
+-
+
+## Entities Mentioned
+
+- [[]] —
+
+## Concepts Introduced
+
+- [[]] —
+
+## Notes
+

package/vault/wiki/concepts/AST-Aware Code Chunking.md

@@ -0,0 +1,44 @@
+---
+type: concept
+title: "AST-Aware Code Chunking"
+created: 2026-04-30
+status: developing
+tags:
+  - chunking
+  - code-rag
+  - embeddings
+  - semantic-search
+related:
+  - "[[Semantic Codebase Indexing]]"
+  - "[[Context Engine (AI Coding)]]"
+  - "[[Contextualized Text Embedding]]"
+sources:
+  - "[[cast-code-chunking-paper]]"
+  - "[[code-chunk-library-supermemory]]"
+updated: 2026-05-02
+
+---# AST-Aware Code Chunking
+
+## Definition
+
+Splitting source code into retrievable chunks at Abstract Syntax Tree (AST) boundaries instead of arbitrary character/line limits. Preserves semantic coherence: functions, classes, and methods remain intact within chunks rather than being split mid-body.
+
+## Why It Matters
+
+Line-based chunking (e.g., split every 500 chars) produces fragments like `subtotal += item.price * item.qu` — meaningless without surrounding context. AST-aware chunking ensures each chunk is a complete, self-contained syntactic unit.
+
+## Algorithm (cAST / code-chunk)
+
+1. **Parse** source into AST via tree-sitter
+2. **Extract entities**: functions, methods, classes, imports with signatures and docstrings
+3. **Build scope tree**: hierarchical parent-child relationships (method → class → file)
+4. **Greedy window assignment**: pack complete AST nodes into chunks up to max non-whitespace character limit
+5. **Recurse into oversized nodes**: if a single function exceeds chunk limit, recurse into its children
+6. **Merge adjacent small windows**: reduce fragmentation
+7. **Add contextualized text**: prepend file path, scope chain, signatures, imports before raw code
+
+## Performance Impact
+
+- cAST paper: +4.3 Recall@5 on RepoEval, +2.67 Pass@1 on SWE-bench generation
+- code-chunk implementation: 70.1% Recall@5 vs 42.4% fixed-size baseline (with hard negatives + IoU threshold)
+- Vectara NAACL 2025: chunking strategy matters as much or more than embedding model choice