ultimate-pi 0.3.0 → 0.4.0

package/firecrawl/README.md

@@ -1,49 +0,0 @@
-# Firecrawl Self-Hosted
-
-Web scraping infrastructure for the pi harness. Runs entirely in Docker — no local source checkout needed.
-
-## Quick Start
-
-```bash
-# 1. Create your env file
-cp .env.template .env
-# Edit .env if needed (defaults work for local dev)
-
-# 2. Pull pre-built images and start
-docker compose -f firecrawl/docker-compose.yaml up -d
-
-# 3. Verify
-curl http://localhost:3002/v1/health
-```
-
-## Services
-
-| Service | Port | Description |
-|---------|------|-------------|
-| `api` | 3002 | Firecrawl API + Bull queue UI at `/admin/<BULL_AUTH_KEY>/queues` |
-| `playwright-service` | 3000 (internal) | Headless browser rendering |
-| `redis` | 6379 (internal) | Job queue backing store |
-| `rabbitmq` | 5672 (internal) | Message broker for workers |
-| `nuq-postgres` | 5432 (internal) | Crawl state persistence |
-| `searxng` | 8080 | Metasearch engine for `/search` API |
-| `searxng-valkey` | 6379 (internal) | SearXNG cache |
-
-## Configuration
-
-All config lives in `.env`. See `.env.template` for all available options.
-
-### Enabling AI features
-
-Uncomment and set `OPENAI_API_KEY` in `.env`. Also supports Ollama and any OpenAI-compatible endpoint.
-
-### Using local builds instead of pre-built images
-
-Flip the `image:` ↔ `build:` lines in `docker-compose.yaml` for `api`, `playwright-service`, and `nuq-postgres`. You'll need the Firecrawl repo checked out alongside this one at `../firecrawl-selfhosted/apps/`.
-
-## SDK Usage
-
-When using Firecrawl SDKs with a self-hosted instance, pass the base URL — no API key required:
-
-```js
-const client = new Firecrawl({ apiKey: "", baseUrl: "http://localhost:3002" });
-```

package/firecrawl/docker-compose.yaml

@@ -1,201 +0,0 @@
-name: firecrawl
-
-# Firecrawl self-hosted — pre-built image variant.
-# Uses GHCR images so no local source checkout is needed.
-# To build locally instead, flip build: ↔ image: lines in each service.
-
-x-common-service: &common-service
-  image: ghcr.io/firecrawl/firecrawl
-  restart: unless-stopped
-  ulimits:
-    nofile:
-      soft: 65535
-      hard: 65535
-  networks:
-    - backend
-  extra_hosts:
-    - "host.docker.internal:host-gateway"
-  logging:
-    driver: "json-file"
-    options:
-      max-size: "10m"
-      max-file: "3"
-      compress: "true"
-
-x-common-env: &common-env
-  REDIS_URL: ${REDIS_URL:-redis://redis:6379}
-  REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
-  PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}
-  POSTGRES_USER: ${POSTGRES_USER:-postgres}
-  POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-postgres}"
-  POSTGRES_DB: ${POSTGRES_DB:-postgres}
-  POSTGRES_HOST: ${POSTGRES_HOST:-nuq-postgres}
-  POSTGRES_PORT: ${POSTGRES_PORT:-5432}
-  USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION:-false}
-  NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE:-8}
-  CRAWL_CONCURRENT_REQUESTS: ${CRAWL_CONCURRENT_REQUESTS:-10}
-  MAX_CONCURRENT_JOBS: ${MAX_CONCURRENT_JOBS:-5}
-  BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-5}
-  OPENAI_API_KEY: ${OPENAI_API_KEY}
-  OPENAI_BASE_URL: ${OPENAI_BASE_URL}
-  MODEL_NAME: ${MODEL_NAME}
-  MODEL_EMBEDDING_NAME: ${MODEL_EMBEDDING_NAME} 
-  OLLAMA_BASE_URL: ${OLLAMA_BASE_URL} 
-  AUTUMN_SECRET_KEY: ${AUTUMN_SECRET_KEY}
-  SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
-  BULL_AUTH_KEY: ${BULL_AUTH_KEY}
-  TEST_API_KEY: ${TEST_API_KEY}
-  SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
-  SUPABASE_URL: ${SUPABASE_URL}
-  SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
-  SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
-  LOGGING_LEVEL: ${LOGGING_LEVEL}
-  PROXY_SERVER: ${PROXY_SERVER}
-  PROXY_USERNAME: ${PROXY_USERNAME}
-  PROXY_PASSWORD: ${PROXY_PASSWORD}
-  SEARXNG_ENDPOINT: ${SEARXNG_ENDPOINT}
-  SEARXNG_ENGINES: ${SEARXNG_ENGINES}
-  SEARXNG_CATEGORIES: ${SEARXNG_CATEGORIES}
-
-services:
-  playwright-service:
-    image: ghcr.io/firecrawl/playwright-service:latest
-    restart: unless-stopped
-    environment:
-      PORT: 3000
-      PROXY_SERVER: ${PROXY_SERVER}
-      PROXY_USERNAME: ${PROXY_USERNAME}
-      PROXY_PASSWORD: ${PROXY_PASSWORD}
-      ALLOW_LOCAL_WEBHOOKS: ${ALLOW_LOCAL_WEBHOOKS}
-      BLOCK_MEDIA: ${BLOCK_MEDIA}
-      MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}
-    networks:
-      - backend
-    cpus: 2.0
-    mem_limit: 4G
-    memswap_limit: 4G
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "10m"
-        max-file: "3"
-        compress: "true"
-    tmpfs:
-      - /tmp/.cache:noexec,nosuid,size=1g
-
-  api:
-    <<: *common-service
-    environment:
-      <<: *common-env
-      HOST: "0.0.0.0"
-      PORT: ${INTERNAL_PORT:-3002}
-      EXTRACT_WORKER_PORT: ${EXTRACT_WORKER_PORT:-3004}
-      WORKER_PORT: ${WORKER_PORT:-3005}
-      NUQ_RABBITMQ_URL: amqp://rabbitmq:5672
-      HARNESS_STARTUP_TIMEOUT_MS: ${HARNESS_STARTUP_TIMEOUT_MS:-60000}
-      ENV: local
-    command: node dist/src/harness.js --start-docker
-    depends_on:
-      redis:
-        condition: service_started
-      playwright-service:
-        condition: service_started
-      rabbitmq:
-        condition: service_healthy
-    ports:
-      - "${PORT:-3002}:${INTERNAL_PORT:-3002}"
-    cpus: 4.0
-    mem_limit: 8G
-    memswap_limit: 8G
-
-  redis:
-    image: redis:alpine
-    restart: unless-stopped
-    networks:
-      - backend
-    command: redis-server --bind 0.0.0.0
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "5m"
-        max-file: "2"
-        compress: "true"
-    
-  rabbitmq:
-    image: rabbitmq:3-management
-    restart: unless-stopped
-    networks:
-      - backend
-    command: rabbitmq-server
-    healthcheck:
-      test: ["CMD", "rabbitmq-diagnostics", "-q", "check_running"]
-      interval: 5s
-      timeout: 5s
-      retries: 3
-      start_period: 5s
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "5m"
-        max-file: "2"
-        compress: "true"
-  
-  nuq-postgres:
-    image: ghcr.io/firecrawl/nuq-postgres:latest
-    restart: unless-stopped
-    environment:
-      POSTGRES_USER: ${POSTGRES_USER:-postgres}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
-      POSTGRES_DB: ${POSTGRES_DB:-postgres}
-    networks:
-      - backend
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "10m"
-        max-file: "3"
-        compress: "true"
-
-  # SearXNG metasearch engine — used by Firecrawl /search API when SEARXNG_ENDPOINT is set
-  searxng-valkey:
-    image: docker.io/valkey/valkey:9-alpine
-    command: valkey-server --save 30 1 --loglevel warning
-    restart: unless-stopped
-    networks:
-      - backend
-    volumes:
-      - searxng-valkey-data:/data
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "5m"
-        max-file: "2"
-        compress: "true"
-
-  searxng:
-    image: docker.io/searxng/searxng:latest
-    restart: unless-stopped
-    ports:
-      - "${SEARXNG_EXTERNAL_PORT:-8080}:8080"
-    networks:
-      - backend
-    env_file: ./searxng/searxng.env
-    volumes:
-      - ./searxng/settings.yml:/etc/searxng/settings.yml
-      - searxng-data:/var/cache/searxng
-    depends_on:
-      - searxng-valkey
-    logging:
-      driver: "json-file"
-      options:
-        max-size: "10m"
-        max-file: "3"
-        compress: "true"
-
-volumes:
-  searxng-valkey-data:
-  searxng-data:
-
-networks:
-  backend:
-    driver: bridge

package/firecrawl/searxng/searxng.env

@@ -1,3 +0,0 @@
-SEARXNG_PORT=8080
-SEARXNG_BIND_ADDRESS=0.0.0.0
-SEARXNG_SECRET=fcaacc9a734a7f85634f48aa23bfde183ef646fd0c51054f95de6fd73226b04e

package/firecrawl/searxng/settings.yml

@@ -1,85 +0,0 @@
-# SearXNG settings for Firecrawl self-hosted integration
-# Docs: https://docs.searxng.org/admin/settings/settings.html
-
-general:
-  debug: false
-  instance_name: "Firecrawl Search"
-  enable_metrics: false
-
-brand:
-  docs_url: https://docs.searxng.org/
-  public_instances: https://searx.space
-  wiki_url: https://github.com/searxng/searxng/wiki
-  issue_url: https://github.com/searxng/searxng/issues
-
-search:
-  safe_search: 0
-  autocomplete: ""
-  formats:
-    - html
-    - json
-  ban_time_on_fail: 5
-  max_ban_time_on_fail: 120
-
-server:
-  port: 8080
-  bind_address: "0.0.0.0"
-  base_url: false
-  limiter: false
-  secret_key: "fcaacc9a734a7f85634f48aa23bfde183ef646fd0c51054f95de6fd73226b04e"
-  image_proxy: false
-  method: "GET"
-
-valkey:
-  url: valkey://searxng-valkey:6379/0
-
-ui:
-  default_theme: simple
-  default_locale: ""
-
-outgoing:
-  request_timeout: 10.0
-  pool_connections: 100
-  pool_maxsize: 20
-  enable_http2: true
-
-plugins:
-  searx.plugins.calculator.SXNGPlugin:
-    active: true
-  searx.plugins.hash_plugin.SXNGPlugin:
-    active: true
-  searx.plugins.self_info.SXNGPlugin:
-    active: true
-  searx.plugins.unit_converter.SXNGPlugin:
-    active: true
-  searx.plugins.tracker_url_remover.SXNGPlugin:
-    active: true
-
-categories_as_tabs:
-  general:
-  images:
-  videos:
-  news:
-
-engines:
-  - name: duckduckgo
-    engine: duckduckgo
-    shortcut: ddg
-  - name: google
-    engine: google
-    shortcut: go
-  - name: brave
-    engine: brave
-    shortcut: br
-  - name: startpage
-    engine: startpage
-    shortcut: sp
-  - name: wikipedia
-    engine: wikipedia
-    shortcut: wp
-
-doi_resolvers:
-  oadoi.org: "https://oadoi.org/"
-  doi.org: "https://doi.org/"
-
-default_doi_resolver: "oadoi.org"