dtSpark 1.0.4__py3-none-any.whl

dtSpark/resources/config.yaml.template

@@ -0,0 +1,631 @@
+logging:
+  level: INFO
+  rotation_backup_count: 5
+  log_to_console: false
+
+# Database Configuration
+# Supports: SQLite (default), MySQL, MariaDB, PostgreSQL, Microsoft SQL Server
+database:
+  type: sqlite  # Database type: sqlite, mysql, mariadb, postgresql, mssql
+
+  # SQLite configuration (local file-based database)
+  sqlite:
+    path: "./data/conversations.db"  # Path to SQLite database file
+
+  # MySQL/MariaDB configuration (remote database server)
+  mysql:
+    host: localhost
+    port: 3306
+    database: dtawsbedrockcli
+    username: null  # Set to null to prompt on startup
+    password: null  # Set to null to prompt on startup (recommended for security)
+    ssl: false  # Use SSL/TLS connection
+
+  # PostgreSQL configuration (remote database server)
+  postgresql:
+    host: localhost
+    port: 5432
+    database: dtawsbedrockcli
+    username: null  # Set to null to prompt on startup
+    password: null  # Set to null to prompt on startup (recommended for security)
+    ssl: false  # Use SSL/TLS connection
+
+  # Microsoft SQL Server configuration (remote database server)
+  mssql:
+    host: localhost
+    port: 1433
+    database: dtawsbedrockcli
+    username: null  # Set to null to prompt on startup
+    password: null  # Set to null to prompt on startup (recommended for security)
+    ssl: false  # Use SSL/TLS connection
+    driver: "ODBC Driver 17 for SQL Server"  # ODBC driver name
+
+  # Notes:
+  # - SQLite is the default and requires no additional configuration
+  # - For remote databases (MySQL, PostgreSQL, MSSQL):
+  #   - Set credentials directly in config (less secure) OR
+  #   - Leave credentials as null to be prompted on startup (more secure)
+  # - Ensure the required Python package is installed:
+  #   - MySQL: pip install mysql-connector-python
+  #   - PostgreSQL: pip install psycopg2-binary
+  #   - MSSQL: pip install pyodbc
+
+# Interface Configuration
+interface:
+  type: cli  # Interface type: 'cli' (command-line) or 'web' (web browser)
+  web:
+    host: 127.0.0.1  # Localhost only for security (do not change)
+    port: 0  # 0 = automatically select random available port, or specify a port number
+    session_timeout_minutes: 0  # 0 = never expires, or set to minutes of inactivity before session expires
+    dark_theme: true  # Use dark theme for web interface
+    ssl:
+      enabled: true  # Enable HTTPS with self-signed certificate
+      auto_generate_cert: true  # Automatically generate self-signed certificate if not found
+      cert_file: certs/ssl_cert.pem  # Path to SSL certificate file
+      key_file: certs/ssl_key.pem  # Path to SSL private key file
+    auto_open_browser: true  # Automatically open web browser when server starts
+
+# LLM Provider Configuration
+llm_providers:
+  # Mandatory Model - if set, forces this model for ALL conversations (cannot be changed by user)
+  # When set, model selection and changing is disabled, and any model specified in predefined conversations is ignored
+  # Example: mandatory_model: "claude-3-5-sonnet-20241022"
+  mandatory_model: null  # Set to a model ID to force that model for all conversations, or null to allow user choice
+
+  # Mandatory Provider - if set, forces the use of a specific provider/access method for the mandatory model
+  # Valid values: "AWS Bedrock", "Ollama", "Anthropic Direct"
+  # If not set, the application will automatically search all enabled providers for the model
+  # Example: mandatory_provider: "Anthropic Direct"
+  mandatory_provider: null  # Set to force a specific provider, or null to auto-detect
+
+  # AWS Bedrock Configuration
+  aws_bedrock:
+    enabled: true  # Set to false to disable AWS Bedrock
+    region: us-east-1  # AWS region for Bedrock API
+
+    # Authentication method: Choose one of the following:
+    #
+    # 1. SSO Profile (recommended for interactive use)
+    sso_profile: default  # AWS SSO profile name
+    #
+    # 2. API Keys (for programmatic access, CI/CD, or when SSO is not available)
+    # Uncomment and set these to use API keys instead of SSO:
+    # access_key_id: YOUR_ACCESS_KEY_ID
+    # secret_access_key: YOUR_SECRET_ACCESS_KEY
+    # session_token: YOUR_SESSION_TOKEN  # Optional: for temporary credentials
+    #
+    # Note: API keys take precedence over SSO profile if both are configured
+    #
+    # Security Warning: Do NOT commit API keys to version control!
+
+    cost_tracking:
+      enabled: false  # Set to true to enable AWS Bedrock cost gathering and display
+
+  # Ollama Configuration (local LLM server)
+  ollama:
+    enabled: false  # Set to true to enable Ollama
+    base_url: "http://localhost:11434"  # Ollama API URL
+    verify_ssl: true  # Set to false to skip SSL certificate verification (for self-signed certs)
+                      # Only disable if connecting to Ollama via HTTPS with a self-signed certificate
+
+  # Anthropic Direct API Configuration
+  anthropic:
+    enabled: false  # Set to true to enable Anthropic direct API
+    api_key: null  # Your Anthropic API key (or set via ANTHROPIC_API_KEY environment variable)
+    rate_limit_max_retries: 5  # Maximum retry attempts for rate limit errors (default: 5)
+    rate_limit_base_delay: 2.0  # Base delay in seconds for exponential backoff (default: 2.0)
+    # Rate limit handling uses exponential backoff: 2^0=1s, 2^1=2s, 2^2=4s, 2^3=8s, 2^4=16s
+
+bedrock:
+  max_tokens: 8192  # Maximum tokens per response
+  temperature: 0.7
+  request_timeout: 300  # Timeout in seconds for model inference requests (default: 5 minutes)
+#  model_id: null  # Optional: Lock to specific model (disables model selection/changing)
+
+conversation:
+#  global_instructions_path: null  # Optional: Path to file containing global instructions
+  rollup_threshold: 0.3  # Trigger compaction at 30% of context window (default for cost control)
+                         # Lower values compact more aggressively, saving tokens but potentially losing context
+                         # Higher values (e.g., 0.8) preserve more context but use more tokens
+                         # Note: Per-conversation compaction_threshold can override this in the database
+  rollup_summary_ratio: 0.3  # Summarise to 30% of original length
+  max_tool_result_tokens: 10000  # Maximum tokens per tool result
+  max_tool_iterations: 25  # Maximum number of consecutive tool calls before stopping (default: 25)
+  max_tool_selections: 30  # Maximum number of tools to send with each request (reduces token usage)
+  emergency_rollup_threshold: 0.95  # Force rollup at 95% even during tool use to prevent max_tokens errors
+
+# Model Context Window Limits
+# Maps model ID patterns to context window sizes (in tokens)
+# Used for intelligent context compaction when approaching model limits
+# The system matches model IDs using partial matching (model ID contains pattern)
+model_context_limits:
+  # Anthropic Claude Models (Direct API and Bedrock)
+  # These limits apply to models accessed via AWS Bedrock or Anthropic Direct API
+  anthropic:
+    # Claude 4 family
+    claude-opus-4:
+      context_window: 200000
+      max_output: 32000
+    claude-sonnet-4:
+      context_window: 200000
+      max_output: 64000
+    claude-opus-4.5:
+      context_window: 200000
+      max_output: 64000
+    claude-sonnet-4.5:
+      context_window: 200000
+      max_output: 64000
+
+    # Claude 3.5 family
+    claude-3-5-sonnet:
+      context_window: 200000
+      max_output: 8192
+    claude-3-5-haiku:
+      context_window: 200000
+      max_output: 8192
+
+    # Claude 3 family (legacy)
+    claude-3-opus:
+      context_window: 200000
+      max_output: 4096
+    claude-3-sonnet:
+      context_window: 200000
+      max_output: 4096
+    claude-3-haiku:
+      context_window: 200000
+      max_output: 4096
+
+    # Default for unknown Anthropic/Claude models
+    default:
+      context_window: 200000
+      max_output: 4096
+
+  # AWS Bedrock Models (non-Anthropic)
+  aws_bedrock:
+    # Amazon Titan Text
+    amazon.titan-text-express:
+      context_window: 8192
+      max_output: 8192
+    amazon.titan-text-lite:
+      context_window: 4096
+      max_output: 4096
+    amazon.titan-text-premier:
+      context_window: 32000
+      max_output: 8192
+
+    # Meta Llama (on Bedrock)
+    meta.llama3-8b:
+      context_window: 8192
+      max_output: 2048
+    meta.llama3-70b:
+      context_window: 8192
+      max_output: 2048
+    meta.llama3-1:
+      context_window: 128000
+      max_output: 8192
+    meta.llama3-2:
+      context_window: 128000
+      max_output: 8192
+    meta.llama3-3:
+      context_window: 128000
+      max_output: 8192
+
+    # Mistral (on Bedrock)
+    mistral.mistral-7b:
+      context_window: 32000
+      max_output: 8192
+    mistral.mixtral-8x7b:
+      context_window: 32000
+      max_output: 8192
+    mistral.mistral-large:
+      context_window: 128000
+      max_output: 8192
+
+    # Cohere
+    cohere.command-r:
+      context_window: 128000
+      max_output: 4096
+    cohere.command-r-plus:
+      context_window: 128000
+      max_output: 4096
+
+    # AI21 Labs
+    ai21.jamba:
+      context_window: 256000
+      max_output: 4096
+    ai21.j2:
+      context_window: 8192
+      max_output: 8192
+
+    # Default for unknown Bedrock models
+    default:
+      context_window: 8192
+      max_output: 4096
+
+  # Ollama Models (local)
+  ollama:
+    # Llama family
+    llama3:
+      context_window: 8192
+      max_output: 4096
+    llama3.1:
+      context_window: 128000
+      max_output: 8192
+    llama3.2:
+      context_window: 128000
+      max_output: 8192
+    llama3.3:
+      context_window: 128000
+      max_output: 8192
+
+    # Mistral family
+    mistral:
+      context_window: 32000
+      max_output: 8192
+    mistral-nemo:
+      context_window: 128000
+      max_output: 8192
+    mixtral:
+      context_window: 32000
+      max_output: 8192
+
+    # Qwen family
+    qwen:
+      context_window: 32000
+      max_output: 8192
+    qwen2:
+      context_window: 32000
+      max_output: 8192
+    qwen2.5:
+      context_window: 128000
+      max_output: 8192
+    qwen2.5-coder:
+      context_window: 128000
+      max_output: 8192
+
+    # Code models
+    codellama:
+      context_window: 16000
+      max_output: 4096
+    deepseek-coder:
+      context_window: 16000
+      max_output: 4096
+    starcoder:
+      context_window: 8192
+      max_output: 4096
+
+    # Other popular models
+    gemma:
+      context_window: 8192
+      max_output: 4096
+    gemma2:
+      context_window: 8192
+      max_output: 8192
+    phi3:
+      context_window: 128000
+      max_output: 4096
+    phi4:
+      context_window: 16000
+      max_output: 4096
+    command-r:
+      context_window: 128000
+      max_output: 4096
+    yi:
+      context_window: 200000
+      max_output: 4096
+
+    # Default for unknown Ollama models
+    default:
+      context_window: 4096
+      max_output: 2048
+
+  # Global default (fallback for any unmatched model/provider)
+  default:
+    context_window: 8192
+    max_output: 4096
+
+token_management:
+  enabled: false  # Set to true to enable token usage management and limits
+  max_input_tokens: 100000  # Maximum input tokens in the rolling window
+  max_output_tokens: 50000  # Maximum output tokens in the rolling window
+  period_hours: 24  # Tracking period in hours (rolling window)
+  allow_override: true  # Allow user to override token limits at 100%
+
+# Prompt Inspection (Cyber Security)
+# Detects and mitigates security risks in user prompts
+prompt_inspection:
+  enabled: false  # Set to true to enable prompt inspection
+  inspection_level: basic  # Inspection level: basic, standard, strict
+                           # basic: Fast pattern matching only
+                           # standard: Pattern matching + keyword analysis
+                           # strict: Pattern matching + LLM semantic analysis
+  action: warn  # Action when violations detected: block, warn, sanitise, log_only
+                # block: Reject prompt completely
+                # warn: Show warning and ask for confirmation
+                # sanitise: Attempt to clean the prompt (with confirmation)
+                # log_only: Log violation but allow prompt
+
+  # LLM-based inspection (for 'strict' level)
+  llm_inspection:
+    enabled: false  # Set to true to enable LLM semantic analysis
+    model: anthropic.claude-3-haiku-20240307-v1:0  # Fast, cheap model for analysis
+    provider: AWS Bedrock  # Provider: AWS Bedrock, Ollama, Anthropic Direct
+    max_tokens: 500  # Maximum tokens for analysis response
+    confidence_threshold: 0.7  # 0.0-1.0, higher = more strict
+
+  # Pattern-based detection
+  patterns:
+    check_prompt_injection: true  # Detect attempts to override system instructions
+    check_jailbreak: true  # Detect attempts to bypass safety guidelines
+    check_code_injection: true  # Detect malicious code patterns
+    check_pii: false  # Detect potential PII exposure
+    check_excessive_length: true  # Detect unusually long prompts
+    max_prompt_length: 50000  # Maximum prompt length in characters
+
+  # Content moderation (optional)
+  content_moderation:
+    check_profanity: false  # Check for profanity
+    check_toxicity: false  # Check for toxic content (requires AWS Comprehend)
+
+  # Blocklist/Allowlist
+  custom_patterns_file: null  # Optional: path to custom regex patterns file
+  whitelist_users: []  # User GUIDs exempt from inspection
+
+  # Logging and audit
+  log_violations: true  # Log violations to database for audit trail
+  alert_on_repeated_violations: true  # Alert when user exceeds threshold
+  violation_threshold: 5  # Alert after N violations in 24 hours
+
+# AWS Comprehend (optional, for content moderation)
+# Note: This service incurs additional AWS costs
+aws_comprehend:
+  enabled: false  # Set to true to enable AWS Comprehend integration
+  detect_pii: false  # Detect personally identifiable information
+  detect_toxic_content: false  # Detect toxic/offensive content
+
+# Predefined Conversations
+# Define conversations that are automatically created and synchronised on startup.
+# These conversations are MANAGED BY CONFIGURATION and have the following protections:
+#   - Instructions CANNOT be changed by users
+#   - Model CANNOT be changed by users
+#   - Files CANNOT be deleted by users
+# Configuration changes are automatically synchronised on application startup.
+#
+# Instructions and Files Loading:
+#   - Instructions can be:
+#     1. Inline text (multi-line strings)
+#     2. Path to file loaded via ResourceManager (package resources)
+#     3. Direct file path (absolute or relative)
+#   - Files are loaded using:
+#     1. ResourceManager (for package resources)
+#     2. Direct file path (fallback if ResourceManager returns None)
+#
+# Model and Provider Selection:
+#   - If llm_providers.mandatory_model is set, it overrides conversation model settings
+#   - If provider is omitted or null, the system auto-detects the provider for the model
+#   - Provider options: "AWS Bedrock", "Ollama", or "Anthropic Direct"
+#
+predefined_conversations:
+  enabled: false  # Set to true to enable predefined conversations
+
+  conversations:
+    # Example 1: Code Review Assistant with inline instructions
+    # - name: "Code Review Assistant"
+    #   instructions: |
+    #     You are a senior software engineer specialising in code review.
+    #     Focus on code quality, security, performance, and maintainability.
+    #     Provide constructive feedback with specific examples.
+    #   files: []  # No files attached by default
+    #   model: "claude-sonnet-4.5"
+    #   provider: null  # If null or omitted, will auto-detect the provider for this model
+    #   enabled: true
+    #
+    # Example 2: Simple assistant with inline instructions and no files
+    # - name: "General Assistant"
+    #   instructions: "You are a helpful assistant."  # Short inline text (no file loading attempted)
+    #   files: []
+    #   model: "claude-sonnet-4"
+    #   enabled: true
+
+mcp_config:
+  enabled: false  # Set to true to enable MCP integration
+  servers: []
+    # Example MCP server configurations:
+    #
+    # ============================================================================
+    # TRANSPORT TYPES
+    # ============================================================================
+    # - stdio: Local server process (command + args)
+    # - http: Remote Streamable HTTP server (url required)
+    # - sse: Remote Server-Sent Events server (url required)
+    #
+    # ============================================================================
+    # AUTHENTICATION OPTIONS (for http and sse transports)
+    # ============================================================================
+    # auth_type: none | bearer | api_key | basic | custom
+    #
+    # - none: No authentication (default)
+    # - bearer: Authorization: Bearer <token>
+    # - api_key: Custom header with API key (default: X-API-Key)
+    # - basic: HTTP Basic authentication (username + password)
+    # - custom: Use custom_headers dictionary for full control
+    #
+    # ============================================================================
+    # SSL/TLS OPTIONS (for http and sse transports)
+    # ============================================================================
+    # ssl_verify: true | false
+    #
+    # - true: Verify SSL certificates (default, recommended for production)
+    # - false: Skip SSL certificate verification (for self-signed certificates)
+    #
+    # WARNING: Disabling SSL verification reduces security. Only use this for:
+    # - Development/testing environments with self-signed certificates
+    # - Internal servers with custom certificate authorities
+    #
+    # ============================================================================
+    # EXAMPLES
+    # ============================================================================
+    #
+    # Test server (included with the project):
+    # - name: test-server
+    #   transport: stdio
+    #   command: python
+    #   args:
+    #     - ./tests/test_mcp_server.py
+    #   enabled: true
+    #
+    # Stdio transport (local server process):
+    # - name: filesystem
+    #   transport: stdio
+    #   command: uvx
+    #   args:
+    #     - mcp-server-filesystem
+    #     - /path/to/allowed/directory
+    #   enabled: true
+    #
+    # Stdio transport with environment variables:
+    # - name: api-tools
+    #   transport: stdio
+    #   command: uvx
+    #   args:
+    #     - mcp-server-api-tools
+    #   env:
+    #     API_KEY: your-api-key-here
+    #     API_ENDPOINT: https://api.example.com
+    #   enabled: true
+    #
+    # HTTP transport (no authentication):
+    # - name: local-http
+    #   transport: http
+    #   url: http://localhost:8000/mcp
+    #   timeout: 30
+    #   enabled: true
+    #
+    # HTTP transport with Bearer token authentication:
+    # - name: remote-api
+    #   transport: http
+    #   url: https://api.example.com/mcp
+    #   auth_type: bearer
+    #   auth_token: your-bearer-token-here  # Or use env var: ${REMOTE_API_TOKEN}
+    #   timeout: 60
+    #   enabled: true
+    #
+    # HTTP transport with API key authentication:
+    # - name: api-service
+    #   transport: http
+    #   url: https://service.example.com/mcp
+    #   auth_type: api_key
+    #   auth_token: your-api-key-here
+    #   auth_header_name: X-API-Key  # Default header name, can be customised
+    #   enabled: true
+    #
+    # HTTP transport with Basic authentication:
+    # - name: internal-service
+    #   transport: http
+    #   url: https://internal.example.com/mcp
+    #   auth_type: basic
+    #   basic_username: service-user
+    #   basic_password: service-password
+    #   enabled: true
+    #
+    # HTTP transport with custom headers:
+    # - name: custom-auth-service
+    #   transport: http
+    #   url: https://custom.example.com/mcp
+    #   auth_type: custom
+    #   custom_headers:
+    #     X-Tenant-ID: "my-tenant"
+    #     X-Custom-Auth: "secret-value"
+    #     X-Request-Source: "dt-spark"
+    #   enabled: true
+    #
+    # SSE transport with Bearer token:
+    # - name: sse-events
+    #   transport: sse
+    #   url: https://events.example.com/mcp
+    #   auth_type: bearer
+    #   auth_token: your-sse-token-here
+    #   timeout: 120  # SSE connections often need longer timeouts
+    #   enabled: true
+    #
+    # SSE transport with API key:
+    # - name: sse-service
+    #   transport: sse
+    #   url: https://stream.example.com/mcp
+    #   auth_type: api_key
+    #   auth_token: your-api-key
+    #   auth_header_name: Authorization  # Some services use Authorization header for API keys
+    #   enabled: true
+    #
+    # HTTP transport with self-signed certificate (SSL verification disabled):
+    # - name: internal-dev-server
+    #   transport: http
+    #   url: https://dev-internal.local:8443/mcp
+    #   auth_type: api_key
+    #   auth_token: dev-api-key
+    #   ssl_verify: false  # Skip SSL verification for self-signed certificates
+    #   timeout: 30
+    #   enabled: true
+    #
+    # SSE transport with self-signed certificate:
+    # - name: internal-sse-dev
+    #   transport: sse
+    #   url: https://dev-events.local:8443/mcp
+    #   auth_type: bearer
+    #   auth_token: dev-bearer-token
+    #   ssl_verify: false  # Skip SSL verification for self-signed certificates
+    #   timeout: 120
+    #   enabled: true
+
+# Embedded Filesystem Tools
+# Built-in tools for accessing local filesystem (alternative to MCP filesystem server)
+embedded_tools:
+  filesystem:
+    enabled: false  # Set to true to enable embedded filesystem tools
+    allowed_path: ./  # Root path that tools can access (absolute or relative)
+                      # Tools cannot access files outside this directory
+    access_mode: read  # Access mode: "read" (read-only) or "read_write" (read and write)
+                       # read: Only list, search, and read operations
+                       # read_write: Adds write and create directory operations
+
+    # Security Notes:
+    # - All file paths are validated to ensure they are within allowed_path
+    # - Attempting to access files outside allowed_path will be rejected
+    # - Use read-only mode when write operations are not needed
+    # - Consider using specific subdirectories rather than root paths
+
+# Tool Permissions
+# Controls how tool usage permissions are handled
+tool_permissions:
+  auto_approve: false  # Set to true to automatically approve all tool usage without prompting
+                       # When true: Tools run without user confirmation (not stored in database)
+                       # When false: User is prompted on first-time tool usage (choice is stored)
+                       #
+                       # Use Cases:
+                       # - Development/testing: Set to true to avoid constant prompts
+                       # - Trusted environments: Set to true when all tools are known to be safe
+                       # - Production/security-conscious: Set to false to maintain user control
+                       #
+                       # Note: When reverted to false, users will be prompted again for tools
+                       # that were auto-approved (since permissions weren't stored)
+
+# Daemon Configuration
+# Background process for executing autonomous actions independently
+daemon:
+  poll_interval: 30           # Seconds between database polls for action changes
+  heartbeat_interval: 60      # Seconds between heartbeat updates to registry
+  lock_timeout: 300           # Seconds before stale execution locks are cleared
+  pid_file: "./daemon.pid"  # PID file location for daemon process
+
+  # Usage:
+  #   dtSpark daemon start           # Start daemon in background
+  #   dtSpark daemon start --foreground  # Run in foreground (for debugging)
+  #   dtSpark daemon status          # Check if daemon is running
+  #   dtSpark daemon stop            # Stop the daemon gracefully
+  #   dtSpark daemon restart         # Restart the daemon
+  #
+  # The daemon:
+  #   - Runs autonomous actions on their configured schedules
+  #   - Detects changes to actions made via Web UI or CLI
+  #   - Prevents duplicate execution when both daemon and UI try to run same action
+  #   - Uses the same database and configuration as the main application

dtSpark/safety/__init__.py

@@ -0,0 +1,22 @@
+"""
+Safety module for prompt inspection and security.
+
+This module provides:
+- Pattern-based prompt inspection
+- LLM-based semantic analysis
+- Cyber Security audit trail
+- Multi-provider LLM support
+
+
+"""
+
+from .prompt_inspector import PromptInspector, InspectionResult
+from .violation_logger import ViolationLogger
+from .patterns import PatternMatcher
+
+__all__ = [
+    'PromptInspector',
+    'InspectionResult',
+    'ViolationLogger',
+    'PatternMatcher',
+]