openviking 0.3.10.dev29__tar.gz → 0.3.11.dev39__tar.gz

openviking-0.3.11.dev39/.github/workflows/_docs-deploy.yml

@@ -0,0 +1,48 @@
+name: 18. _Docs Deploy
+
+on:
+  workflow_call:
+
+jobs:
+  deploy:
+    name: Build and Deploy Docs
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: docs/package-lock.json
+
+      - name: Install docs dependencies
+        run: npm ci
+        working-directory: docs
+
+      - name: Build docs
+        run: npm run docs:build
+        working-directory: docs
+        env:
+          DOCS_BASE: /
+
+      - name: Configure GitHub Pages
+        uses: actions/configure-pages@v6
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v4
+        with:
+          path: docs/.vitepress/dist
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v5

openviking-0.3.11.dev39/.github/workflows/_docs.yml

@@ -0,0 +1,31 @@
+name: 17. _Docs Build
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    name: Build Docs
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: docs/package-lock.json
+
+      - name: Install docs dependencies
+        run: npm ci
+        working-directory: docs
+
+      - name: Build docs
+        run: npm run docs:build
+        working-directory: docs
+        env:
+          DOCS_BASE: /

openviking-0.3.11.dev39/.github/workflows/api_test.yml

@@ -0,0 +1,515 @@
+name: 06. API Integration Tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 1,4,7,10,13 * * *'
+  pull_request:
+    branches:
+      - 'main'
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+      - 'LICENSE'
+      - 'CONTRIBUTING.md'
+      - '**.png'
+      - '**.jpg'
+      - '**.jpeg'
+      - '**.gif'
+      - '**.svg'
+      - '.gitignore'
+      - '.editorconfig'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  api-tests:
+    name: API Integration Tests (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' || github.repository == 'volcengine/OpenViking'
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        os: ${{ ((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && fromJSON('["ubuntu-24.04", "macos-14", "windows-latest"]') || fromJSON('["ubuntu-24.04"]') }}
+    
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+          fetch-depth: 100
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+
+      - name: Cache Python dependencies (Unix)
+        if: runner.os != 'Windows'
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Cache Python dependencies (Windows)
+        if: runner.os == 'Windows'
+        uses: actions/cache@v5
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install system dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update -y && sudo apt-get install -y cmake build-essential ffmpeg --no-install-recommends
+
+      - name: Install system dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install cmake ffmpeg
+
+      - name: Install system dependencies (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          echo "cmake is pre-installed on Windows runner and will also be installed via pip"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Install build dependencies (system pip)
+        run: |
+          python -m pip install --upgrade pip
+          pip install "setuptools>=70.1" setuptools_scm cmake wheel
+          echo "---"
+          python -c "import setuptools; print(f'setuptools version: {setuptools.__version__}')"
+
+      - name: Install dependencies using uv sync
+        run: |
+          echo "Installing dependencies using uv sync..."
+          uv sync --frozen
+
+      - name: Install build dependencies (uv pip)
+        run: |
+          echo "Installing build dependencies to uv environment..."
+          uv pip install setuptools setuptools_scm cmake wheel maturin
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Build C++ extensions
+        shell: bash
+        run: |
+          export OV_SKIP_OV_BUILD=1
+          mkdir -p openviking/bin
+          touch openviking/bin/ov
+          chmod +x openviking/bin/ov
+          echo "OV_SKIP_OV_BUILD=1 set, skipping ov CLI Rust build (ragfs build still needed for server)"
+          
+          if [ -z "$SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENVIKING" ]; then
+            PRETEND_VERSION=$(git describe --tags --always 2>/dev/null | sed -E 's/^v?([0-9]+\.[0-9]+\.[0-9]+).*/\1.dev0/' || echo "0.0.0.dev0")
+            echo "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OPENVIKING=$PRETEND_VERSION" >> $GITHUB_ENV
+            echo "Using pretend version: $PRETEND_VERSION"
+          fi
+          
+          uv run python setup.py build_ext --inplace
+
+      - name: Build ragfs-python native extension
+        shell: bash
+        run: |
+          RAGFS_LIB_DIR="openviking/lib"
+          RAGFS_SO_COUNT=$(ls -1 "$RAGFS_LIB_DIR"/ragfs_python*.so "$RAGFS_LIB_DIR"/ragfs_python*.pyd "$RAGFS_LIB_DIR"/ragfs_python*.dylib 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0")
+          
+          if [ "$RAGFS_SO_COUNT" -gt 0 ]; then
+            echo "✅ ragfs_python native extension already exists:"
+            ls -la "$RAGFS_LIB_DIR"/ragfs_python*
+            exit 0
+          fi
+          
+          echo "ragfs_python native lib not found after build_ext, building via maturin..."
+          TMPDIR_RAGFS=$(mktemp -d)
+          cd crates/ragfs-python
+          uv run --no-project maturin build --release --features s3 --out "$TMPDIR_RAGFS"
+          cd ../..
+          
+          WHL_FILE=$(ls -1 "$TMPDIR_RAGFS"/ragfs_python-*.whl 2>/dev/null | head -1)
+          if [ -z "$WHL_FILE" ]; then
+            echo "❌ ERROR: maturin build produced no wheel"
+            ls -la "$TMPDIR_RAGFS"/
+            exit 1
+          fi
+          
+          echo "Extracting ragfs_python from wheel: $WHL_FILE"
+          mkdir -p "$RAGFS_LIB_DIR"
+          WHL_FILE_PY=$(echo "$WHL_FILE" | sed 's/\\/\\\\/g')
+          RAGFS_LIB_DIR_PY=$(echo "$RAGFS_LIB_DIR" | sed 's/\\/\\\\/g')
+          uv run python -c "
+          import zipfile, os, sys, stat
+          with zipfile.ZipFile('${WHL_FILE_PY}') as zf:
+              for name in zf.namelist():
+                  bn = os.path.basename(name)
+                  if bn.startswith('ragfs_python') and (bn.endswith('.so') or bn.endswith('.pyd') or bn.endswith('.dylib')):
+                      dst = os.path.join('${RAGFS_LIB_DIR_PY}', bn)
+                      with zf.open(name) as src, open(dst, 'wb') as f:
+                          f.write(src.read())
+                      os.chmod(dst, 0o755)
+                      print(f'  [OK] ragfs-python: extracted {bn} -> {dst}')
+                      sys.exit(0)
+          print('[ERROR] No ragfs_python native library found in wheel')
+          sys.exit(1)
+          "
+          rm -rf "$TMPDIR_RAGFS"
+          
+          echo "Verifying ragfs_python native extension:"
+          ls -la "$RAGFS_LIB_DIR"/ragfs_python*
+
+      - name: Install API test dependencies
+        run: |
+          cd tests/api_test
+          uv pip install -r requirements.txt
+
+      - name: Create OpenViking config file (Unix)
+        if: runner.os != 'Windows'
+        id: create-config-unix
+        env:
+          VLM_API_KEY: ${{ secrets.VLM_API_KEY }}
+          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
+        run: |
+          mkdir -p $HOME/.openviking
+          
+          HAS_SECRETS=false
+          if [ -n "$VLM_API_KEY" ] && [ -n "$EMBEDDING_API_KEY" ]; then
+            HAS_SECRETS=true
+            echo "Using full configuration with VLM and Embedding"
+          else
+            echo "Using minimal configuration (no VLM/Embedding)"
+          fi
+          
+          echo "HAS_SECRETS=$HAS_SECRETS" >> $GITHUB_ENV
+          
+          if [ "$HAS_SECRETS" = "true" ]; then
+            cat > $HOME/.openviking/ov.conf << EOF
+          {
+            "server": {
+              "root_api_key": "test-root-api-key"
+            },
+            "vlm": {
+              "provider": "volcengine",
+              "api_key": "$VLM_API_KEY",
+              "model": "doubao-seed-2-0-mini-260215",
+              "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+              "temperature": 0.1,
+              "max_retries": 3
+            },
+            "embedding": {
+              "dense": {
+                "provider": "volcengine",
+                "api_key": "$EMBEDDING_API_KEY",
+                "model": "doubao-embedding-vision-251215",
+                "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+                "dimension": 1024,
+                "input": "multimodal"
+              }
+            }
+          }
+          EOF
+          else
+            cat > $HOME/.openviking/ov.conf << EOF
+          {
+            "server": {
+              "root_api_key": "test-root-api-key"
+            },
+            "vlm": {
+              "provider": "volcengine",
+              "api_key": "dummy-vlm-api-key",
+              "model": "doubao-seed-2-0-mini-260215",
+              "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+              "temperature": 0.1,
+              "max_retries": 3
+            },
+            "embedding": {
+              "dense": {
+                "provider": "volcengine",
+                "api_key": "dummy-embedding-api-key",
+                "model": "doubao-embedding-vision-251215",
+                "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+                "dimension": 1024,
+                "input": "multimodal"
+              }
+            }
+          }
+          EOF
+          fi
+          
+          echo "Config file created at $HOME/.openviking/ov.conf"
+
+      - name: Create OpenViking config file (Windows)
+        if: runner.os == 'Windows'
+        id: create-config-windows
+        env:
+          VLM_API_KEY: ${{ secrets.VLM_API_KEY }}
+          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
+        shell: pwsh
+        run: |
+          $configDir = "$env:USERPROFILE\.openviking"
+          New-Item -ItemType Directory -Force -Path $configDir | Out-Null
+          
+          $hasSecrets = "false"
+          if ($env:VLM_API_KEY -and $env:EMBEDDING_API_KEY) {
+            $hasSecrets = "true"
+            Write-Host "Using full configuration with VLM and Embedding"
+          } else {
+            Write-Host "Using minimal configuration (no VLM/Embedding)"
+          }
+          
+          echo "HAS_SECRETS=$hasSecrets" >> $env:GITHUB_ENV
+          
+          if ($hasSecrets -eq "true") {
+            $config = @"
+          {
+            "server": {
+              "root_api_key": "test-root-api-key"
+            },
+            "vlm": {
+              "provider": "volcengine",
+              "api_key": "$env:VLM_API_KEY",
+              "model": "doubao-seed-2-0-mini-260215",
+              "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+              "temperature": 0.1,
+              "max_retries": 3
+            },
+            "embedding": {
+              "dense": {
+                "provider": "volcengine",
+                "api_key": "$env:EMBEDDING_API_KEY",
+                "model": "doubao-embedding-vision-251215",
+                "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+                "dimension": 1024,
+                "input": "multimodal"
+              }
+            }
+          }
+          "@
+          } else {
+            $config = @"
+          {
+            "server": {
+              "root_api_key": "test-root-api-key"
+            },
+            "vlm": {
+              "provider": "volcengine",
+              "api_key": "dummy-vlm-api-key",
+              "model": "doubao-seed-2-0-mini-260215",
+              "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+              "temperature": 0.1,
+              "max_retries": 3
+            },
+            "embedding": {
+              "dense": {
+                "provider": "volcengine",
+                "api_key": "dummy-embedding-api-key",
+                "model": "doubao-embedding-vision-251215",
+                "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+                "dimension": 1024,
+                "input": "multimodal"
+              }
+            }
+          }
+          "@
+          }
+          
+          $config | Out-File -FilePath "$configDir\ov.conf" -Encoding utf8
+          Write-Host "Config file created at $configDir\ov.conf"
+
+      - name: Find available port and start OpenViking Server (Unix)
+        if: runner.os != 'Windows'
+        id: start-server-unix
+        run: |
+          find_available_port() {
+            local port=1933
+            while true; do
+              if ! lsof -Pi :$port -sTCP:LISTEN -t >/dev/null 2>&1; then
+                echo $port
+                return
+              fi
+              port=$((port + 1))
+            done
+          }
+          
+          SERVER_PORT=$(find_available_port)
+          echo "Using port: $SERVER_PORT"
+          echo "SERVER_PORT=$SERVER_PORT" >> $GITHUB_ENV
+          
+          echo "Starting OpenViking Server on port $SERVER_PORT..."
+          export ROOT_API_KEY=test-root-api-key
+          export SERVER_PORT=$SERVER_PORT
+          nohup uv run python -m openviking.server.bootstrap > openviking-server.log 2>&1 &
+          echo $! > openviking-server.pid
+          echo "SERVER_PID=$(cat openviking-server.pid)" >> $GITHUB_ENV
+          
+          echo "Waiting for server to start..."
+          for i in {1..30}; do
+            if curl -s http://127.0.0.1:$SERVER_PORT/health | grep -q '"healthy":true'; then
+              echo "Server is ready!"
+              break
+            fi
+            echo "Waiting... ($i/30)"
+            sleep 2
+          done
+          
+          if ! curl -s http://127.0.0.1:$SERVER_PORT/health | grep -q '"healthy":true'; then
+            echo "Server failed to start!"
+            echo "Server logs:"
+            cat openviking-server.log
+            exit 1
+          fi
+
+      - name: Find available port and start OpenViking Server (Windows)
+        if: runner.os == 'Windows'
+        id: start-server-windows
+        shell: pwsh
+        run: |
+          $port = 1933
+          Write-Host "Using port: $port"
+          echo "SERVER_PORT=$port" >> $env:GITHUB_ENV
+          
+          Write-Host "Starting OpenViking Server on port $port..."
+          
+          $logFile = Join-Path $PWD "openviking-server.log"
+          $errFile = Join-Path $PWD "openviking-server-error.log"
+          $batchFile = Join-Path $PWD "start-server.bat"
+          
+          $batchContent = "@echo off`r`nset ROOT_API_KEY=test-root-api-key`r`nset SERVER_PORT=$port`r`nuv run python -m openviking.server.bootstrap`r`n"
+          Set-Content -Path $batchFile -Value $batchContent -Encoding ASCII
+          
+          Write-Host "Batch file created at: $batchFile"
+          Write-Host "Batch file content:"
+          Get-Content $batchFile
+          
+          $psi = New-Object System.Diagnostics.ProcessStartInfo
+          $psi.FileName = "cmd.exe"
+          $psi.Arguments = "/C `"$batchFile`""
+          $psi.UseShellExecute = $true
+          $psi.WindowStyle = [System.Diagnostics.ProcessWindowStyle]::Hidden
+          $psi.WorkingDirectory = $PWD.Path
+          
+          $process = [System.Diagnostics.Process]::Start($psi)
+          
+          Write-Host "Server process started with PID: $($process.Id)"
+          echo "SERVER_PID=$($process.Id)" >> $env:GITHUB_ENV
+          
+          Write-Host "Waiting for server to start..."
+          $ready = $false
+          for ($i = 1; $i -le 30; $i++) {
+            Start-Sleep -Seconds 2
+            try {
+              $response = Invoke-WebRequest -Uri "http://127.0.0.1:$port/health" -UseBasicParsing -TimeoutSec 5
+              if ($response.Content -match '"healthy":true') {
+                Write-Host "Server is ready!"
+                $ready = $true
+                break
+              }
+            } catch {
+              Write-Host "Waiting... ($i/30)"
+            }
+          }
+          
+          if (-not $ready) {
+            Write-Host "Server failed to start!"
+            Write-Host "Checking if process is still running..."
+            $proc = Get-Process -Id $process.Id -ErrorAction SilentlyContinue
+            if ($proc) {
+              Write-Host "Process is still running with PID: $($proc.Id)"
+            } else {
+              Write-Host "Process has exited"
+            }
+            exit 1
+          }
+
+      - name: Run API Tests (Unix)
+        if: runner.os != 'Windows'
+        id: run-tests-unix
+        run: |
+          cd tests/api_test
+          export OPENVIKING_API_KEY=test-root-api-key
+          export SERVER_URL=http://127.0.0.1:${{ env.SERVER_PORT }}
+          
+          if [ "${{ env.HAS_SECRETS }}" = "true" ]; then
+            echo "Running full test suite with VLM/Embedding"
+            uv run python -m pytest . -v --html=api-test-report.html --self-contained-html \
+              --ignore=scenarios/resources_retrieval_slow/
+          else
+            echo "Running basic tests only (no VLM/Embedding)"
+            uv run python -m pytest . -v --html=api-test-report.html --self-contained-html \
+              --ignore=retrieval/ --ignore=resources/test_pack.py --ignore=resources/test_wait_processed.py \
+              --ignore=admin/ --ignore=skills/ --ignore=system/test_system_status.py --ignore=system/test_is_healthy.py --ignore=system/test_system_wait.py \
+              --ignore=scenarios/ -k "not test_observer"
+          fi
+        continue-on-error: true
+
+      - name: Run API Tests (Windows)
+        if: runner.os == 'Windows'
+        id: run-tests-windows
+        shell: pwsh
+        run: |
+          cd tests/api_test
+          $env:OPENVIKING_API_KEY = "test-root-api-key"
+          $env:SERVER_URL = "http://127.0.0.1:${{ env.SERVER_PORT }}"
+          
+          if ($env:HAS_SECRETS -eq "true") {
+            Write-Host "Running full test suite with VLM/Embedding (Windows: skipping filesystem tests)"
+            uv run python -m pytest . -v --html=api-test-report.html --self-contained-html --ignore=filesystem/ --ignore=scenarios/resources_retrieval_slow/
+          } else {
+            Write-Host "Running basic tests only (no VLM/Embedding, Windows: skipping filesystem tests)"
+            uv run python -m pytest . -v --html=api-test-report.html --self-contained-html --ignore=retrieval/ --ignore=resources/test_pack.py --ignore=resources/test_wait_processed.py --ignore=admin/ --ignore=skills/ --ignore=system/test_system_status.py --ignore=system/test_is_healthy.py --ignore=system/test_system_wait.py --ignore=filesystem/ --ignore=scenarios/ -k "not test_observer"
+          }
+        continue-on-error: true
+
+      - name: Upload test reports
+        uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: api-test-reports-${{ matrix.os }}-${{ github.run_id }}
+          path: |
+            tests/api_test/api-test-report.html
+            openviking-server.log
+
+      - name: Stop OpenViking Server (Unix)
+        if: runner.os != 'Windows' && always()
+        run: |
+          if [ -f openviking-server.pid ]; then
+            kill $(cat openviking-server.pid) 2>/dev/null || true
+            pkill -f "openviking.server.bootstrap" 2>/dev/null || true
+          fi
+
+      - name: Stop OpenViking Server (Windows)
+        if: runner.os == 'Windows' && always()
+        shell: pwsh
+        run: |
+          if ($env:SERVER_PID) {
+            Stop-Process -Id $env:SERVER_PID -Force -ErrorAction SilentlyContinue
+          }
+          Get-Process -Name python -ErrorAction SilentlyContinue | Stop-Process -Force
+
+      - name: Check test results (Unix)
+        if: runner.os != 'Windows' && steps.run-tests-unix.outcome != 'success'
+        run: |
+          echo "API tests failed!"
+          exit 1
+
+      - name: Check test results (Windows)
+        if: runner.os == 'Windows' && steps.run-tests-windows.outcome != 'success'
+        shell: pwsh
+        run: |
+          Write-Host "API tests failed!"
+          exit 1