synapse-hub 0.0.0-stage → 2.0.1

package/tools/matrix_sharder.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+Enterprise Duration-Balanced Test Matrix Sharder
+------------------------------------------------
+Balances parallel CI/CD test execution across N containers using historical
+runtimes and the Greedy LPT (Longest Processing Time First) Bin-Packing algorithm.
+Eliminates CI runner bottlenecks where one worker runs for 15m while others finish in 2m.
+"""
+
+import os
+import sys
+import json
+import sqlite3
+import argparse
+from pathlib import Path
+from typing import List, Dict, Any
+
+# Enforce UTF-8 stdout encoding for Windows console compatibility
+if hasattr(sys.stdout, 'reconfigure'):
+    try:
+        sys.stdout.reconfigure(encoding='utf-8')
+        sys.stderr.reconfigure(encoding='utf-8')
+    except Exception:
+        pass
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+HUB_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
+CACHE_DB_PATH = os.path.join(SCRIPT_DIR, "memory_cache.db")
+
+
+def find_test_files(target_dir: str) -> List[str]:
+    """Discovers all executable test scripts across Katalon, Playwright, or Pytest suites."""
+    target_path = Path(target_dir)
+    test_files = []
+    valid_exts = {".groovy", ".ts", ".js", ".py"}
+
+    for root, _, files in os.walk(target_path):
+        for f in files:
+            # Match standard test naming patterns
+            lower = f.lower()
+            if any(lower.endswith(ext) for ext in valid_exts):
+                if "test" in lower or "spec" in lower or "tc_" in lower or "scripts" in root.lower():
+                    rel_p = os.path.relpath(os.path.join(root, f), target_dir).replace("\\", "/")
+                    test_files.append(rel_p)
+
+    return sorted(test_files)
+
+
+def estimate_test_duration(file_path: str, full_path: str) -> float:
+    """Estimates test execution duration in seconds using historical cache or file heuristic."""
+    # Check SQLite historical runtime cache if available
+    if os.path.exists(CACHE_DB_PATH):
+        try:
+            conn = sqlite3.connect(CACHE_DB_PATH)
+            cur = conn.cursor()
+            cur.execute("CREATE TABLE IF NOT EXISTS test_runtimes (test_path TEXT PRIMARY KEY, duration REAL)")
+            cur.execute("SELECT duration FROM test_runtimes WHERE test_path = ?", (file_path,))
+            row = cur.fetchone()
+            conn.close()
+            if row and row[0]:
+                return float(row[0])
+        except Exception:
+            pass
+
+    # Heuristic fallback: ~0.5s per line of script
+    try:
+        if os.path.exists(full_path):
+            with open(full_path, "r", encoding="utf-8", errors="ignore") as fp:
+                lines = len(fp.readlines())
+            return max(5.0, round(lines * 0.25, 1))
+    except Exception:
+        pass
+
+    return 10.0
+
+
+def shard_tests_lpt(test_files: List[str], target_dir: str, num_shards: int) -> List[Dict[str, Any]]:
+    """
+    Applies Longest Processing Time First (LPT) bin-packing to distribute tests evenly.
+    """
+    num_shards = max(1, num_shards)
+    if not test_files:
+        return [{"shard_index": i + 1, "total_duration": 0.0, "tests": []} for i in range(num_shards)]
+
+    # 1. Calculate duration for each test
+    tests_with_cost = []
+    for tf in test_files:
+        fp = os.path.join(target_dir, tf)
+        dur = estimate_test_duration(tf, fp)
+        tests_with_cost.append((tf, dur))
+
+    # 2. Sort descending by duration
+    tests_with_cost.sort(key=lambda x: x[1], reverse=True)
+
+    # 3. Initialize shards
+    shards = [{"shard_index": i + 1, "total_duration": 0.0, "tests": []} for i in range(num_shards)]
+
+    # 4. Greedily assign to the shard with the least current cumulative duration
+    for test_file, dur in tests_with_cost:
+        # Find shard with min duration
+        min_shard = min(shards, key=lambda s: s["total_duration"])
+        min_shard["tests"].append(test_file)
+        min_shard["total_duration"] = round(min_shard["total_duration"] + dur, 2)
+
+    return shards
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Duration-Balanced Test Matrix Sharder")
+    parser.add_argument("target", nargs="?", default=os.getcwd(), help="Target repository root")
+    parser.add_argument("--shards", "-n", type=int, default=4, help="Number of parallel shards/workers (default: 4)")
+    parser.add_argument("--json", action="store_true", help="Output matrix in JSON format")
+    args = parser.parse_args()
+
+    target_dir = os.path.abspath(args.target)
+    test_files = find_test_files(target_dir)
+    shards = shard_tests_lpt(test_files, target_dir, args.shards)
+
+    if args.json:
+        print(json.dumps({"total_tests": len(test_files), "shards_count": args.shards, "shards": shards}, indent=2))
+    else:
+        print(f"\n======================================================================")
+        print(f"      DURATION-BALANCED TEST MATRIX SHARDER ({args.shards} PARALLEL WORKERS)    ")
+        print(f"======================================================================")
+        print(f"  Target Scope : {target_dir}")
+        print(f"  Total Tests  : {len(test_files)} files\n")
+
+        for s in shards:
+            print(f"📦 Shard {s['shard_index']}: {len(s['tests'])} tests (~{s['total_duration']}s total)")
+            for t in s['tests'][:5]:
+                print(f"   • {t}")
+            if len(s['tests']) > 5:
+                print(f"   ... and {len(s['tests']) - 5} more")
+            print()
+
+        # Calculate imbalance delta
+        durations = [s["total_duration"] for s in shards]
+        delta = max(durations) - min(durations) if durations else 0
+        print(f"⚡ Shard Imbalance Delta: {delta:.1f}s (Optimal parallel distribution)")
+        print(f"======================================================================\n")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tools/mcp_excel_server.ps1

@@ -0,0 +1,251 @@
+param()
+
+[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
+
+function Read-ExcelSheet {
+    param(
+        [string]$FilePath,
+        [string]$SheetName = ""
+    )
+
+    if ([string]::IsNullOrWhiteSpace($FilePath) -or -not (Test-Path -LiteralPath $FilePath)) {
+        if ($env:EXCEL_TEST_PATH -and (Test-Path -LiteralPath $env:EXCEL_TEST_PATH)) {
+            $FilePath = $env:EXCEL_TEST_PATH
+        } else {
+            # Search workspace or Downloads for any recent xlsx file
+            $found = Get-ChildItem -Path "." -Filter "*.xlsx" -Recurse -Depth 2 -ErrorAction SilentlyContinue | Select-Object -First 1
+            if (-not $found) {
+                $found = Get-ChildItem -Path "$env:USERPROFILE\Downloads" -Filter "*.xlsx" -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1
+            }
+            if ($found) { $FilePath = $found.FullName }
+        }
+    }
+
+    if ([string]::IsNullOrWhiteSpace($FilePath) -or -not (Test-Path -LiteralPath $FilePath)) {
+        throw "Excel file not found. Please provide a valid file_path."
+    }
+
+    $fullPath = (Get-Item $FilePath).FullName
+    $zip = [System.IO.Compression.ZipFile]::OpenRead($fullPath)
+
+    try {
+        # 1. Read shared strings
+        $sharedStrings = New-Object System.Collections.Generic.List[string]
+        $ssEntry = $zip.GetEntry("xl/sharedStrings.xml")
+        if ($ssEntry) {
+            $stream = $ssEntry.Open()
+            $reader = New-Object System.IO.StreamReader($stream, [System.Text.Encoding]::UTF8)
+            $ssXml = [xml]$reader.ReadToEnd()
+            $reader.Close()
+            $stream.Close()
+            foreach ($si in $ssXml.sst.si) {
+                $sharedStrings.Add($si.InnerText)
+            }
+        }
+
+        # 2. Determine target sheet entry from workbook.xml
+        $targetSheetEntryName = "xl/worksheets/sheet1.xml"
+        $wbEntry = $zip.GetEntry("xl/workbook.xml")
+        if ($wbEntry) {
+            $stream = $wbEntry.Open()
+            $reader = New-Object System.IO.StreamReader($stream, [System.Text.Encoding]::UTF8)
+            $wbXml = [xml]$reader.ReadToEnd()
+            $reader.Close()
+            $stream.Close()
+
+            $sheets = $wbXml.workbook.sheets.sheet
+            $matchedSheetIndex = 1
+            if ($SheetName -and $sheets) {
+                $idx = 1
+                foreach ($s in $sheets) {
+                    if ($s.name -eq $SheetName -or $s.name -like "*$SheetName*") {
+                        $matchedSheetIndex = $idx
+                        break
+                    }
+                    $idx++
+                }
+            }
+            $targetSheetEntryName = "xl/worksheets/sheet$matchedSheetIndex.xml"
+        }
+
+        # Fallback if entry doesn't exist
+        $sEntry = $zip.GetEntry($targetSheetEntryName)
+        if (-not $sEntry) {
+            $firstSheet = $zip.Entries | Where-Object { $_.FullName -like "xl/worksheets/sheet*.xml" } | Select-Object -First 1
+            if ($firstSheet) { $sEntry = $firstSheet }
+        }
+
+        if (-not $sEntry) {
+            throw "No worksheet found inside Excel archive."
+        }
+
+        $stream = $sEntry.Open()
+        $reader = New-Object System.IO.StreamReader($stream, [System.Text.Encoding]::UTF8)
+        $sXml = [xml]$reader.ReadToEnd()
+        $reader.Close()
+        $stream.Close()
+
+        $rows = $sXml.worksheet.sheetData.row
+        $currentModule = "General"
+        $testcases = @()
+
+        foreach ($r in $rows) {
+            $rowNum = [int]$r.r
+            $cells = @{}
+            foreach ($c in $r.c) {
+                $colLetter = $c.r -replace '\d+'
+                $val = ""
+                if ($c.v) {
+                    $raw = $c.v
+                    if ($c.t -eq "s") {
+                        $val = $sharedStrings[[int]$raw]
+                    } else {
+                        $val = $raw
+                    }
+                }
+                $cells[$colLetter] = $val
+            }
+
+            $colA = if ($cells.ContainsKey("A")) { $cells["A"].Trim() } else { "" }
+            $colB = if ($cells.ContainsKey("B")) { $cells["B"].Trim() } else { "" }
+
+            if ($colB -match "PHAN HE|Phan he|PHÂN HỆ|Phân hệ|MODULE|Module") {
+                $currentModule = $colB.Trim()
+            } elseif ($colA -match "PHAN HE|Phan he|PHÂN HỆ|Phân hệ|MODULE|Module") {
+                $currentModule = $colA.Trim()
+            }
+
+            if ($colA -match '^TC[-_]?\d+') {
+                $testcases += [PSCustomObject]@{
+                    Row = $rowNum
+                    Module = $currentModule
+                    ID = $colA
+                    Name = if ($cells.ContainsKey("B")) { $cells["B"].Trim() } else { "" }
+                    PreCondition = if ($cells.ContainsKey("C")) { $cells["C"].Trim() } else { "" }
+                    Steps = if ($cells.ContainsKey("D")) { $cells["D"].Trim() } else { "" }
+                    DataTest = if ($cells.ContainsKey("E")) { $cells["E"].Trim() } else { "" }
+                    Expected = if ($cells.ContainsKey("F")) { $cells["F"].Trim() } else { "" }
+                    Progress = if ($cells.ContainsKey("G")) { $cells["G"].Trim() } else { "" }
+                    Status = if ($cells.ContainsKey("H")) { $cells["H"].Trim() } else { "" }
+                    QC = if ($cells.ContainsKey("I")) { $cells["I"].Trim() } else { "" }
+                }
+            }
+        }
+        return $testcases
+    } finally {
+        $zip.Dispose()
+    }
+}
+
+# Alias for backwards compatibility
+function Read-EnterpriseSheet {
+    param([string]$FilePath)
+    return Read-ExcelSheet -FilePath $FilePath
+}
+
+# Stdio JSON-RPC Loop
+$inputReader = [System.Console]::In
+while (($line = $inputReader.ReadLine()) -ne $null) {
+    if ([string]::IsNullOrWhiteSpace($line)) { continue }
+    $id = $null
+    try {
+        $req = $line | ConvertFrom-Json
+        $id = $req.id
+        $method = $req.method
+
+        if ($method -eq "initialize") {
+            $resp = @{
+                jsonrpc = "2.0"
+                id = $id
+                result = @{
+                    protocolVersion = "2024-11-05"
+                    capabilities = @{ tools = @{} }
+                    serverInfo = @{ name = "excel-qa-mcp-server"; version = "2.0.1" }
+                }
+            }
+            [System.Console]::Out.WriteLine(($resp | ConvertTo-Json -Compress -Depth 10))
+        } elseif ($method -eq "notifications/initialized") {
+        } elseif ($method -eq "ping") {
+            $resp = @{ jsonrpc = "2.0"; id = $id; result = @{} }
+            [System.Console]::Out.WriteLine(($resp | ConvertTo-Json -Compress))
+        } elseif ($method -eq "tools/list") {
+            $resp = @{
+                jsonrpc = "2.0"
+                id = $id
+                result = @{
+                    tools = @(
+                        @{
+                            name = "read_excel_sheet"
+                            description = "Extract test cases and matrix rows from an Excel worksheet."
+                            inputSchema = @{
+                                type = "object"
+                                properties = @{
+                                    file_path = @{ type = "string"; description = "Path to Excel file (.xlsx)" }
+                                    sheet_name = @{ type = "string"; description = "Worksheet name (defaults to first sheet)" }
+                                }
+                            }
+                        },
+                        @{
+                            name = "read_Enterprise_sheet"
+                            description = "Compatibility alias for read_excel_sheet"
+                            inputSchema = @{
+                                type = "object"
+                                properties = @{
+                                    file_path = @{ type = "string"; description = "Path to Excel file (.xlsx)" }
+                                }
+                            }
+                        }
+                    )
+                }
+            }
+            [System.Console]::Out.WriteLine(($resp | ConvertTo-Json -Compress -Depth 10))
+        } elseif ($method -eq "tools/call") {
+            $toolName = $req.params.name
+            if ($toolName -eq "read_excel_sheet" -or $toolName -eq "read_Enterprise_sheet") {
+                $fPath = $req.params.arguments.file_path
+                $sName = if ($req.params.arguments.sheet_name) { $req.params.arguments.sheet_name } else { "" }
+                $tcs = Read-ExcelSheet -FilePath $fPath -SheetName $sName
+                $textResult = $tcs | ConvertTo-Json -Depth 5
+                $resp = @{
+                    jsonrpc = "2.0"
+                    id = $id
+                    result = @{
+                        content = @(
+                            @{ type = "text"; text = $textResult }
+                        )
+                    }
+                }
+                [System.Console]::Out.WriteLine(($resp | ConvertTo-Json -Compress -Depth 10))
+            } else {
+                $resp = @{
+                    jsonrpc = "2.0"
+                    id = $id
+                    error = @{ code = -32601; message = "Tool not found: $toolName" }
+                }
+                [System.Console]::Out.WriteLine(($resp | ConvertTo-Json -Compress))
+            }
+        } else {
+            if ($null -ne $id) {
+                $resp = @{
+                    jsonrpc = "2.0"
+                    id = $id
+                    error = @{ code = -32601; message = "Method not found: $method" }
+                }
+                [System.Console]::Out.WriteLine(($resp | ConvertTo-Json -Compress))
+            }
+        }
+    } catch {
+        $errMsg = $_.Exception.Message
+        [System.Console]::Error.WriteLine("[MCP Excel Server Error] $errMsg")
+        $errCode = if ($null -eq $id) { -32700 } else { -32603 }
+        $resp = @{
+            jsonrpc = "2.0"
+            id = $id
+            error = @{
+                code = $errCode
+                message = "MCP Server Exception: $errMsg"
+            }
+        }
+        [System.Console]::Out.WriteLine(($resp | ConvertTo-Json -Compress))
+    }
+}