shipwright-cli 1.10.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -36
- package/completions/_shipwright +212 -32
- package/completions/shipwright.bash +97 -25
- package/docs/strategy/01-market-research.md +619 -0
- package/docs/strategy/02-mission-and-brand.md +587 -0
- package/docs/strategy/03-gtm-and-roadmap.md +759 -0
- package/docs/strategy/QUICK-START.txt +289 -0
- package/docs/strategy/README.md +172 -0
- package/package.json +4 -2
- package/scripts/sw +208 -1
- package/scripts/sw-activity.sh +500 -0
- package/scripts/sw-adaptive.sh +925 -0
- package/scripts/sw-adversarial.sh +1 -1
- package/scripts/sw-architecture-enforcer.sh +1 -1
- package/scripts/sw-auth.sh +613 -0
- package/scripts/sw-autonomous.sh +664 -0
- package/scripts/sw-changelog.sh +704 -0
- package/scripts/sw-checkpoint.sh +1 -1
- package/scripts/sw-ci.sh +602 -0
- package/scripts/sw-cleanup.sh +1 -1
- package/scripts/sw-code-review.sh +637 -0
- package/scripts/sw-connect.sh +1 -1
- package/scripts/sw-context.sh +605 -0
- package/scripts/sw-cost.sh +1 -1
- package/scripts/sw-daemon.sh +432 -130
- package/scripts/sw-dashboard.sh +1 -1
- package/scripts/sw-db.sh +540 -0
- package/scripts/sw-decompose.sh +539 -0
- package/scripts/sw-deps.sh +551 -0
- package/scripts/sw-developer-simulation.sh +1 -1
- package/scripts/sw-discovery.sh +412 -0
- package/scripts/sw-docs-agent.sh +539 -0
- package/scripts/sw-docs.sh +1 -1
- package/scripts/sw-doctor.sh +59 -1
- package/scripts/sw-dora.sh +615 -0
- package/scripts/sw-durable.sh +710 -0
- package/scripts/sw-e2e-orchestrator.sh +535 -0
- package/scripts/sw-eventbus.sh +393 -0
- package/scripts/sw-feedback.sh +471 -0
- package/scripts/sw-fix.sh +1 -1
- package/scripts/sw-fleet-discover.sh +567 -0
- package/scripts/sw-fleet-viz.sh +404 -0
- package/scripts/sw-fleet.sh +8 -1
- package/scripts/sw-github-app.sh +596 -0
- package/scripts/sw-github-checks.sh +1 -1
- package/scripts/sw-github-deploy.sh +1 -1
- package/scripts/sw-github-graphql.sh +1 -1
- package/scripts/sw-guild.sh +569 -0
- package/scripts/sw-heartbeat.sh +1 -1
- package/scripts/sw-hygiene.sh +559 -0
- package/scripts/sw-incident.sh +617 -0
- package/scripts/sw-init.sh +88 -1
- package/scripts/sw-instrument.sh +699 -0
- package/scripts/sw-intelligence.sh +1 -1
- package/scripts/sw-jira.sh +1 -1
- package/scripts/sw-launchd.sh +363 -28
- package/scripts/sw-linear.sh +1 -1
- package/scripts/sw-logs.sh +1 -1
- package/scripts/sw-loop.sh +64 -3
- package/scripts/sw-memory.sh +1 -1
- package/scripts/sw-mission-control.sh +487 -0
- package/scripts/sw-model-router.sh +545 -0
- package/scripts/sw-otel.sh +596 -0
- package/scripts/sw-oversight.sh +689 -0
- package/scripts/sw-pipeline-composer.sh +1 -1
- package/scripts/sw-pipeline-vitals.sh +1 -1
- package/scripts/sw-pipeline.sh +687 -24
- package/scripts/sw-pm.sh +693 -0
- package/scripts/sw-pr-lifecycle.sh +522 -0
- package/scripts/sw-predictive.sh +1 -1
- package/scripts/sw-prep.sh +1 -1
- package/scripts/sw-ps.sh +1 -1
- package/scripts/sw-public-dashboard.sh +798 -0
- package/scripts/sw-quality.sh +595 -0
- package/scripts/sw-reaper.sh +1 -1
- package/scripts/sw-recruit.sh +573 -0
- package/scripts/sw-regression.sh +642 -0
- package/scripts/sw-release-manager.sh +736 -0
- package/scripts/sw-release.sh +706 -0
- package/scripts/sw-remote.sh +1 -1
- package/scripts/sw-replay.sh +520 -0
- package/scripts/sw-retro.sh +691 -0
- package/scripts/sw-scale.sh +444 -0
- package/scripts/sw-security-audit.sh +505 -0
- package/scripts/sw-self-optimize.sh +1 -1
- package/scripts/sw-session.sh +1 -1
- package/scripts/sw-setup.sh +1 -1
- package/scripts/sw-standup.sh +712 -0
- package/scripts/sw-status.sh +1 -1
- package/scripts/sw-strategic.sh +658 -0
- package/scripts/sw-stream.sh +450 -0
- package/scripts/sw-swarm.sh +583 -0
- package/scripts/sw-team-stages.sh +511 -0
- package/scripts/sw-templates.sh +1 -1
- package/scripts/sw-testgen.sh +515 -0
- package/scripts/sw-tmux-pipeline.sh +554 -0
- package/scripts/sw-tmux.sh +1 -1
- package/scripts/sw-trace.sh +485 -0
- package/scripts/sw-tracker-github.sh +188 -0
- package/scripts/sw-tracker-jira.sh +172 -0
- package/scripts/sw-tracker-linear.sh +251 -0
- package/scripts/sw-tracker.sh +117 -2
- package/scripts/sw-triage.sh +603 -0
- package/scripts/sw-upgrade.sh +1 -1
- package/scripts/sw-ux.sh +677 -0
- package/scripts/sw-webhook.sh +627 -0
- package/scripts/sw-widgets.sh +530 -0
- package/scripts/sw-worktree.sh +1 -1
|
@@ -0,0 +1,642 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ╔═══════════════════════════════════════════════════════════════════════════╗
|
|
3
|
+
# ║ shipwright regression — Regression Detection Pipeline ║
|
|
4
|
+
# ║ Captures metrics · Detects regressions · Tracks baselines · Reports ║
|
|
5
|
+
# ╚═══════════════════════════════════════════════════════════════════════════╝
|
|
6
|
+
set -euo pipefail
|
|
7
|
+
trap 'echo "ERROR: $BASH_SOURCE:$LINENO exited with status $?" >&2' ERR
|
|
8
|
+
|
|
9
|
+
VERSION="2.0.0"
|
|
10
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
11
|
+
REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
12
|
+
|
|
13
|
+
# ─── Colors (matches Seth's tmux theme) ─────────────────────────────────────
|
|
14
|
+
CYAN='\033[38;2;0;212;255m' # #00d4ff — primary accent
|
|
15
|
+
PURPLE='\033[38;2;124;58;237m' # #7c3aed — secondary
|
|
16
|
+
BLUE='\033[38;2;0;102;255m' # #0066ff — tertiary
|
|
17
|
+
GREEN='\033[38;2;74;222;128m' # success
|
|
18
|
+
YELLOW='\033[38;2;250;204;21m' # warning
|
|
19
|
+
RED='\033[38;2;248;113;113m' # error
|
|
20
|
+
DIM='\033[2m'
|
|
21
|
+
BOLD='\033[1m'
|
|
22
|
+
RESET='\033[0m'
|
|
23
|
+
|
|
24
|
+
# ─── Cross-platform compatibility ──────────────────────────────────────────
|
|
25
|
+
# shellcheck source=lib/compat.sh
|
|
26
|
+
[[ -f "$SCRIPT_DIR/lib/compat.sh" ]] && source "$SCRIPT_DIR/lib/compat.sh"
|
|
27
|
+
|
|
28
|
+
# ─── Output Helpers ─────────────────────────────────────────────────────────
|
|
29
|
+
info() { echo -e "${CYAN}${BOLD}▸${RESET} $*"; }
|
|
30
|
+
success() { echo -e "${GREEN}${BOLD}✓${RESET} $*"; }
|
|
31
|
+
warn() { echo -e "${YELLOW}${BOLD}⚠${RESET} $*"; }
|
|
32
|
+
error() { echo -e "${RED}${BOLD}✗${RESET} $*" >&2; }
|
|
33
|
+
|
|
34
|
+
now_iso() { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
|
|
35
|
+
now_epoch() { date +%s; }
|
|
36
|
+
|
|
37
|
+
format_duration() {
|
|
38
|
+
local secs="$1"
|
|
39
|
+
if [[ "$secs" -ge 3600 ]]; then
|
|
40
|
+
printf "%dh %dm %ds" $((secs/3600)) $((secs%3600/60)) $((secs%60))
|
|
41
|
+
elif [[ "$secs" -ge 60 ]]; then
|
|
42
|
+
printf "%dm %ds" $((secs/60)) $((secs%60))
|
|
43
|
+
else
|
|
44
|
+
printf "%ds" "$secs"
|
|
45
|
+
fi
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
# ─── Structured Event Log ──────────────────────────────────────────────────
|
|
49
|
+
EVENTS_FILE="${HOME}/.shipwright/events.jsonl"
|
|
50
|
+
|
|
51
|
+
emit_event() {
|
|
52
|
+
local event_type="$1"
|
|
53
|
+
shift
|
|
54
|
+
local json_fields=""
|
|
55
|
+
for kv in "$@"; do
|
|
56
|
+
local key="${kv%%=*}"
|
|
57
|
+
local val="${kv#*=}"
|
|
58
|
+
if [[ "$val" =~ ^-?[0-9]+\.?[0-9]*$ ]]; then
|
|
59
|
+
json_fields="${json_fields},\"${key}\":${val}"
|
|
60
|
+
else
|
|
61
|
+
val="${val//\"/\\\"}"
|
|
62
|
+
json_fields="${json_fields},\"${key}\":\"${val}\""
|
|
63
|
+
fi
|
|
64
|
+
done
|
|
65
|
+
mkdir -p "${HOME}/.shipwright"
|
|
66
|
+
echo "{\"ts\":\"$(now_iso)\",\"ts_epoch\":$(now_epoch),\"type\":\"${event_type}\"${json_fields}}" >> "$EVENTS_FILE"
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
# ─── Regression Storage ────────────────────────────────────────────────────
|
|
70
|
+
BASELINES_DIR="${HOME}/.shipwright/baselines"
|
|
71
|
+
LATEST_BASELINE="${BASELINES_DIR}/latest.json"
|
|
72
|
+
THRESHOLDS_FILE="${HOME}/.shipwright/regression-thresholds.json"
|
|
73
|
+
|
|
74
|
+
ensure_baseline_dir() {
|
|
75
|
+
mkdir -p "$BASELINES_DIR"
|
|
76
|
+
if [[ ! -f "$THRESHOLDS_FILE" ]]; then
|
|
77
|
+
cat > "$THRESHOLDS_FILE" <<'THRESHOLDS'
|
|
78
|
+
{
|
|
79
|
+
"test_count_decrease": true,
|
|
80
|
+
"pass_rate_drop": 5.0,
|
|
81
|
+
"line_count_increase": 20.0,
|
|
82
|
+
"syntax_errors": true,
|
|
83
|
+
"function_count_decrease": true
|
|
84
|
+
}
|
|
85
|
+
THRESHOLDS
|
|
86
|
+
fi
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
# ─── Metric Collection ─────────────────────────────────────────────────────
|
|
90
|
+
|
|
91
|
+
# Collect current test count and pass rate from all test suites
|
|
92
|
+
collect_test_metrics() {
|
|
93
|
+
local test_count=0
|
|
94
|
+
local pass_count=0
|
|
95
|
+
local fail_count=0
|
|
96
|
+
|
|
97
|
+
# Check for test output files and parse them
|
|
98
|
+
if [[ -f "$REPO_DIR/.claude/pipeline-artifacts/test-results.json" ]]; then
|
|
99
|
+
pass_count=$(jq -r '.summary.passed // 0' "$REPO_DIR/.claude/pipeline-artifacts/test-results.json" 2>/dev/null || echo "0")
|
|
100
|
+
fail_count=$(jq -r '.summary.failed // 0' "$REPO_DIR/.claude/pipeline-artifacts/test-results.json" 2>/dev/null || echo "0")
|
|
101
|
+
test_count=$((pass_count + fail_count))
|
|
102
|
+
fi
|
|
103
|
+
|
|
104
|
+
# Fallback: run tests and capture output
|
|
105
|
+
if [[ "$test_count" -eq 0 ]]; then
|
|
106
|
+
if [[ -f "$REPO_DIR/scripts/sw-pipeline-test.sh" ]]; then
|
|
107
|
+
local test_output
|
|
108
|
+
test_output=$("$REPO_DIR/scripts/sw-pipeline-test.sh" 2>&1 || true)
|
|
109
|
+
pass_count=$(echo "$test_output" | grep -c "^✓ " || true)
|
|
110
|
+
fail_count=$(echo "$test_output" | grep -c "^✗ " || true)
|
|
111
|
+
test_count=$((pass_count + fail_count))
|
|
112
|
+
fi
|
|
113
|
+
fi
|
|
114
|
+
|
|
115
|
+
local pass_rate=0
|
|
116
|
+
if [[ "$test_count" -gt 0 ]]; then
|
|
117
|
+
pass_rate=$(awk "BEGIN { printf \"%.1f\", ($pass_count / $test_count) * 100 }")
|
|
118
|
+
fi
|
|
119
|
+
|
|
120
|
+
echo "$test_count"
|
|
121
|
+
echo "$pass_count"
|
|
122
|
+
echo "$fail_count"
|
|
123
|
+
echo "$pass_rate"
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
# Collect script metrics: count, total lines, function count
|
|
127
|
+
collect_script_metrics() {
|
|
128
|
+
local script_count=0
|
|
129
|
+
local total_lines=0
|
|
130
|
+
local function_count=0
|
|
131
|
+
local syntax_errors=0
|
|
132
|
+
|
|
133
|
+
# Count .sh files
|
|
134
|
+
script_count=$(find "$REPO_DIR/scripts" -maxdepth 1 -name "*.sh" -type f 2>/dev/null | wc -l)
|
|
135
|
+
|
|
136
|
+
# Count total lines in all scripts
|
|
137
|
+
total_lines=$(find "$REPO_DIR/scripts" -maxdepth 1 -name "*.sh" -type f 2>/dev/null -exec wc -l {} + | awk '{sum+=$1} END {print sum}')
|
|
138
|
+
|
|
139
|
+
# Count functions (grep for function definitions)
|
|
140
|
+
function_count=$(find "$REPO_DIR/scripts" -maxdepth 1 -name "*.sh" -type f 2>/dev/null -exec grep -h "^[a-z_][a-z0-9_]*() {" {} + 2>/dev/null | wc -l)
|
|
141
|
+
|
|
142
|
+
# Check for syntax errors
|
|
143
|
+
while IFS= read -r script; do
|
|
144
|
+
if ! bash -n "$script" 2>/dev/null; then
|
|
145
|
+
((syntax_errors++))
|
|
146
|
+
fi
|
|
147
|
+
done < <(find "$REPO_DIR/scripts" -maxdepth 1 -name "*.sh" -type f 2>/dev/null)
|
|
148
|
+
|
|
149
|
+
echo "$script_count"
|
|
150
|
+
echo "$total_lines"
|
|
151
|
+
echo "$function_count"
|
|
152
|
+
echo "$syntax_errors"
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
# Collect all current metrics into a baseline object
|
|
156
|
+
collect_all_metrics() {
|
|
157
|
+
local test_data
|
|
158
|
+
test_data=$(collect_test_metrics)
|
|
159
|
+
local test_count=$(echo "$test_data" | sed -n '1p')
|
|
160
|
+
local pass_count=$(echo "$test_data" | sed -n '2p')
|
|
161
|
+
local fail_count=$(echo "$test_data" | sed -n '3p')
|
|
162
|
+
local pass_rate=$(echo "$test_data" | sed -n '4p')
|
|
163
|
+
|
|
164
|
+
local script_data
|
|
165
|
+
script_data=$(collect_script_metrics)
|
|
166
|
+
local script_count=$(echo "$script_data" | sed -n '1p')
|
|
167
|
+
local total_lines=$(echo "$script_data" | sed -n '2p')
|
|
168
|
+
local function_count=$(echo "$script_data" | sed -n '3p')
|
|
169
|
+
local syntax_errors=$(echo "$script_data" | sed -n '4p')
|
|
170
|
+
|
|
171
|
+
cat <<METRICS
|
|
172
|
+
{
|
|
173
|
+
"timestamp": "$(now_iso)",
|
|
174
|
+
"epoch": $(now_epoch),
|
|
175
|
+
"test_count": $test_count,
|
|
176
|
+
"pass_count": $pass_count,
|
|
177
|
+
"fail_count": $fail_count,
|
|
178
|
+
"pass_rate": $pass_rate,
|
|
179
|
+
"script_count": $script_count,
|
|
180
|
+
"total_lines": $total_lines,
|
|
181
|
+
"function_count": $function_count,
|
|
182
|
+
"syntax_errors": $syntax_errors
|
|
183
|
+
}
|
|
184
|
+
METRICS
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
# ─── Baseline Commands ────────────────────────────────────────────────────
|
|
188
|
+
|
|
189
|
+
cmd_baseline() {
|
|
190
|
+
local save_flag="${1:-}"
|
|
191
|
+
|
|
192
|
+
ensure_baseline_dir
|
|
193
|
+
|
|
194
|
+
info "Collecting current metrics..."
|
|
195
|
+
local metrics
|
|
196
|
+
metrics=$(collect_all_metrics)
|
|
197
|
+
|
|
198
|
+
local timestamp
|
|
199
|
+
timestamp=$(echo "$metrics" | jq -r '.timestamp')
|
|
200
|
+
local epoch
|
|
201
|
+
epoch=$(echo "$metrics" | jq -r '.epoch')
|
|
202
|
+
|
|
203
|
+
# Create timestamped baseline file
|
|
204
|
+
local baseline_file
|
|
205
|
+
baseline_file="${BASELINES_DIR}/baseline-$(echo "$timestamp" | sed 's/[:T-]//g' | sed 's/Z$//').json"
|
|
206
|
+
|
|
207
|
+
local tmp_file
|
|
208
|
+
tmp_file=$(mktemp "${baseline_file}.tmp.XXXXXX")
|
|
209
|
+
|
|
210
|
+
echo "$metrics" > "$tmp_file"
|
|
211
|
+
mv "$tmp_file" "$baseline_file"
|
|
212
|
+
|
|
213
|
+
# Update latest symlink
|
|
214
|
+
rm -f "$LATEST_BASELINE"
|
|
215
|
+
ln -s "$(basename "$baseline_file")" "$LATEST_BASELINE"
|
|
216
|
+
|
|
217
|
+
emit_event "regression.baseline" \
|
|
218
|
+
"timestamp=${timestamp}" \
|
|
219
|
+
"test_count=$(echo "$metrics" | jq -r '.test_count')" \
|
|
220
|
+
"pass_rate=$(echo "$metrics" | jq -r '.pass_rate')" \
|
|
221
|
+
"script_count=$(echo "$metrics" | jq -r '.script_count')" \
|
|
222
|
+
"total_lines=$(echo "$metrics" | jq -r '.total_lines')"
|
|
223
|
+
|
|
224
|
+
success "Baseline saved: $baseline_file"
|
|
225
|
+
|
|
226
|
+
if [[ "$save_flag" == "--save" ]]; then
|
|
227
|
+
success "Baseline committed as reference point"
|
|
228
|
+
fi
|
|
229
|
+
|
|
230
|
+
# Print summary
|
|
231
|
+
echo ""
|
|
232
|
+
echo -e "${CYAN}${BOLD}Metrics${RESET}"
|
|
233
|
+
echo " Test Count: $(echo "$metrics" | jq -r '.test_count')"
|
|
234
|
+
echo " Pass Rate: $(echo "$metrics" | jq -r '.pass_rate')%"
|
|
235
|
+
echo " Scripts: $(echo "$metrics" | jq -r '.script_count')"
|
|
236
|
+
echo " Lines: $(echo "$metrics" | jq -r '.total_lines')"
|
|
237
|
+
echo " Functions: $(echo "$metrics" | jq -r '.function_count')"
|
|
238
|
+
echo " Syntax Err: $(echo "$metrics" | jq -r '.syntax_errors')"
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
# ─── Regression Check ─────────────────────────────────────────────────────
|
|
242
|
+
|
|
243
|
+
# Compare current metrics against baseline with configured thresholds
|
|
244
|
+
cmd_check() {
|
|
245
|
+
ensure_baseline_dir
|
|
246
|
+
|
|
247
|
+
if [[ ! -f "$LATEST_BASELINE" ]]; then
|
|
248
|
+
error "No baseline found. Run 'shipwright regression baseline' first."
|
|
249
|
+
exit 1
|
|
250
|
+
fi
|
|
251
|
+
|
|
252
|
+
info "Comparing current metrics against baseline..."
|
|
253
|
+
|
|
254
|
+
# Resolve symlink to actual file
|
|
255
|
+
local baseline_file
|
|
256
|
+
baseline_file="$BASELINES_DIR/$(basename "$(readlink "$LATEST_BASELINE")")"
|
|
257
|
+
|
|
258
|
+
if [[ ! -f "$baseline_file" ]]; then
|
|
259
|
+
error "Baseline file not found: $baseline_file"
|
|
260
|
+
exit 1
|
|
261
|
+
fi
|
|
262
|
+
|
|
263
|
+
local baseline
|
|
264
|
+
baseline=$(cat "$baseline_file")
|
|
265
|
+
|
|
266
|
+
local current
|
|
267
|
+
current=$(collect_all_metrics)
|
|
268
|
+
|
|
269
|
+
local thresholds
|
|
270
|
+
thresholds=$(cat "$THRESHOLDS_FILE")
|
|
271
|
+
|
|
272
|
+
local regressions=0
|
|
273
|
+
local improvements=0
|
|
274
|
+
|
|
275
|
+
# Helper to compare metrics
|
|
276
|
+
compare_metric() {
|
|
277
|
+
local name="$1"
|
|
278
|
+
local baseline_val="$2"
|
|
279
|
+
local current_val="$3"
|
|
280
|
+
local threshold_key="$4"
|
|
281
|
+
local threshold_val="$5"
|
|
282
|
+
local direction="${6:-decrease}" # decrease or increase
|
|
283
|
+
|
|
284
|
+
local baseline_val_num="${baseline_val//[^0-9.-]/}"
|
|
285
|
+
local current_val_num="${current_val//[^0-9.-]/}"
|
|
286
|
+
|
|
287
|
+
if [[ -z "$baseline_val_num" ]] || [[ -z "$current_val_num" ]]; then
|
|
288
|
+
return
|
|
289
|
+
fi
|
|
290
|
+
|
|
291
|
+
local diff
|
|
292
|
+
local pct_diff=0
|
|
293
|
+
if [[ "$baseline_val_num" != "0" ]]; then
|
|
294
|
+
pct_diff=$(awk "BEGIN { printf \"%.1f\", (($current_val_num - $baseline_val_num) / $baseline_val_num) * 100 }")
|
|
295
|
+
fi
|
|
296
|
+
|
|
297
|
+
if [[ "$direction" == "decrease" ]]; then
|
|
298
|
+
# Metric should not decrease
|
|
299
|
+
if (( $(echo "$current_val_num < $baseline_val_num" | bc -l 2>/dev/null || echo "0") )); then
|
|
300
|
+
echo -e "${RED}✗ $name: $baseline_val_num → $current_val_num (${pct_diff}%)${RESET}"
|
|
301
|
+
((regressions++))
|
|
302
|
+
return 1
|
|
303
|
+
fi
|
|
304
|
+
elif [[ "$direction" == "increase" ]]; then
|
|
305
|
+
# Metric should not increase beyond threshold
|
|
306
|
+
if (( $(echo "$pct_diff > $threshold_val" | bc -l 2>/dev/null || echo "0") )); then
|
|
307
|
+
echo -e "${RED}✗ $name: $baseline_val_num → $current_val_num (+${pct_diff}%)${RESET}"
|
|
308
|
+
((regressions++))
|
|
309
|
+
return 1
|
|
310
|
+
fi
|
|
311
|
+
fi
|
|
312
|
+
|
|
313
|
+
# Improvement
|
|
314
|
+
if [[ "$direction" == "decrease" ]] && (( $(echo "$current_val_num > $baseline_val_num" | bc -l 2>/dev/null || echo "0") )); then
|
|
315
|
+
echo -e "${GREEN}✓ $name: $baseline_val_num → $current_val_num (improved)${RESET}"
|
|
316
|
+
((improvements++))
|
|
317
|
+
elif [[ "$direction" == "increase" ]] && (( $(echo "$current_val_num < $baseline_val_num" | bc -l 2>/dev/null || echo "0") )); then
|
|
318
|
+
echo -e "${GREEN}✓ $name: $baseline_val_num → $current_val_num (improved)${RESET}"
|
|
319
|
+
((improvements++))
|
|
320
|
+
fi
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
echo ""
|
|
324
|
+
info "Regression Analysis"
|
|
325
|
+
echo ""
|
|
326
|
+
|
|
327
|
+
# Test count
|
|
328
|
+
local base_test_count
|
|
329
|
+
base_test_count=$(echo "$baseline" | jq -r '.test_count // 0')
|
|
330
|
+
local curr_test_count
|
|
331
|
+
curr_test_count=$(echo "$current" | jq -r '.test_count // 0')
|
|
332
|
+
compare_metric "Test Count" "$base_test_count" "$curr_test_count" "test_count_decrease" "0" "decrease"
|
|
333
|
+
|
|
334
|
+
# Pass rate
|
|
335
|
+
local base_pass_rate
|
|
336
|
+
base_pass_rate=$(echo "$baseline" | jq -r '.pass_rate // 0')
|
|
337
|
+
local curr_pass_rate
|
|
338
|
+
curr_pass_rate=$(echo "$current" | jq -r '.pass_rate // 0')
|
|
339
|
+
local pass_rate_threshold
|
|
340
|
+
pass_rate_threshold=$(echo "$thresholds" | jq -r '.pass_rate_drop // 5.0')
|
|
341
|
+
local pass_rate_diff
|
|
342
|
+
pass_rate_diff=$(awk "BEGIN { printf \"%.1f\", ($base_pass_rate - $curr_pass_rate) }")
|
|
343
|
+
if (( $(echo "$pass_rate_diff > $pass_rate_threshold" | bc -l 2>/dev/null || echo "0") )); then
|
|
344
|
+
echo -e "${RED}✗ Pass Rate: $base_pass_rate% → $curr_pass_rate% (drop: ${pass_rate_diff}%)${RESET}"
|
|
345
|
+
((regressions++))
|
|
346
|
+
elif (( $(echo "$curr_pass_rate > $base_pass_rate" | bc -l 2>/dev/null || echo "0") )); then
|
|
347
|
+
echo -e "${GREEN}✓ Pass Rate: $base_pass_rate% → $curr_pass_rate%${RESET}"
|
|
348
|
+
((improvements++))
|
|
349
|
+
else
|
|
350
|
+
echo -e "${DIM}= Pass Rate: $base_pass_rate% → $curr_pass_rate%${RESET}"
|
|
351
|
+
fi
|
|
352
|
+
|
|
353
|
+
# Line count (should not increase beyond threshold)
|
|
354
|
+
local base_lines
|
|
355
|
+
base_lines=$(echo "$baseline" | jq -r '.total_lines // 0')
|
|
356
|
+
local curr_lines
|
|
357
|
+
curr_lines=$(echo "$current" | jq -r '.total_lines // 0')
|
|
358
|
+
local line_threshold
|
|
359
|
+
line_threshold=$(echo "$thresholds" | jq -r '.line_count_increase // 20.0')
|
|
360
|
+
compare_metric "Total Lines" "$base_lines" "$curr_lines" "line_count_increase" "$line_threshold" "increase"
|
|
361
|
+
|
|
362
|
+
# Script count
|
|
363
|
+
local base_script_count
|
|
364
|
+
base_script_count=$(echo "$baseline" | jq -r '.script_count // 0')
|
|
365
|
+
local curr_script_count
|
|
366
|
+
curr_script_count=$(echo "$current" | jq -r '.script_count // 0')
|
|
367
|
+
compare_metric "Script Count" "$base_script_count" "$curr_script_count" "" "" "decrease"
|
|
368
|
+
|
|
369
|
+
# Function count
|
|
370
|
+
local base_func_count
|
|
371
|
+
base_func_count=$(echo "$baseline" | jq -r '.function_count // 0')
|
|
372
|
+
local curr_func_count
|
|
373
|
+
curr_func_count=$(echo "$current" | jq -r '.function_count // 0')
|
|
374
|
+
compare_metric "Function Count" "$base_func_count" "$curr_func_count" "" "" "decrease"
|
|
375
|
+
|
|
376
|
+
# Syntax errors
|
|
377
|
+
local base_syntax_errors
|
|
378
|
+
base_syntax_errors=$(echo "$baseline" | jq -r '.syntax_errors // 0')
|
|
379
|
+
local curr_syntax_errors
|
|
380
|
+
curr_syntax_errors=$(echo "$current" | jq -r '.syntax_errors // 0')
|
|
381
|
+
if [[ "$curr_syntax_errors" -gt "$base_syntax_errors" ]]; then
|
|
382
|
+
echo -e "${RED}✗ Syntax Errors: $base_syntax_errors → $curr_syntax_errors${RESET}"
|
|
383
|
+
((regressions++))
|
|
384
|
+
elif [[ "$curr_syntax_errors" -lt "$base_syntax_errors" ]]; then
|
|
385
|
+
echo -e "${GREEN}✓ Syntax Errors: $base_syntax_errors → $curr_syntax_errors${RESET}"
|
|
386
|
+
((improvements++))
|
|
387
|
+
else
|
|
388
|
+
echo -e "${DIM}= Syntax Errors: $base_syntax_errors → $curr_syntax_errors${RESET}"
|
|
389
|
+
fi
|
|
390
|
+
|
|
391
|
+
echo ""
|
|
392
|
+
if [[ "$regressions" -eq 0 ]]; then
|
|
393
|
+
success "No regressions detected"
|
|
394
|
+
emit_event "regression.check" "status=pass" "regressions=0" "improvements=$improvements"
|
|
395
|
+
return 0
|
|
396
|
+
else
|
|
397
|
+
error "$regressions regression(s) detected, $improvements improvement(s)"
|
|
398
|
+
emit_event "regression.check" "status=fail" "regressions=$regressions" "improvements=$improvements"
|
|
399
|
+
return 1
|
|
400
|
+
fi
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
# ─── Report Generation ────────────────────────────────────────────────────
|
|
404
|
+
|
|
405
|
+
cmd_report() {
|
|
406
|
+
local format="${1:-text}"
|
|
407
|
+
|
|
408
|
+
ensure_baseline_dir
|
|
409
|
+
|
|
410
|
+
if [[ ! -f "$LATEST_BASELINE" ]]; then
|
|
411
|
+
error "No baseline found. Run 'shipwright regression baseline' first."
|
|
412
|
+
exit 1
|
|
413
|
+
fi
|
|
414
|
+
|
|
415
|
+
local baseline_file
|
|
416
|
+
baseline_file="$BASELINES_DIR/$(basename "$(readlink "$LATEST_BASELINE")")"
|
|
417
|
+
|
|
418
|
+
if [[ ! -f "$baseline_file" ]]; then
|
|
419
|
+
error "Baseline file not found: $baseline_file"
|
|
420
|
+
exit 1
|
|
421
|
+
fi
|
|
422
|
+
|
|
423
|
+
local baseline
|
|
424
|
+
baseline=$(cat "$baseline_file")
|
|
425
|
+
|
|
426
|
+
local current
|
|
427
|
+
current=$(collect_all_metrics)
|
|
428
|
+
|
|
429
|
+
case "$format" in
|
|
430
|
+
json)
|
|
431
|
+
jq -n \
|
|
432
|
+
--argjson baseline "$baseline" \
|
|
433
|
+
--argjson current "$current" \
|
|
434
|
+
'{baseline: $baseline, current: $current}'
|
|
435
|
+
;;
|
|
436
|
+
markdown|md)
|
|
437
|
+
local baseline_ts
|
|
438
|
+
baseline_ts=$(echo "$baseline" | jq -r '.timestamp')
|
|
439
|
+
local current_ts
|
|
440
|
+
current_ts=$(echo "$current" | jq -r '.timestamp')
|
|
441
|
+
|
|
442
|
+
cat <<REPORT
|
|
443
|
+
# Regression Report
|
|
444
|
+
|
|
445
|
+
Generated: $(date)
|
|
446
|
+
|
|
447
|
+
## Baseline Information
|
|
448
|
+
|
|
449
|
+
- Timestamp: $baseline_ts
|
|
450
|
+
- Test Count: $(echo "$baseline" | jq -r '.test_count')
|
|
451
|
+
- Pass Rate: $(echo "$baseline" | jq -r '.pass_rate')%
|
|
452
|
+
- Scripts: $(echo "$baseline" | jq -r '.script_count')
|
|
453
|
+
- Total Lines: $(echo "$baseline" | jq -r '.total_lines')
|
|
454
|
+
- Functions: $(echo "$baseline" | jq -r '.function_count')
|
|
455
|
+
- Syntax Errors: $(echo "$baseline" | jq -r '.syntax_errors')
|
|
456
|
+
|
|
457
|
+
## Current Metrics
|
|
458
|
+
|
|
459
|
+
- Timestamp: $current_ts
|
|
460
|
+
- Test Count: $(echo "$current" | jq -r '.test_count')
|
|
461
|
+
- Pass Rate: $(echo "$current" | jq -r '.pass_rate')%
|
|
462
|
+
- Scripts: $(echo "$current" | jq -r '.script_count')
|
|
463
|
+
- Total Lines: $(echo "$current" | jq -r '.total_lines')
|
|
464
|
+
- Functions: $(echo "$current" | jq -r '.function_count')
|
|
465
|
+
- Syntax Errors: $(echo "$current" | jq -r '.syntax_errors')
|
|
466
|
+
|
|
467
|
+
## Deltas
|
|
468
|
+
|
|
469
|
+
| Metric | Baseline | Current | Change |
|
|
470
|
+
|--------|----------|---------|--------|
|
|
471
|
+
| Test Count | $(echo "$baseline" | jq -r '.test_count') | $(echo "$current" | jq -r '.test_count') | $(awk "BEGIN {printf \"%+d\", $(echo "$current" | jq -r '.test_count') - $(echo "$baseline" | jq -r '.test_count')}") |
|
|
472
|
+
| Pass Rate | $(echo "$baseline" | jq -r '.pass_rate')% | $(echo "$current" | jq -r '.pass_rate')% | $(awk "BEGIN {printf \"%+.1f%%\", $(echo "$current" | jq -r '.pass_rate') - $(echo "$baseline" | jq -r '.pass_rate')}") |
|
|
473
|
+
| Total Lines | $(echo "$baseline" | jq -r '.total_lines') | $(echo "$current" | jq -r '.total_lines') | $(awk "BEGIN {printf \"%+d\", $(echo "$current" | jq -r '.total_lines') - $(echo "$baseline" | jq -r '.total_lines')}") |
|
|
474
|
+
| Scripts | $(echo "$baseline" | jq -r '.script_count') | $(echo "$current" | jq -r '.script_count') | $(awk "BEGIN {printf \"%+d\", $(echo "$current" | jq -r '.script_count') - $(echo "$baseline" | jq -r '.script_count')}") |
|
|
475
|
+
| Functions | $(echo "$baseline" | jq -r '.function_count') | $(echo "$current" | jq -r '.function_count') | $(awk "BEGIN {printf \"%+d\", $(echo "$current" | jq -r '.function_count') - $(echo "$baseline" | jq -r '.function_count')}") |
|
|
476
|
+
| Syntax Errors | $(echo "$baseline" | jq -r '.syntax_errors') | $(echo "$current" | jq -r '.syntax_errors') | $(awk "BEGIN {printf \"%+d\", $(echo "$current" | jq -r '.syntax_errors') - $(echo "$baseline" | jq -r '.syntax_errors')}") |
|
|
477
|
+
|
|
478
|
+
REPORT
|
|
479
|
+
;;
|
|
480
|
+
*)
|
|
481
|
+
# Default: text format
|
|
482
|
+
info "Regression Report"
|
|
483
|
+
echo ""
|
|
484
|
+
echo -e "${BOLD}Baseline${RESET}"
|
|
485
|
+
echo " Timestamp: $(echo "$baseline" | jq -r '.timestamp')"
|
|
486
|
+
echo " Test Count: $(echo "$baseline" | jq -r '.test_count')"
|
|
487
|
+
echo " Pass Rate: $(echo "$baseline" | jq -r '.pass_rate')%"
|
|
488
|
+
echo " Scripts: $(echo "$baseline" | jq -r '.script_count')"
|
|
489
|
+
echo " Total Lines: $(echo "$baseline" | jq -r '.total_lines')"
|
|
490
|
+
echo " Functions: $(echo "$baseline" | jq -r '.function_count')"
|
|
491
|
+
echo " Syntax Errors: $(echo "$baseline" | jq -r '.syntax_errors')"
|
|
492
|
+
echo ""
|
|
493
|
+
echo -e "${BOLD}Current${RESET}"
|
|
494
|
+
echo " Timestamp: $(echo "$current" | jq -r '.timestamp')"
|
|
495
|
+
echo " Test Count: $(echo "$current" | jq -r '.test_count')"
|
|
496
|
+
echo " Pass Rate: $(echo "$current" | jq -r '.pass_rate')%"
|
|
497
|
+
echo " Scripts: $(echo "$current" | jq -r '.script_count')"
|
|
498
|
+
echo " Total Lines: $(echo "$current" | jq -r '.total_lines')"
|
|
499
|
+
echo " Functions: $(echo "$current" | jq -r '.function_count')"
|
|
500
|
+
echo " Syntax Errors: $(echo "$current" | jq -r '.syntax_errors')"
|
|
501
|
+
;;
|
|
502
|
+
esac
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
# ─── History Command ─────────────────────────────────────────────────────
|
|
506
|
+
|
|
507
|
+
cmd_history() {
|
|
508
|
+
ensure_baseline_dir
|
|
509
|
+
|
|
510
|
+
if [[ ! -d "$BASELINES_DIR" ]] || [[ -z "$(ls -A "$BASELINES_DIR" 2>/dev/null || true)" ]]; then
|
|
511
|
+
warn "No baselines found. Run 'shipwright regression baseline' to create one."
|
|
512
|
+
exit 0
|
|
513
|
+
fi
|
|
514
|
+
|
|
515
|
+
info "Baseline History (last 10)"
|
|
516
|
+
echo ""
|
|
517
|
+
|
|
518
|
+
local count=0
|
|
519
|
+
while IFS= read -r baseline_file; do
|
|
520
|
+
((count++))
|
|
521
|
+
if [[ "$count" -gt 10 ]]; then
|
|
522
|
+
break
|
|
523
|
+
fi
|
|
524
|
+
|
|
525
|
+
local timestamp
|
|
526
|
+
timestamp=$(jq -r '.timestamp' "$baseline_file" 2>/dev/null || echo "unknown")
|
|
527
|
+
local test_count
|
|
528
|
+
test_count=$(jq -r '.test_count // 0' "$baseline_file" 2>/dev/null || echo "0")
|
|
529
|
+
local pass_rate
|
|
530
|
+
pass_rate=$(jq -r '.pass_rate // 0' "$baseline_file" 2>/dev/null || echo "0")
|
|
531
|
+
local lines
|
|
532
|
+
lines=$(jq -r '.total_lines // 0' "$baseline_file" 2>/dev/null || echo "0")
|
|
533
|
+
|
|
534
|
+
local marker=" "
|
|
535
|
+
if [[ "$(basename "$baseline_file")" == "$(basename "$(readlink "$LATEST_BASELINE" 2>/dev/null || echo "")")" ]]; then
|
|
536
|
+
marker="${GREEN}*${RESET}"
|
|
537
|
+
fi
|
|
538
|
+
|
|
539
|
+
printf "%s %-30s Tests: %3d Pass: %5.1f%% Lines: %6d\n" \
|
|
540
|
+
"$marker" "$timestamp" "$test_count" "$pass_rate" "$lines"
|
|
541
|
+
done < <(find "$BASELINES_DIR" -name "baseline-*.json" -type f | sort -rn | head -10)
|
|
542
|
+
|
|
543
|
+
echo ""
|
|
544
|
+
echo -e "${DIM}${CYAN}*${RESET}${DIM} = Latest baseline${RESET}"
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
# ─── Help Command ────────────────────────────────────────────────────────
|
|
548
|
+
|
|
549
|
+
cmd_help() {
|
|
550
|
+
cat <<HELP
|
|
551
|
+
${CYAN}${BOLD}shipwright regression${RESET} — Detect regressions after merge
|
|
552
|
+
|
|
553
|
+
${BOLD}USAGE${RESET}
|
|
554
|
+
shipwright regression <command> [options]
|
|
555
|
+
|
|
556
|
+
${BOLD}COMMANDS${RESET}
|
|
557
|
+
baseline [--save] Capture current metrics as baseline
|
|
558
|
+
check Compare current state against saved baseline (exit 1 if regressions)
|
|
559
|
+
report [--json|--md] Generate detailed regression report
|
|
560
|
+
history Show baseline history (last 10)
|
|
561
|
+
help Show this help
|
|
562
|
+
|
|
563
|
+
${BOLD}METRICS TRACKED${RESET}
|
|
564
|
+
• Test count (must not decrease)
|
|
565
|
+
• Test suite pass rate (must not drop >5% by default)
|
|
566
|
+
• Total script line count (must not increase >20% by default)
|
|
567
|
+
• Script count (must not decrease)
|
|
568
|
+
• Function count (must not decrease)
|
|
569
|
+
• Bash syntax errors (must not increase)
|
|
570
|
+
|
|
571
|
+
${BOLD}BASELINE STORAGE${RESET}
|
|
572
|
+
Baselines stored in: ~/.shipwright/baselines/
|
|
573
|
+
Latest symlink: ~/.shipwright/baselines/latest.json
|
|
574
|
+
Thresholds: ~/.shipwright/regression-thresholds.json
|
|
575
|
+
|
|
576
|
+
${BOLD}EXAMPLES${RESET}
|
|
577
|
+
${DIM}# Capture baseline after successful merge${RESET}
|
|
578
|
+
shipwright regression baseline --save
|
|
579
|
+
|
|
580
|
+
${DIM}# Check for regressions before deploying${RESET}
|
|
581
|
+
shipwright regression check
|
|
582
|
+
|
|
583
|
+
${DIM}# Generate a detailed report${RESET}
|
|
584
|
+
shipwright regression report --markdown
|
|
585
|
+
|
|
586
|
+
${DIM}# View historical baselines${RESET}
|
|
587
|
+
shipwright regression history
|
|
588
|
+
|
|
589
|
+
${BOLD}EXIT CODES${RESET}
|
|
590
|
+
0 No regressions detected
|
|
591
|
+
1 Regressions found or error
|
|
592
|
+
|
|
593
|
+
HELP
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
# ─── Main Router ────────────────────────────────────────────────────────
|
|
597
|
+
|
|
598
|
+
main() {
|
|
599
|
+
local cmd="${1:-help}"
|
|
600
|
+
shift 2>/dev/null || true
|
|
601
|
+
|
|
602
|
+
case "$cmd" in
|
|
603
|
+
baseline)
|
|
604
|
+
cmd_baseline "$@"
|
|
605
|
+
;;
|
|
606
|
+
check)
|
|
607
|
+
cmd_check "$@"
|
|
608
|
+
;;
|
|
609
|
+
report)
|
|
610
|
+
# Handle --json and --markdown flags
|
|
611
|
+
local format="text"
|
|
612
|
+
for arg in "$@"; do
|
|
613
|
+
case "$arg" in
|
|
614
|
+
--json)
|
|
615
|
+
format="json"
|
|
616
|
+
;;
|
|
617
|
+
--markdown|--md)
|
|
618
|
+
format="markdown"
|
|
619
|
+
;;
|
|
620
|
+
esac
|
|
621
|
+
done
|
|
622
|
+
cmd_report "$format"
|
|
623
|
+
;;
|
|
624
|
+
history)
|
|
625
|
+
cmd_history "$@"
|
|
626
|
+
;;
|
|
627
|
+
help|--help|-h)
|
|
628
|
+
cmd_help
|
|
629
|
+
;;
|
|
630
|
+
*)
|
|
631
|
+
error "Unknown command: $cmd"
|
|
632
|
+
echo ""
|
|
633
|
+
cmd_help
|
|
634
|
+
exit 1
|
|
635
|
+
;;
|
|
636
|
+
esac
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
# Guard against being sourced
|
|
640
|
+
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
641
|
+
main "$@"
|
|
642
|
+
fi
|