pluginbench 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pluginbench/__init__.py +3 -0
- pluginbench/__main__.py +3 -0
- pluginbench/assessment-charts.html +790 -0
- pluginbench/assessment.py +316 -0
- pluginbench/assessment_charts.py +1225 -0
- pluginbench/assessment_metrics.py +1956 -0
- pluginbench/assessment_models.py +127 -0
- pluginbench/baselines.py +792 -0
- pluginbench/claude.py +374 -0
- pluginbench/cli.py +672 -0
- pluginbench/code_diff.py +207 -0
- pluginbench/codex.py +451 -0
- pluginbench/codex_events.py +585 -0
- pluginbench/collectors.py +591 -0
- pluginbench/compare.py +927 -0
- pluginbench/components.py +129 -0
- pluginbench/coverage_report.py +610 -0
- pluginbench/datasets.py +207 -0
- pluginbench/evaluation.example.yaml +45 -0
- pluginbench/evaluation.py +1651 -0
- pluginbench/evaluation_report.py +1004 -0
- pluginbench/fallow_parse.mjs +101 -0
- pluginbench/harbor_backend.py +2946 -0
- pluginbench/isolation.py +314 -0
- pluginbench/judge.py +683 -0
- pluginbench/language_analysis.py +915 -0
- pluginbench/ledger.py +217 -0
- pluginbench/measurement_profiles.py +616 -0
- pluginbench/native_treatment_runtime.js +221 -0
- pluginbench/native_treatments.py +155 -0
- pluginbench/observer_data/go-analysis-v1/go.mod +10 -0
- pluginbench/observer_data/go-analysis-v1/go.sum +10 -0
- pluginbench/observer_data/go-analysis-v1/main.go +64 -0
- pluginbench/observer_data/oxc-parser-0.126.0/package-lock.json +449 -0
- pluginbench/observer_data/oxc-parser-0.126.0/package.json +5 -0
- pluginbench/parallel.py +101 -0
- pluginbench/processes.py +100 -0
- pluginbench/remeasure.py +992 -0
- pluginbench/run_lock.py +227 -0
- pluginbench/run_status.py +518 -0
- pluginbench/study.py +1858 -0
- pluginbench/study_review.py +924 -0
- pluginbench/submissions.py +486 -0
- pluginbench/task_sources.py +403 -0
- pluginbench/treatments.py +388 -0
- pluginbench/trial_containers.py +94 -0
- pluginbench/vendor/ECHARTS-LICENSE +222 -0
- pluginbench/vendor/ECHARTS-NOTICE +5 -0
- pluginbench/vendor/ZRENDER-LICENSE +29 -0
- pluginbench/vendor/echarts.min.js +48 -0
- pluginbench-0.5.0.dist-info/METADATA +294 -0
- pluginbench-0.5.0.dist-info/RECORD +58 -0
- pluginbench-0.5.0.dist-info/WHEEL +4 -0
- pluginbench-0.5.0.dist-info/entry_points.txt +2 -0
- pluginbench-0.5.0.dist-info/licenses/LICENSE +201 -0
- pluginbench-0.5.0.dist-info/licenses/src/pluginbench/vendor/ECHARTS-LICENSE +222 -0
- pluginbench-0.5.0.dist-info/licenses/src/pluginbench/vendor/ECHARTS-NOTICE +5 -0
- pluginbench-0.5.0.dist-info/licenses/src/pluginbench/vendor/ZRENDER-LICENSE +29 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// Syntax observer for Fallow 3.18.0's locked Oxc parser, not a task dependency.
|
|
2
|
+
// npm's provenance and the Rust crate both identify this exact upstream commit.
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { lstatSync, readFileSync, readdirSync, readlinkSync, statSync, writeFileSync } from 'node:fs';
|
|
5
|
+
import { join, resolve, sep } from 'node:path';
|
|
6
|
+
import { pathToFileURL } from 'node:url';
|
|
7
|
+
|
|
8
|
+
const [cache, snapshot, input, output, expectedDependencyDigest] = process.argv.slice(2);
|
|
9
|
+
const version = '0.126.0';
|
|
10
|
+
const commit = '836fe9d51c11981162461216f37a067fc14441a7';
|
|
11
|
+
const integrity = 'sha512-FktCvLby/mOHyuijZt22+nOt10dS24gGUZE3XwIbUg7Kf4+rer3/5T7RgwzazlNuVsCjPloZ3p8E+4ONT3A8Kw==';
|
|
12
|
+
const hash = bytes => `sha256:${createHash('sha256').update(bytes).digest('hex')}`;
|
|
13
|
+
const lockBytes = readFileSync(join(cache, 'package-lock.json'));
|
|
14
|
+
const lock = JSON.parse(lockBytes);
|
|
15
|
+
const pkg = JSON.parse(readFileSync(join(cache, 'node_modules/oxc-parser/package.json')));
|
|
16
|
+
const lockedParser = lock.packages['node_modules/oxc-parser'];
|
|
17
|
+
if (pkg.version !== version || lockedParser.version !== version || lockedParser.integrity !== integrity) {
|
|
18
|
+
throw new Error('observer requires the pinned oxc-parser 0.126.0 package and npm lock');
|
|
19
|
+
}
|
|
20
|
+
const packageHashes = {};
|
|
21
|
+
const dependencyEntries = [];
|
|
22
|
+
const visit = (directory, relative = '', bindDependencies = true) => {
|
|
23
|
+
for (const name of readdirSync(directory).sort()) {
|
|
24
|
+
const path = join(directory, name);
|
|
25
|
+
const key = relative ? `${relative}/${name}` : name;
|
|
26
|
+
const link = lstatSync(path).isSymbolicLink();
|
|
27
|
+
if (bindDependencies && link) dependencyEntries.push([key, Buffer.from(`link:${readlinkSync(path)}`)]);
|
|
28
|
+
let stat;
|
|
29
|
+
try {
|
|
30
|
+
stat = statSync(path);
|
|
31
|
+
} catch (error) {
|
|
32
|
+
// verify.py records dangling link targets without dereferencing them.
|
|
33
|
+
if (expectedDependencyDigest && link && error.code === 'ENOENT') continue;
|
|
34
|
+
throw error;
|
|
35
|
+
}
|
|
36
|
+
if (stat.isDirectory()) visit(path, key, bindDependencies && !link);
|
|
37
|
+
else if (stat.isFile()) {
|
|
38
|
+
const bytes = readFileSync(path);
|
|
39
|
+
packageHashes[key] = hash(bytes);
|
|
40
|
+
if (bindDependencies && !link) dependencyEntries.push([key, bytes]);
|
|
41
|
+
}
|
|
42
|
+
else throw new Error(`unsupported observer dependency: ${key}`);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
visit(join(cache, 'node_modules'));
|
|
46
|
+
// Match verify.py's dependency_digest: Path-component order, link
|
|
47
|
+
// targets (without following directory links), and NUL-separated file bytes.
|
|
48
|
+
// Sorted recursive traversal follows pathlib.Path order (a/child before a.js).
|
|
49
|
+
const dependencyDigest = createHash('sha256');
|
|
50
|
+
for (const [key, bytes] of dependencyEntries) {
|
|
51
|
+
dependencyDigest.update(`${key}\0`);
|
|
52
|
+
dependencyDigest.update(bytes);
|
|
53
|
+
dependencyDigest.update('\0');
|
|
54
|
+
}
|
|
55
|
+
const observedDependencyDigest = dependencyDigest.digest('hex');
|
|
56
|
+
if (expectedDependencyDigest && observedDependencyDigest !== expectedDependencyDigest) {
|
|
57
|
+
throw new Error('observer dependency cache changed');
|
|
58
|
+
}
|
|
59
|
+
const bindings = Object.entries(lock.packages).filter(([name]) => name.startsWith('node_modules/@oxc-parser/binding-'));
|
|
60
|
+
// npm installs every libc variant on Linux; oxc-parser loads the one for this runtime.
|
|
61
|
+
const libc = process.platform === 'linux' ? (process.report.getReport().header.glibcVersionRuntime ? '-gnu' : '-musl') : '';
|
|
62
|
+
const runtimeBinding = `node_modules/@oxc-parser/binding-${process.platform}-${process.arch}${libc}`;
|
|
63
|
+
const installedBindings = bindings.filter(([name]) => name === runtimeBinding && Object.keys(packageHashes).some(key => key.startsWith(name.slice('node_modules/'.length) + '/')));
|
|
64
|
+
if (installedBindings.length !== 1 || installedBindings[0][1].version !== version || !installedBindings[0][1].integrity) {
|
|
65
|
+
throw new Error('observer requires exactly one pinned native Oxc binding');
|
|
66
|
+
}
|
|
67
|
+
const { parseSync } = await import(pathToFileURL(join(cache, 'node_modules/oxc-parser/src-js/index.js')));
|
|
68
|
+
const discovery = JSON.parse(readFileSync(input));
|
|
69
|
+
if (!Array.isArray(discovery.files) || discovery.file_count !== discovery.files.length || new Set(discovery.files).size !== discovery.files.length) {
|
|
70
|
+
throw new Error('invalid Fallow input inventory');
|
|
71
|
+
}
|
|
72
|
+
const root = resolve(snapshot);
|
|
73
|
+
const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
|
|
74
|
+
const files = discovery.files.map(path => {
|
|
75
|
+
const absolute = resolve(root, path);
|
|
76
|
+
if (!absolute.startsWith(root + sep)) throw new Error(`input escapes snapshot: ${path}`);
|
|
77
|
+
const bytes = readFileSync(absolute);
|
|
78
|
+
let errors;
|
|
79
|
+
try {
|
|
80
|
+
// Defaults match SourceType::from_path and ParseOptions::default; semantic
|
|
81
|
+
// lint is disabled. Reject primary parse errors rather than guessing the
|
|
82
|
+
// producer's token-count-dependent JSX retry result.
|
|
83
|
+
errors = parseSync(path, decoder.decode(bytes)).errors.map(error => error.message);
|
|
84
|
+
} catch (error) {
|
|
85
|
+
errors = [String(error)];
|
|
86
|
+
}
|
|
87
|
+
return { path, source_digest: hash(bytes), status: errors.length ? 'invalid' : 'valid', errors };
|
|
88
|
+
});
|
|
89
|
+
writeFileSync(output, JSON.stringify({
|
|
90
|
+
schema_version: 1,
|
|
91
|
+
parser: 'oxc-parser', version, commit,
|
|
92
|
+
parser_options: 'fallow-3.18.0-default-primary',
|
|
93
|
+
lock_digest: hash(lockBytes),
|
|
94
|
+
lock_source: lockBytes.toString('utf-8'),
|
|
95
|
+
package_integrity: integrity,
|
|
96
|
+
binding: Object.fromEntries(installedBindings),
|
|
97
|
+
dependency_hashes: packageHashes,
|
|
98
|
+
dependency_digest: observedDependencyDigest,
|
|
99
|
+
expected_dependency_digest: expectedDependencyDigest ?? null,
|
|
100
|
+
files,
|
|
101
|
+
}, null, 2) + '\n');
|