fc-data 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datasmith/__init__.py +330 -0
- datasmith/__init__.pyi +194 -0
- datasmith/agents/__init__.py +31 -0
- datasmith/agents/classifiers.py +272 -0
- datasmith/agents/codex.py +25 -0
- datasmith/agents/config.py +108 -0
- datasmith/agents/extractors.py +197 -0
- datasmith/agents/installed/README.md +52 -0
- datasmith/agents/installed/__init__.py +22 -0
- datasmith/agents/installed/base.py +240 -0
- datasmith/agents/installed/claude.py +134 -0
- datasmith/agents/installed/codex.py +91 -0
- datasmith/agents/installed/gemini.py +118 -0
- datasmith/agents/installed/none.py +27 -0
- datasmith/agents/sandbox.py +547 -0
- datasmith/agents/synthesizer.py +439 -0
- datasmith/agents/templates/AGENTS.md.j2 +150 -0
- datasmith/agents/templates/sandbox_verify.py +428 -0
- datasmith/docker/__init__.py +31 -0
- datasmith/docker/context.py +112 -0
- datasmith/docker/images.py +158 -0
- datasmith/docker/publish.py +56 -0
- datasmith/docker/templates/Dockerfile.base +26 -0
- datasmith/docker/templates/Dockerfile.pr +42 -0
- datasmith/docker/templates/Dockerfile.repo +11 -0
- datasmith/docker/templates/docker_build_base.sh +780 -0
- datasmith/docker/templates/docker_build_env.sh +309 -0
- datasmith/docker/templates/docker_build_final.sh +106 -0
- datasmith/docker/templates/docker_build_pkg.sh +99 -0
- datasmith/docker/templates/docker_build_run.sh +124 -0
- datasmith/docker/templates/entrypoint.sh +62 -0
- datasmith/docker/templates/parser.py +1405 -0
- datasmith/docker/templates/profile.sh +199 -0
- datasmith/docker/templates/pytest_runner.py +692 -0
- datasmith/docker/templates/run-tests.sh +197 -0
- datasmith/docker/verifiers.py +131 -0
- datasmith/filters.py +154 -0
- datasmith/github/__init__.py +22 -0
- datasmith/github/client.py +333 -0
- datasmith/github/hooks.py +50 -0
- datasmith/github/links.py +110 -0
- datasmith/github/models.py +206 -0
- datasmith/github/render.py +173 -0
- datasmith/github/search.py +66 -0
- datasmith/github/templates/comment.md.j2 +5 -0
- datasmith/github/templates/final.md.j2 +66 -0
- datasmith/github/templates/issues.md.j2 +21 -0
- datasmith/github/templates/repo.md.j2 +1 -0
- datasmith/preflight.py +162 -0
- datasmith/publish/__init__.py +13 -0
- datasmith/publish/huggingface.py +104 -0
- datasmith/publish/pipeline.py +60 -0
- datasmith/publish/records.py +91 -0
- datasmith/py.typed +1 -0
- datasmith/resolution/__init__.py +14 -0
- datasmith/resolution/blocklist.py +145 -0
- datasmith/resolution/cache.py +120 -0
- datasmith/resolution/constants.py +277 -0
- datasmith/resolution/dependency_resolver.py +174 -0
- datasmith/resolution/git_utils.py +378 -0
- datasmith/resolution/import_analyzer.py +66 -0
- datasmith/resolution/metadata_parser.py +412 -0
- datasmith/resolution/models.py +41 -0
- datasmith/resolution/orchestrator.py +522 -0
- datasmith/resolution/package_filters.py +312 -0
- datasmith/resolution/python_manager.py +110 -0
- datasmith/runners/__init__.py +15 -0
- datasmith/runners/base.py +112 -0
- datasmith/runners/classify_prs.py +48 -0
- datasmith/runners/render_problems.py +113 -0
- datasmith/runners/resolve_packages.py +66 -0
- datasmith/runners/scrape_commits.py +166 -0
- datasmith/runners/scrape_repos.py +44 -0
- datasmith/runners/synthesize_images.py +310 -0
- datasmith/update/__init__.py +5 -0
- datasmith/update/cli.py +169 -0
- datasmith/update/offline.py +173 -0
- datasmith/update/pipeline.py +497 -0
- datasmith/utils/__init__.py +18 -0
- datasmith/utils/core.py +67 -0
- datasmith/utils/db.py +156 -0
- datasmith/utils/tokens.py +65 -0
- fc_data-0.2.0.dist-info/METADATA +441 -0
- fc_data-0.2.0.dist-info/RECORD +87 -0
- fc_data-0.2.0.dist-info/WHEEL +4 -0
- fc_data-0.2.0.dist-info/entry_points.txt +2 -0
- fc_data-0.2.0.dist-info/licenses/LICENSE +28 -0
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# docker_build_env.sh (drop-in replacement)
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
# shellcheck disable=SC1091
|
|
6
|
+
source /etc/profile.d/asv_utils.sh || true
|
|
7
|
+
micromamba activate base
|
|
8
|
+
|
|
9
|
+
# -------------------------- ARG PARSING --------------------------
|
|
10
|
+
ENV_PAYLOAD=""
|
|
11
|
+
|
|
12
|
+
usage() {
|
|
13
|
+
cat >&2 <<'USAGE'
|
|
14
|
+
Usage: $(basename "$0") [--env-payload PATH|-e PATH]
|
|
15
|
+
|
|
16
|
+
Options:
|
|
17
|
+
-e, --env-payload Path to a json file (or "-" for stdin) containing the
|
|
18
|
+
environment payload: either "" or a JSON object with two
|
|
19
|
+
lists of strings, "constraints" and "to_install".
|
|
20
|
+
Example:
|
|
21
|
+
""
|
|
22
|
+
or
|
|
23
|
+
{
|
|
24
|
+
"dependencies": ["pkgA==1.2.3","pkgB==4.5.6"],
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
This script expects an env payload because dependency discovery and
|
|
28
|
+
time-capped pinning are handled upstream.
|
|
29
|
+
USAGE
|
|
30
|
+
exit 2
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
while [[ $# -gt 0 ]]; do
|
|
34
|
+
case "$1" in
|
|
35
|
+
-e|--env-payload)
|
|
36
|
+
[[ $# -ge 2 ]] || usage
|
|
37
|
+
ENV_PAYLOAD="$2"; shift 2;;
|
|
38
|
+
-h|--help) usage;;
|
|
39
|
+
*) echo "Unknown argument: $1" >&2; usage;;
|
|
40
|
+
esac
|
|
41
|
+
done
|
|
42
|
+
|
|
43
|
+
[[ -n "${ENV_PAYLOAD}" ]] || { echo "Error: --env-payload is required." >&2; usage; }
|
|
44
|
+
[[ "${ENV_PAYLOAD}" == "-" || -f "${ENV_PAYLOAD}" ]] || { echo "Error: env payload path not found: ${ENV_PAYLOAD}" >&2; exit 2; }
|
|
45
|
+
|
|
46
|
+
parse_env_payload() {
|
|
47
|
+
local path="$1"
|
|
48
|
+
python - "$path" <<'PY'
|
|
49
|
+
import json
|
|
50
|
+
import os
|
|
51
|
+
import sys
|
|
52
|
+
import tempfile
|
|
53
|
+
|
|
54
|
+
path = sys.argv[1]
|
|
55
|
+
if path == "-":
|
|
56
|
+
raw = sys.stdin.read()
|
|
57
|
+
else:
|
|
58
|
+
with open(path, encoding="utf-8") as f:
|
|
59
|
+
raw = f.read()
|
|
60
|
+
|
|
61
|
+
raw_stripped = raw.strip()
|
|
62
|
+
|
|
63
|
+
dependencies = []
|
|
64
|
+
|
|
65
|
+
# Accept: empty string, JSON object {"dependencies": [...]}, or JSON array [...]
|
|
66
|
+
if raw_stripped == "" or raw_stripped == '""':
|
|
67
|
+
pass
|
|
68
|
+
else:
|
|
69
|
+
try:
|
|
70
|
+
data = json.loads(raw)
|
|
71
|
+
except Exception as e:
|
|
72
|
+
print(f"ERROR: invalid JSON in {path}: {e}", file=sys.stderr)
|
|
73
|
+
sys.exit(3)
|
|
74
|
+
|
|
75
|
+
def take_list(key):
|
|
76
|
+
v = data.get(key, [])
|
|
77
|
+
if v is None:
|
|
78
|
+
v = []
|
|
79
|
+
if not isinstance(v, list):
|
|
80
|
+
print(f"ERROR: '{key}' must be a list of strings", file=sys.stderr)
|
|
81
|
+
sys.exit(3)
|
|
82
|
+
out = []
|
|
83
|
+
for i, item in enumerate(v):
|
|
84
|
+
if not isinstance(item, str):
|
|
85
|
+
print(f"ERROR: '{key}[{i}]' is not a string", file=sys.stderr)
|
|
86
|
+
sys.exit(3)
|
|
87
|
+
s = item.strip()
|
|
88
|
+
if s:
|
|
89
|
+
out.append(s)
|
|
90
|
+
return out
|
|
91
|
+
|
|
92
|
+
if isinstance(data, list):
|
|
93
|
+
# Flat list of dependency strings: ["pkg==1.0", "pkg2==2.0"]
|
|
94
|
+
for i, item in enumerate(data):
|
|
95
|
+
if not isinstance(item, str):
|
|
96
|
+
print(f"ERROR: array element [{i}] is not a string", file=sys.stderr)
|
|
97
|
+
sys.exit(3)
|
|
98
|
+
s = item.strip()
|
|
99
|
+
if s:
|
|
100
|
+
dependencies.append(s)
|
|
101
|
+
elif isinstance(data, dict):
|
|
102
|
+
dependencies = take_list("dependencies")
|
|
103
|
+
else:
|
|
104
|
+
print("ERROR: env payload must be a JSON object, array, or empty string", file=sys.stderr)
|
|
105
|
+
sys.exit(3)
|
|
106
|
+
|
|
107
|
+
# Write temp files
|
|
108
|
+
d_fd, d_path = tempfile.mkstemp(prefix="dependencies-", suffix=".txt")
|
|
109
|
+
with os.fdopen(d_fd, "w", encoding="utf-8") as df:
|
|
110
|
+
for line in dependencies:
|
|
111
|
+
df.write(line.rstrip() + "\n")
|
|
112
|
+
print(d_path)
|
|
113
|
+
PY
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
# Capture the two generated paths safely
|
|
117
|
+
mapfile -t __env_paths < <(parse_env_payload "${ENV_PAYLOAD}")
|
|
118
|
+
if [[ "${#__env_paths[@]}" -ne 1 ]]; then
|
|
119
|
+
echo "Error: failed to parse env payload into two paths." >&2
|
|
120
|
+
exit 3
|
|
121
|
+
fi
|
|
122
|
+
|
|
123
|
+
DEPENDENCIES_PATH="${__env_paths[0]}"
|
|
124
|
+
|
|
125
|
+
# CONSTRAINTS_PATH="${__env_paths[0]}"
|
|
126
|
+
# TO_INSTALL_PATH="${__env_paths[1]}"
|
|
127
|
+
# BANNED_PATH="${__env_paths[2]}"
|
|
128
|
+
|
|
129
|
+
# [[ -f "${CONSTRAINTS_PATH}" ]] || { echo "Error: constraints file missing: ${CONSTRAINTS_PATH}" >&2; exit 3; }
|
|
130
|
+
# [[ -f "${TO_INSTALL_PATH}" ]] || { echo "Error: to-install file missing: ${TO_INSTALL_PATH}" >&2; exit 3; }
|
|
131
|
+
# [[ -f "${BANNED_PATH}" ]] || { echo "Error: banned file missing: ${BANNED_PATH}" >&2; exit 3; }
|
|
132
|
+
|
|
133
|
+
# -------------------------- HELPERS --------------------------
|
|
134
|
+
write_build_vars() {
|
|
135
|
+
local py_versions="$1"
|
|
136
|
+
local import_name="$2"
|
|
137
|
+
|
|
138
|
+
mkdir -p /etc/asv_env
|
|
139
|
+
echo "$py_versions" > /etc/asv_env/py_versions
|
|
140
|
+
echo "$import_name" > /etc/asv_env/import_name
|
|
141
|
+
|
|
142
|
+
cat >/etc/profile.d/asv_build_vars.sh <<EOF
|
|
143
|
+
# Auto-generated during docker_build_env.sh
|
|
144
|
+
export ASV_PY_VERSIONS="${py_versions}"
|
|
145
|
+
export IMPORT_NAME="${import_name}"
|
|
146
|
+
EOF
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
append_install_vars() {
|
|
150
|
+
local extras_all="$1"
|
|
151
|
+
local setuppy_cmd="$2"
|
|
152
|
+
|
|
153
|
+
mkdir -p /etc/asv_env
|
|
154
|
+
printf "%s\n" "$extras_all" > /etc/asv_env/extras_all
|
|
155
|
+
printf "%s\n" "$setuppy_cmd" > /etc/asv_env/setuppy_cmd
|
|
156
|
+
|
|
157
|
+
cat >>/etc/profile.d/asv_build_vars.sh <<EOF
|
|
158
|
+
export ALL_EXTRAS="${extras_all}"
|
|
159
|
+
export SAVED_SETUPPY_CMD="${setuppy_cmd}"
|
|
160
|
+
EOF
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
# -------------------------- ENV DISCOVERY --------------------------
|
|
164
|
+
|
|
165
|
+
URL=$(git remote -v | grep "(fetch)" | awk '{print $2}')
|
|
166
|
+
|
|
167
|
+
# Clone external benchmarks so the ASV config file (asv.*.json) is discoverable
|
|
168
|
+
# below. The pkg stage also clones these in an agent-editable section so the
|
|
169
|
+
# synthesizer can adjust URLs/branches if needed.
|
|
170
|
+
if [[ "$URL" =~ ^(https://)?(www\.)?github\.com/dask/dask(\.git)?$ ]]; then
|
|
171
|
+
git clone --depth 1 https://github.com/dask/dask-benchmarks.git /tmp/repo
|
|
172
|
+
cp -r /tmp/repo/dask/* /workspace/repo/
|
|
173
|
+
rm -rf /tmp/repo
|
|
174
|
+
elif [[ "$URL" =~ ^(https://)?(www\.)?github\.com/dask/distributed(\.git)?$ ]]; then
|
|
175
|
+
git clone --depth 1 https://github.com/dask/dask-benchmarks.git /tmp/repo
|
|
176
|
+
cp -r /tmp/repo/distributed/* /workspace/repo/
|
|
177
|
+
rm -rf /tmp/repo
|
|
178
|
+
elif [[ "$URL" =~ ^(https://)?(www\.)?github\.com/joblib/joblib(\.git)?$ ]]; then
|
|
179
|
+
git clone --depth 1 https://github.com/pierreglaser/joblib_benchmarks.git /tmp/repo
|
|
180
|
+
cp -r /tmp/repo/* /workspace/repo/
|
|
181
|
+
rm -rf /tmp/repo
|
|
182
|
+
elif [[ "$URL" =~ ^(https://)?(www\.)?github\.com/astropy/astropy(\.git)?$ ]]; then
|
|
183
|
+
git clone -b main --depth 1 --single-branch https://github.com/astropy/astropy-benchmarks.git
|
|
184
|
+
fi
|
|
185
|
+
|
|
186
|
+
IMPORT_NAME="$(detect_import_name || true)"
|
|
187
|
+
if [[ -z "$IMPORT_NAME" ]]; then
|
|
188
|
+
echo "WARN: Could not determine import name; the pkg stage will fall back to local detection."
|
|
189
|
+
fi
|
|
190
|
+
|
|
191
|
+
cd_asv_json_dir || { echo "No 'asv.*.json' file found." >&2; exit 1; }
|
|
192
|
+
|
|
193
|
+
CONF_NAME="$(asv_conf_name || true)"
|
|
194
|
+
if [[ -z "${CONF_NAME:-}" ]]; then
|
|
195
|
+
echo "No 'asv.*.json' file found." >&2
|
|
196
|
+
exit 1
|
|
197
|
+
fi
|
|
198
|
+
|
|
199
|
+
# PY_VERSIONS=$(python - <<PY
|
|
200
|
+
# import asv
|
|
201
|
+
# cfg = asv.config.Config.load("$CONF_NAME")
|
|
202
|
+
# cfg.pythons = [v for v in cfg.pythons if tuple(map(int, v.split('.'))) >= (3,7)]
|
|
203
|
+
# print(" ".join(cfg.pythons))
|
|
204
|
+
# PY
|
|
205
|
+
# )
|
|
206
|
+
source /etc/profile.d/asv_build_vars.sh || true
|
|
207
|
+
PY_VERSIONS="${PY_VERSION:-${ASV_PY_VERSIONS:-}}"
|
|
208
|
+
|
|
209
|
+
if [[ -z "$PY_VERSIONS" ]]; then
|
|
210
|
+
echo "No Satisfying PY_VERSIONS found in $CONF_NAME" >&2
|
|
211
|
+
cat "$CONF_NAME" >&2
|
|
212
|
+
exit 1
|
|
213
|
+
fi
|
|
214
|
+
|
|
215
|
+
for version in $PY_VERSIONS; do
|
|
216
|
+
ENV_NAME="asv_${version}"
|
|
217
|
+
if ! micromamba env list | awk '{print $1}' | grep -qx "$ENV_NAME"; then
|
|
218
|
+
echo "Error: Expected micromamba environment '$ENV_NAME' not found."
|
|
219
|
+
exit 1
|
|
220
|
+
fi
|
|
221
|
+
done
|
|
222
|
+
|
|
223
|
+
cd /workspace/repo
|
|
224
|
+
|
|
225
|
+
# Extract commit date and set UV_EXCLUDE_NEWER for time-consistent package resolution
|
|
226
|
+
# COMMIT_DATE=$(git log -1 --format=%cI 2>/dev/null || echo "")
|
|
227
|
+
# if [[ -n "$COMMIT_DATE" ]]; then
|
|
228
|
+
# COMMIT_DATE_BUFFER=$(date -d "$COMMIT_DATE +1 year" -Iseconds)
|
|
229
|
+
# export UV_EXCLUDE_NEWER="$COMMIT_DATE_BUFFER"
|
|
230
|
+
# echo "Using UV_EXCLUDE_NEWER=$UV_EXCLUDE_NEWER for time-capped resolution"
|
|
231
|
+
# fi
|
|
232
|
+
|
|
233
|
+
# write_vars "UV_EXCLUDE_NEWER" "$UV_EXCLUDE_NEWER"
|
|
234
|
+
|
|
235
|
+
# Derive package name (for uninstall sweep)
|
|
236
|
+
if [ -f pyproject.toml ]; then
|
|
237
|
+
PKG_NAME=$(python -c "import tomli; d=tomli.load(open('pyproject.toml','rb')); print(d.get('project',{}).get('name',''))")
|
|
238
|
+
elif [ -f setup.cfg ]; then
|
|
239
|
+
PKG_NAME=$(python -c "from setuptools.config import read_configuration as r;print(r('setup.cfg')['metadata'].get('name',''))")
|
|
240
|
+
elif [ -f setup.py ]; then
|
|
241
|
+
PKG_NAME=$(python setup.py --name 2>/dev/null || true)
|
|
242
|
+
else
|
|
243
|
+
PKG_NAME=""
|
|
244
|
+
fi
|
|
245
|
+
|
|
246
|
+
for v in $PY_VERSIONS; do
|
|
247
|
+
e="asv_$v"
|
|
248
|
+
micromamba env list | awk '{print $1}' | grep -qx "$e" || { echo "missing $e"; exit 1; }
|
|
249
|
+
PYTHON_BIN="/opt/conda/envs/$e/bin/python"
|
|
250
|
+
# Uninstall PUT by uv pip and micromamba to ensure clean re-install
|
|
251
|
+
[ -n "$PKG_NAME" ] && uv pip uninstall --python "$PYTHON_BIN" "$PKG_NAME" || true
|
|
252
|
+
[ -n "$IMPORT_NAME" ] && uv pip uninstall --python "$PYTHON_BIN" "$IMPORT_NAME" || true
|
|
253
|
+
[ -n "$PKG_NAME" ] && micromamba remove -n "$e" -y "$PKG_NAME" || true
|
|
254
|
+
[ -n "$IMPORT_NAME" ] && micromamba remove -n "$e" -y "$IMPORT_NAME" || true
|
|
255
|
+
done
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
export UV_NO_PROGRESS=1
|
|
259
|
+
|
|
260
|
+
uv_install() {
|
|
261
|
+
local python_bin=$1; shift
|
|
262
|
+
# Try with build isolation first; on failure, retry without it.
|
|
263
|
+
uv pip install --python "$python_bin" "$@" || \
|
|
264
|
+
uv pip install --python "$python_bin" --no-build-isolation "$@"
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
UV_OPTS=(--upgrade)
|
|
268
|
+
for version in $PY_VERSIONS; do
|
|
269
|
+
ENV_NAME="asv_${version}"
|
|
270
|
+
PYTHON_BIN="/opt/conda/envs/$ENV_NAME/bin/python"
|
|
271
|
+
echo "Installing dependencies in $ENV_NAME from $DEPENDENCIES_PATH"
|
|
272
|
+
if [ -s "$DEPENDENCIES_PATH" ]; then
|
|
273
|
+
# Ensure build tooling is present inside target env
|
|
274
|
+
uv_install "$PYTHON_BIN" --upgrade pip setuptools wheel >/dev/null 2>&1 || true
|
|
275
|
+
|
|
276
|
+
# If a numpy pin exists, install it first to satisfy legacy build chains
|
|
277
|
+
if grep -Eqi '^\s*numpy(==|>=|<=|~=|>|<|\s|$)' "$DEPENDENCIES_PATH"; then
|
|
278
|
+
# Collect numpy spec lines and install them up-front
|
|
279
|
+
mapfile -t __npys < <(awk 'BEGIN{IGNORECASE=1} $0 ~ /^[[:space:]]*numpy(==|>=|<=|~=|>|<|[[:space:]]|$)/{gsub(/^[[:space:]]+|[[:space:]]+$/,"",$0); print $0}' "$DEPENDENCIES_PATH")
|
|
280
|
+
if (( ${#__npys[@]} > 0 )); then
|
|
281
|
+
uv_install "$PYTHON_BIN" "${__npys[@]}" || true
|
|
282
|
+
fi
|
|
283
|
+
fi
|
|
284
|
+
|
|
285
|
+
# Install the full resolved set (build isolation preferred via uv_install)
|
|
286
|
+
uv_install "$PYTHON_BIN" "${UV_OPTS[@]}" -r "$DEPENDENCIES_PATH"
|
|
287
|
+
fi
|
|
288
|
+
done
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
# -------------------------- PERSIST METADATA --------------------------
|
|
292
|
+
# Persist base variables for downstream stages
|
|
293
|
+
write_build_vars "$PY_VERSIONS" "${IMPORT_NAME:-}"
|
|
294
|
+
|
|
295
|
+
REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
|
|
296
|
+
# Extras detection retained (metadata only)
|
|
297
|
+
ALL_EXTRAS="$(detect_extras --repo-root "$REPO_ROOT" 2>/dev/null | tr -s ' ' | tr ' ' ',')"
|
|
298
|
+
echo $ALL_EXTRAS
|
|
299
|
+
|
|
300
|
+
SETUPPY_CMD="develop"
|
|
301
|
+
append_install_vars "${ALL_EXTRAS}" "${SETUPPY_CMD}"
|
|
302
|
+
|
|
303
|
+
# Snapshot each env's from-history packages
|
|
304
|
+
for version in $PY_VERSIONS; do
|
|
305
|
+
ENV_NAME="asv_${version}"
|
|
306
|
+
micromamba -n "$ENV_NAME" env export --from-history > "/etc/asv_env/installed_packages_${version}"
|
|
307
|
+
done
|
|
308
|
+
|
|
309
|
+
echo "Environment setup complete."
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
show_help() {
|
|
5
|
+
cat <<EOF
|
|
6
|
+
Usage: $(basename "$0") [ASV_BENCHMARKS_FALLBACK]
|
|
7
|
+
|
|
8
|
+
Options:
|
|
9
|
+
ASV_BENCHMARKS_FALLBACK Optional fallback file with pre-computed benchmark
|
|
10
|
+
names (one per line). Used only when live ASV
|
|
11
|
+
discovery fails.
|
|
12
|
+
-h, --help Show this help message and exit
|
|
13
|
+
|
|
14
|
+
Description:
|
|
15
|
+
Finalises the Docker image by activating the conda environment, installing
|
|
16
|
+
runtime dependencies, and discovering ASV benchmarks.
|
|
17
|
+
|
|
18
|
+
Benchmarks are discovered via \`asv run --bench just-discover\` which scans
|
|
19
|
+
the codebase for benchmark classes/functions. The result is written to
|
|
20
|
+
/workspace/repo/asv_benchmarks.txt and exported as ASV_BENCHMARKS in
|
|
21
|
+
/etc/profile.d/asv_build_vars.sh so downstream scripts (run-tests.sh,
|
|
22
|
+
profile.sh) can use them.
|
|
23
|
+
EOF
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
|
27
|
+
show_help
|
|
28
|
+
exit 0
|
|
29
|
+
fi
|
|
30
|
+
|
|
31
|
+
FALLBACK_FILE="${1:-}"
|
|
32
|
+
|
|
33
|
+
cd /workspace/repo || exit 1
|
|
34
|
+
|
|
35
|
+
source /etc/profile.d/asv_utils.sh || true
|
|
36
|
+
source /etc/profile.d/asv_build_vars.sh || true
|
|
37
|
+
cat <<'EOF' >> ~/.bashrc
|
|
38
|
+
source /etc/profile.d/asv_utils.sh || true
|
|
39
|
+
source /etc/profile.d/asv_build_vars.sh || true
|
|
40
|
+
EOF
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
set +u
|
|
44
|
+
micromamba shell init --shell bash
|
|
45
|
+
echo "micromamba activate $ENV_NAME" >> ~/.bashrc
|
|
46
|
+
eval "$(micromamba shell hook --shell=bash)"
|
|
47
|
+
micromamba activate "$ENV_NAME" || true
|
|
48
|
+
set -u
|
|
49
|
+
|
|
50
|
+
micromamba run -n "$ENV_NAME" uv pip install git+https://github.com/formula-code/snapshot-tester.git || true
|
|
51
|
+
micromamba run -n "$ENV_NAME" uv pip install -q --upgrade coverage || true
|
|
52
|
+
|
|
53
|
+
# ── Discover ASV benchmarks ──────────────────────────────────────────
|
|
54
|
+
# Step 1: `asv run --bench just-discover` scans the codebase for benchmark
|
|
55
|
+
# classes/functions and writes benchmarks.json (no benchmarks are
|
|
56
|
+
# actually executed).
|
|
57
|
+
# Step 2: Benchmarks.load() reads benchmarks.json and we dump the names
|
|
58
|
+
# one-per-line to asv_benchmarks.txt.
|
|
59
|
+
echo "[final] Discovering ASV benchmarks from ${CONF_NAME:-<unknown>} ..."
|
|
60
|
+
REAL_CONF="$(realpath "$CONF_NAME")"
|
|
61
|
+
CONF_DIR="$(dirname "$REAL_CONF")"
|
|
62
|
+
|
|
63
|
+
# asv must be run from the directory containing the config
|
|
64
|
+
pushd "$CONF_DIR" > /dev/null
|
|
65
|
+
micromamba run -n "$ENV_NAME" asv machine --yes --config "$REAL_CONF" \
|
|
66
|
+
--machine dockertest --num_cpu 1 --ram 4GB 2>&1 | tail -1 || true
|
|
67
|
+
micromamba run -n "$ENV_NAME" asv run --bench just-discover \
|
|
68
|
+
--config "$REAL_CONF" --python=same --machine=dockertest || true
|
|
69
|
+
popd > /dev/null
|
|
70
|
+
|
|
71
|
+
# Extract benchmark names from the generated benchmarks.json
|
|
72
|
+
python - "$CONF_NAME" /workspace/repo/asv_benchmarks.txt <<'EXTRACT_EOF' || true
|
|
73
|
+
import sys
|
|
74
|
+
from asv.config import Config
|
|
75
|
+
from asv.benchmarks import Benchmarks
|
|
76
|
+
|
|
77
|
+
conf = Config.load(sys.argv[1])
|
|
78
|
+
bm = Benchmarks.load(conf)
|
|
79
|
+
with open(sys.argv[2], "w") as f:
|
|
80
|
+
for b in sorted(bm._all_benchmarks.keys()):
|
|
81
|
+
f.write(b + "\n")
|
|
82
|
+
EXTRACT_EOF
|
|
83
|
+
|
|
84
|
+
# Fall back to the pre-computed file if live discovery produced nothing.
|
|
85
|
+
if [ ! -s /workspace/repo/asv_benchmarks.txt ] && [ -n "$FALLBACK_FILE" ] && [ -s "$FALLBACK_FILE" ]; then
|
|
86
|
+
echo "[final] ASV discovery produced no results; falling back to pre-computed benchmarks."
|
|
87
|
+
cp "$FALLBACK_FILE" /workspace/repo/asv_benchmarks.txt
|
|
88
|
+
fi
|
|
89
|
+
|
|
90
|
+
if [ -s /workspace/repo/asv_benchmarks.txt ]; then
|
|
91
|
+
COUNT=$(wc -l < /workspace/repo/asv_benchmarks.txt)
|
|
92
|
+
echo "[final] Discovered $COUNT ASV benchmarks."
|
|
93
|
+
else
|
|
94
|
+
echo "[final] WARNING: No ASV benchmarks discovered."
|
|
95
|
+
fi
|
|
96
|
+
|
|
97
|
+
# ── Resolve benchmark directory ──────────────────────────────────────
|
|
98
|
+
ABS_BENCHMARK_DIR=$(micromamba run -n "$ENV_NAME" python -c 'import os,asv; from pathlib import Path; cp=Path(os.environ["CONF_NAME"]).resolve(); cfg=asv.config.Config.load(cp); raw=os.path.expandvars(str(cfg.benchmark_dir)); b=Path(raw).expanduser(); b=(cp.parent/b) if not b.is_absolute() else b; print(b.resolve())')
|
|
99
|
+
|
|
100
|
+
# ── Persist to asv_build_vars.sh for downstream scripts ──────────────
|
|
101
|
+
cat >>/etc/profile.d/asv_build_vars.sh <<VARS
|
|
102
|
+
export BENCHMARK_DIR=$ABS_BENCHMARK_DIR
|
|
103
|
+
export ASV_BENCHMARKS=/workspace/repo/asv_benchmarks.txt
|
|
104
|
+
VARS
|
|
105
|
+
|
|
106
|
+
echo "Docker ASV benchmark finalization complete."
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Purpose: Build/install the repo (editable) in one or more ASV micromamba envs, then run health checks.
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
|
|
5
|
+
###### SETUP CODE (NOT TO BE MODIFIED) ######
|
|
6
|
+
source /etc/profile.d/asv_utils.sh || true
|
|
7
|
+
source /etc/profile.d/asv_build_vars.sh || true
|
|
8
|
+
|
|
9
|
+
REPO_ROOT=${ROOT_PATH:-$PWD} # Usually /workspace/repo
|
|
10
|
+
TARGET_VERSIONS="${PY_VERSION:-${ASV_PY_VERSIONS:-}}"
|
|
11
|
+
EXTRAS="${ALL_EXTRAS:+[$ALL_EXTRAS]}"
|
|
12
|
+
|
|
13
|
+
log() { printf "\033[1;34m[build]\033[0m %s\n" "$*"; }
|
|
14
|
+
warn() { printf "\033[1;33m[warn]\033[0m %s\n" "$*" >&2; }
|
|
15
|
+
die() { printf "\033[1;31m[fail]\033[0m %s\n" "$*" >&2; exit 1; }
|
|
16
|
+
|
|
17
|
+
export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-2}"
|
|
18
|
+
export NPY_NUM_BUILD_JOBS="${NPY_NUM_BUILD_JOBS:-2}"
|
|
19
|
+
###### END SETUP CODE ######
|
|
20
|
+
|
|
21
|
+
# =====================================================================
|
|
22
|
+
# Agent guidance
|
|
23
|
+
# =====================================================================
|
|
24
|
+
# GOAL: Install the project in EDITABLE mode into each asv_{version} env.
|
|
25
|
+
#
|
|
26
|
+
# You may:
|
|
27
|
+
# - Add conda/pip dependencies needed to build this project.
|
|
28
|
+
# - Run repo-specific pre-steps (submodules, Cython generation, env vars).
|
|
29
|
+
# - Set CFLAGS/CXXFLAGS/LDFLAGS as needed.
|
|
30
|
+
# - Modify source files if needed (e.g. fix a missing #include).
|
|
31
|
+
# - Modify the external benchmarks section below (URLs, branches, paths).
|
|
32
|
+
#
|
|
33
|
+
# You must:
|
|
34
|
+
# - Keep this script idempotent.
|
|
35
|
+
# - Use editable install: `uv pip install -e .` or `pip install -e .`.
|
|
36
|
+
# - Not modify the SETUP CODE block above.
|
|
37
|
+
|
|
38
|
+
# =====================================================================
|
|
39
|
+
# External benchmarks (agent-editable)
|
|
40
|
+
# =====================================================================
|
|
41
|
+
# Some repos store ASV benchmarks in a separate repository. Clone them
|
|
42
|
+
# here so they are baked into the Docker image and available at runtime.
|
|
43
|
+
# Modify URLs, branches, or destination paths as needed.
|
|
44
|
+
ORIGIN_URL=$(git -C "$REPO_ROOT" remote get-url origin 2>/dev/null || true)
|
|
45
|
+
|
|
46
|
+
if [[ "$ORIGIN_URL" =~ github\.com/dask/dask(\.git)?$ ]]; then
|
|
47
|
+
log "Cloning dask-benchmarks"
|
|
48
|
+
git clone --depth 1 https://github.com/dask/dask-benchmarks.git /tmp/benchmarks
|
|
49
|
+
cp -rf /tmp/benchmarks/dask/* "$REPO_ROOT/"
|
|
50
|
+
rm -rf /tmp/benchmarks
|
|
51
|
+
elif [[ "$ORIGIN_URL" =~ github\.com/dask/distributed(\.git)?$ ]]; then
|
|
52
|
+
log "Cloning dask-benchmarks (distributed)"
|
|
53
|
+
git clone --depth 1 https://github.com/dask/dask-benchmarks.git /tmp/benchmarks
|
|
54
|
+
cp -rf /tmp/benchmarks/distributed/* "$REPO_ROOT/"
|
|
55
|
+
rm -rf /tmp/benchmarks
|
|
56
|
+
elif [[ "$ORIGIN_URL" =~ github\.com/joblib/joblib(\.git)?$ ]]; then
|
|
57
|
+
log "Cloning joblib_benchmarks"
|
|
58
|
+
git clone --depth 1 https://github.com/pierreglaser/joblib_benchmarks.git /tmp/benchmarks
|
|
59
|
+
cp -rf /tmp/benchmarks/* "$REPO_ROOT/"
|
|
60
|
+
rm -rf /tmp/benchmarks
|
|
61
|
+
elif [[ "$ORIGIN_URL" =~ github\.com/astropy/astropy(\.git)?$ ]]; then
|
|
62
|
+
log "Cloning astropy-benchmarks"
|
|
63
|
+
rm -rf "$REPO_ROOT/astropy-benchmarks"
|
|
64
|
+
git clone -b main --depth 1 --single-branch \
|
|
65
|
+
https://github.com/astropy/astropy-benchmarks.git "$REPO_ROOT/astropy-benchmarks"
|
|
66
|
+
fi
|
|
67
|
+
|
|
68
|
+
# =====================================================================
|
|
69
|
+
# Build & install across envs
|
|
70
|
+
# =====================================================================
|
|
71
|
+
for version in $TARGET_VERSIONS; do
|
|
72
|
+
ENV_NAME="asv_${version}"
|
|
73
|
+
log "==> Building in env: $ENV_NAME (python=$version)"
|
|
74
|
+
|
|
75
|
+
IMP="$(detect_import_name || true)"
|
|
76
|
+
log "Using import name: $IMP"
|
|
77
|
+
|
|
78
|
+
# -----------------------------------------------------------------
|
|
79
|
+
# MODEL EDIT AREA: repo-specific tweaks (optional)
|
|
80
|
+
# -----------------------------------------------------------------
|
|
81
|
+
# Examples (uncomment if needed for this repo):
|
|
82
|
+
# git -C "$REPO_ROOT" submodule update --init --recursive
|
|
83
|
+
# micromamba install -y -n "$ENV_NAME" -c conda-forge openblas libopenmp
|
|
84
|
+
# export CFLAGS="${CFLAGS:-} -Wno-error=incompatible-pointer-types"
|
|
85
|
+
# -----------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
log "Editable install with --no-build-isolation"
|
|
88
|
+
if ! PIP_NO_BUILD_ISOLATION=1 micromamba run -n "$ENV_NAME" python -m pip install --no-build-isolation -v -e "$REPO_ROOT"$EXTRAS; then
|
|
89
|
+
warn "Failed with extras, retrying without"
|
|
90
|
+
PIP_NO_BUILD_ISOLATION=1 micromamba run -n "$ENV_NAME" python -m pip install --no-build-isolation -v -e "$REPO_ROOT"
|
|
91
|
+
fi
|
|
92
|
+
|
|
93
|
+
log "Running smoke checks"
|
|
94
|
+
micromamba run -n "$ENV_NAME" asv_smokecheck.py --import-name "$IMP" --repo-root "$REPO_ROOT" ${RUN_PYTEST_SMOKE:+--pytest-smoke}
|
|
95
|
+
|
|
96
|
+
echo "::import_name=${IMP}::env=${ENV_NAME}"
|
|
97
|
+
done
|
|
98
|
+
|
|
99
|
+
log "All builds complete"
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
cd /workspace/repo || exit 1
|
|
5
|
+
|
|
6
|
+
source /etc/profile.d/asv_utils.sh || true
|
|
7
|
+
source /etc/profile.d/asv_build_vars.sh || true
|
|
8
|
+
|
|
9
|
+
ROOT_PATH=${ROOT_PATH:-$PWD} # Usually /workspace/repo
|
|
10
|
+
REPO_ROOT="$ROOT_PATH"
|
|
11
|
+
version=$(echo $ASV_PY_VERSIONS | awk '{print $1}')
|
|
12
|
+
ENV_NAME="asv_${version}"
|
|
13
|
+
CONF_NAME=$(find . -type f -name "asv.*.json" | head -n 1)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
[ -f docker_build_pkg.sh ] && rm docker_build_pkg.sh
|
|
17
|
+
[ -f docker_build_env.sh ] && rm docker_build_env.sh
|
|
18
|
+
[ -f /workspace/docker_build_base.sh ] && rm /workspace/docker_build_base.sh
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# cd to dirname $CONF_NAME
|
|
22
|
+
CWD=$(pwd)
|
|
23
|
+
cd "$(dirname "$CONF_NAME")"
|
|
24
|
+
asv machine --yes --config "$(basename "$CONF_NAME")" \
|
|
25
|
+
--machine "docker" \
|
|
26
|
+
--num_cpu "1" \
|
|
27
|
+
--ram "4GB"
|
|
28
|
+
cd "$CWD"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
lock_repo_to_current_commit() {
|
|
32
|
+
set -euo pipefail
|
|
33
|
+
|
|
34
|
+
# 0) Sanity checks
|
|
35
|
+
git rev-parse --git-dir >/dev/null 2>&1 || { echo "Not a git repo"; return 1; }
|
|
36
|
+
if git worktree list 2>/dev/null | awk 'NR>1{exit 1}'; then :; else
|
|
37
|
+
echo "Multiple worktrees detected. Aborting for strictness."; return 1
|
|
38
|
+
fi
|
|
39
|
+
|
|
40
|
+
BR=$(
|
|
41
|
+
git symbolic-ref --short HEAD 2>/dev/null \
|
|
42
|
+
|| git rev-parse --abbrev-ref origin/HEAD 2>/dev/null | cut -d/ -f2- \
|
|
43
|
+
|| git remote show origin 2>/dev/null | awk '/HEAD branch/ {print $NF}' \
|
|
44
|
+
|| echo main
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# 2) Pin $BR to the current commit and detach from upstream
|
|
48
|
+
HEAD_SHA="$(git rev-parse --verify HEAD)"
|
|
49
|
+
git checkout -B "$BR" "$HEAD_SHA"
|
|
50
|
+
git branch --unset-upstream 2>/dev/null || true
|
|
51
|
+
|
|
52
|
+
# 3) (Optional) Remove ALL remotes — leave commented so ASV can fetch if needed
|
|
53
|
+
# for r in $(git remote); do git remote remove "$r"; done
|
|
54
|
+
|
|
55
|
+
# 4) Delete ALL refs except the current branch tip
|
|
56
|
+
# (branches, tags, remotes, stash, notes, everything)
|
|
57
|
+
while IFS= read -r ref; do
|
|
58
|
+
[[ "$ref" == "refs/heads/$BR" ]] && continue
|
|
59
|
+
git update-ref -d "$ref" || true
|
|
60
|
+
done < <(git for-each-ref --format='%(refname)')
|
|
61
|
+
|
|
62
|
+
# Explicitly nuke stash and notes namespaces (in case they were packed)
|
|
63
|
+
git update-ref -d refs/stash 2>/dev/null || true
|
|
64
|
+
while IFS= read -r nref; do git update-ref -d "$nref" || true; done \
|
|
65
|
+
< <(git for-each-ref --format='%(refname)' refs/notes)
|
|
66
|
+
|
|
67
|
+
# 5) Remove alternates (could reintroduce hidden objects)
|
|
68
|
+
if [[ -f .git/objects/info/alternates ]]; then
|
|
69
|
+
rm -f .git/objects/info/alternates
|
|
70
|
+
fi
|
|
71
|
+
|
|
72
|
+
# 6) Expire ALL reflogs and delete the logs directory for good measure
|
|
73
|
+
git reflog expire --expire=now --expire-unreachable=now --all || true
|
|
74
|
+
rm -rf .git/logs || true
|
|
75
|
+
|
|
76
|
+
# 7) ***Do NOT prune unreachable objects***
|
|
77
|
+
# Keep objects so raw SHAs remain resolvable for ASV.
|
|
78
|
+
# Also disable future auto-pruning.
|
|
79
|
+
git config gc.auto 0
|
|
80
|
+
git config gc.pruneExpire never
|
|
81
|
+
git config gc.worktreePruneExpire never
|
|
82
|
+
# You may still repack reachable objects without pruning unreachable ones:
|
|
83
|
+
git repack -Ad --write-bitmap-index || true
|
|
84
|
+
# DO NOT run: git gc --prune=now --aggressive
|
|
85
|
+
# DO NOT run: git prune --expire=now
|
|
86
|
+
|
|
87
|
+
# 8) Verification: ensure no ref points to a descendant of HEAD
|
|
88
|
+
if while IFS= read -r ref; do
|
|
89
|
+
[[ "$ref" == "refs/heads/$BR" ]] && continue
|
|
90
|
+
if git merge-base --is-ancestor "$HEAD_SHA" "$(git rev-parse "$ref")"; then
|
|
91
|
+
echo "$ref"
|
|
92
|
+
exit 0
|
|
93
|
+
fi
|
|
94
|
+
done < <(git for-each-ref --format='%(refname)') ; then
|
|
95
|
+
: # no output means OK
|
|
96
|
+
else
|
|
97
|
+
echo "Error: some refs still point ahead of HEAD. Aborting."
|
|
98
|
+
return 1
|
|
99
|
+
fi
|
|
100
|
+
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
HEAD_SHA="$(git rev-parse --verify HEAD)"
|
|
104
|
+
|
|
105
|
+
lock_repo_to_current_commit
|
|
106
|
+
|
|
107
|
+
# install some band-aid packages
|
|
108
|
+
micromamba run -n "$ENV_NAME" uv pip install -q --upgrade jinja2 || true
|
|
109
|
+
micromamba run -n "$ENV_NAME" uv pip install -q --upgrade pytest || true
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# Persist env variables for future shells and auto-activate
|
|
113
|
+
cat >>/etc/profile.d/asv_build_vars.sh <<EOF
|
|
114
|
+
export ENV_NAME=$ENV_NAME
|
|
115
|
+
export CONF_NAME=$CONF_NAME
|
|
116
|
+
export ROOT_PATH=$ROOT_PATH
|
|
117
|
+
export REPO_ROOT=$REPO_ROOT
|
|
118
|
+
export ASV_PY_VERSIONS="$ASV_PY_VERSIONS"
|
|
119
|
+
export HEAD_SHA=$HEAD_SHA
|
|
120
|
+
EOF
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
# Remove the setup script so the agent doesn't see it
|
|
124
|
+
rm -- "$0"
|