bugbug 0.0.623__tar.gz → 0.0.650__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bugbug-0.0.650/.gitignore +62 -0
- bugbug-0.0.650/PKG-INFO +66 -0
- bugbug-0.0.650/VERSION +1 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/bug_features.py +17 -27
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/bug_snapshot.py +11 -12
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/bugzilla.py +83 -10
- bugbug-0.0.650/bugbug/build_platforms.py +195 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/code_search/parser.py +6 -2
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/code_search/searchfox_api.py +8 -10
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/code_search/searchfox_data.py +58 -60
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/code_search/searchfox_download.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/commit_features.py +51 -4
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/db.py +9 -10
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/feature_cleanup.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/github.py +2 -2
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/model.py +84 -20
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/__init__.py +8 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/component.py +11 -4
- bugbug-0.0.650/bugbug/models/component_specific.py +136 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/defect.py +6 -12
- bugbug-0.0.650/bugbug/models/perf_regression_predictor.py +425 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/rcatype.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/regressor.py +1 -1
- bugbug-0.0.650/bugbug/models/severity.py +158 -0
- bugbug-0.0.650/bugbug/models/testconfigdimensions.py +663 -0
- bugbug-0.0.650/bugbug/models/testconfigselect.py +834 -0
- bugbug-0.0.650/bugbug/models/testfailure.py +376 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/testselect.py +489 -56
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/tracking.py +1 -3
- bugbug-0.0.650/bugbug/perf/__init__.py +13 -0
- bugbug-0.0.650/bugbug/perf/commit_risk.py +52 -0
- bugbug-0.0.650/bugbug/perf/datasets.py +215 -0
- bugbug-0.0.650/bugbug/perf/extraction.py +322 -0
- bugbug-0.0.650/bugbug/perf/label_policy.py +185 -0
- bugbug-0.0.650/bugbug/perf/push_status.py +138 -0
- bugbug-0.0.650/bugbug/perf/regression_commits.py +188 -0
- bugbug-0.0.650/bugbug/perf/regression_labeling.py +443 -0
- bugbug-0.0.650/bugbug/perf/selection_pairs.py +145 -0
- bugbug-0.0.650/bugbug/perf/sql/perf_alerts.sql +145 -0
- bugbug-0.0.650/bugbug/perf/sql/perf_pushes.sql +64 -0
- bugbug-0.0.650/bugbug/perf/sql/perf_task_costs.sql +64 -0
- bugbug-0.0.650/bugbug/perf/task_costs.py +169 -0
- bugbug-0.0.650/bugbug/perf/task_labels.py +278 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/phabricator.py +10 -13
- bugbug-0.0.650/bugbug/redash.py +113 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/repository.py +111 -11
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/rust_code_analysis_server.py +4 -2
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/swarm.py +2 -2
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/test_scheduling.py +670 -26
- bugbug-0.0.650/bugbug/test_scheduling_features.py +597 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/code_review/agent.py +146 -32
- bugbug-0.0.650/bugbug/tools/code_review/data_types.py +190 -0
- bugbug-0.0.650/bugbug/tools/code_review/langchain_tools.py +557 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/code_review/prompts.py +74 -1
- bugbug-0.0.650/bugbug/tools/code_review/review_context.py +596 -0
- bugbug-0.0.650/bugbug/tools/code_review/review_context_schema.py +622 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/code_review/scorer.py +4 -2
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/code_review/utils.py +2 -2
- bugbug-0.0.650/bugbug/tools/comment_matching/agent.py +69 -0
- bugbug-0.0.650/bugbug/tools/comment_matching/prompts.py +29 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/comment_resolution/agent.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/core/connection.py +7 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/core/exceptions.py +13 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/core/llms.py +2 -2
- bugbug-0.0.650/bugbug/tools/core/platforms/__init__.py +4 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/core/platforms/base.py +42 -14
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/core/platforms/bugzilla.py +3 -3
- bugbug-0.0.650/bugbug/tools/core/platforms/patch_apply.py +91 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/core/platforms/phabricator.py +463 -78
- bugbug-0.0.650/bugbug/tools/core/text.py +91 -0
- bugbug-0.0.650/bugbug/tools/core/validators.py +31 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/patch_summarization/agent.py +5 -2
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/suggestion_filtering/agent.py +8 -5
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/utils.py +123 -44
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/vectordb.py +4 -5
- bugbug-0.0.650/pyproject.toml +187 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/bug_classifier.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/bug_retriever.py +5 -6
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/check_all_metrics.py +5 -4
- bugbug-0.0.650/scripts/ci_failures_matching.py +326 -0
- bugbug-0.0.650/scripts/comment_classifier.py +94 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/commit_classifier.py +7 -21
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/commit_retriever.py +7 -5
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/compatibility_report_classifier.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/gcp_logs_retriever.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/generate_landings_risk_report.py +93 -91
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/github_issue_classifier.py +1 -1
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/github_issue_retriever.py +2 -3
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/inline_comments_data_collection.py +7 -7
- bugbug-0.0.650/scripts/integration_test.sh +75 -0
- bugbug-0.0.650/scripts/perf_alerts_retriever.py +535 -0
- bugbug-0.0.650/scripts/perf_regression_predictor.py +76 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/regressor_finder.py +181 -308
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/retrieve_ci_failures.py +174 -84
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/retrieve_training_metrics.py +1 -1
- bugbug-0.0.650/scripts/retrieve_try_pushes.py +172 -0
- bugbug-0.0.650/scripts/shadow_scheduler_stats.py +493 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/test_scheduling_history_retriever.py +124 -42
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/testing_policy_stats.py +2 -3
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/trainer.py +4 -1
- bugbug-0.0.623/MANIFEST.in +0 -10
- bugbug-0.0.623/PKG-INFO +0 -14
- bugbug-0.0.623/README.md +0 -153
- bugbug-0.0.623/VERSION +0 -1
- bugbug-0.0.623/bugbug/models/testfailure.py +0 -136
- bugbug-0.0.623/bugbug/test_scheduling_features.py +0 -247
- bugbug-0.0.623/bugbug/tools/build_repair/agent.py +0 -559
- bugbug-0.0.623/bugbug/tools/build_repair/config.py +0 -90
- bugbug-0.0.623/bugbug/tools/build_repair/prompts.py +0 -77
- bugbug-0.0.623/bugbug/tools/build_repair/scorer.py +0 -207
- bugbug-0.0.623/bugbug/tools/build_repair/try_server.py +0 -286
- bugbug-0.0.623/bugbug/tools/build_repair/worktree.py +0 -81
- bugbug-0.0.623/bugbug/tools/code_review/data_types.py +0 -39
- bugbug-0.0.623/bugbug/tools/code_review/langchain_tools.py +0 -83
- bugbug-0.0.623/bugbug.egg-info/PKG-INFO +0 -14
- bugbug-0.0.623/bugbug.egg-info/SOURCES.txt +0 -153
- bugbug-0.0.623/bugbug.egg-info/dependency_links.txt +0 -1
- bugbug-0.0.623/bugbug.egg-info/entry_points.txt +0 -22
- bugbug-0.0.623/bugbug.egg-info/requires.txt +0 -54
- bugbug-0.0.623/bugbug.egg-info/top_level.txt +0 -2
- bugbug-0.0.623/extra-nlp-requirements.txt +0 -1
- bugbug-0.0.623/extra-nn-requirements.txt +0 -0
- bugbug-0.0.623/pyproject.toml +0 -11
- bugbug-0.0.623/requirements.txt +0 -49
- bugbug-0.0.623/scripts/__init__.py +0 -0
- bugbug-0.0.623/scripts/build_repair_eval.py +0 -378
- bugbug-0.0.623/scripts/shadow_scheduler_stats.py +0 -356
- bugbug-0.0.623/setup.cfg +0 -4
- bugbug-0.0.623/setup.py +0 -75
- {bugbug-0.0.623 → bugbug-0.0.650}/LICENSE +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/code_search/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/code_search/function_search.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/code_search/mozilla.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/issue_features.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/issue_snapshot.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/annotateignore.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/bug_nobug.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/defect_enhancement_task.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/defect_enhancement_task_e.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/defect_enhancement_task_h.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/defect_enhancement_task_p.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/defect_enhancement_task_s.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/regression_bug_nobug.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/regressionrange.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/str.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels/tracking.csv +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/labels.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/model_calibration.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/accessibility.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/annotate_ignore.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/assignee.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/backout.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/browsername.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/bugtype.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/defect_enhancement_task.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/devdocneeded.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/fenixcomponent.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/fixtime.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/invalid_compatibility_report.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/needsdiagnosis.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/performancebug.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/qaneeded.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/regression.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/regressionrange.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/spambug.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/stepstoreproduce.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/uplift.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/models/worksforme.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/nlp.py +0 -0
- {bugbug-0.0.623/bugbug/tools → bugbug-0.0.650/bugbug/perf/sql}/__init__.py +0 -0
- {bugbug-0.0.623/bugbug/tools/comment_resolution → bugbug-0.0.650/bugbug/tools}/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/base.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/code_review/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/code_review/database.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/code_review/protocols.py +0 -0
- {bugbug-0.0.623/bugbug/tools/core/platforms → bugbug-0.0.650/bugbug/tools/comment_resolution}/__init__.py +0 -0
- {bugbug-0.0.623/bugbug/tools/build_repair → bugbug-0.0.650/bugbug/tools/core}/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/core/data_types.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/patch_summarization/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/patch_summarization/prompts.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/suggestion_filtering/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/bugbug/tools/suggestion_filtering/prompts.py +0 -0
- {bugbug-0.0.623/bugbug/tools/core → bugbug-0.0.650/scripts}/__init__.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/analyze_training_metrics.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/backout_related_test_regressions.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/check.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/comment_level_labeler.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/comment_resolver_runner.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/generate_sheet.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/get_type_labels.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/get_untriaged.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/maintenance_effectiveness_indicator.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/microannotate_generator.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/past_bugs_by_unit.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/redundant_failures.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/review_comments_retriever.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/revision_retriever.py +0 -0
- {bugbug-0.0.623 → bugbug-0.0.650}/scripts/trainer_extract_args.py +0 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
*model
|
|
2
|
+
*model.zst
|
|
3
|
+
*model.zst.etag
|
|
4
|
+
*data_X
|
|
5
|
+
*data_y
|
|
6
|
+
*data_X.zst
|
|
7
|
+
*data_X.zst.etag
|
|
8
|
+
*data_y.zst
|
|
9
|
+
*data_y.zst.etag
|
|
10
|
+
feature_importance.png
|
|
11
|
+
importance.html
|
|
12
|
+
importances.json
|
|
13
|
+
metrics.json
|
|
14
|
+
probs.json
|
|
15
|
+
http_service/models/*model*
|
|
16
|
+
|
|
17
|
+
data/
|
|
18
|
+
sheets/
|
|
19
|
+
|
|
20
|
+
.mypy_cache/
|
|
21
|
+
.pytest_cache/
|
|
22
|
+
*.pyc
|
|
23
|
+
.coverage
|
|
24
|
+
|
|
25
|
+
# Distribution / packaging
|
|
26
|
+
.Python
|
|
27
|
+
build/
|
|
28
|
+
develop-eggs/
|
|
29
|
+
dist/
|
|
30
|
+
downloads/
|
|
31
|
+
eggs/
|
|
32
|
+
.eggs/
|
|
33
|
+
lib/
|
|
34
|
+
lib64/
|
|
35
|
+
parts/
|
|
36
|
+
sdist/
|
|
37
|
+
var/
|
|
38
|
+
wheels/
|
|
39
|
+
pip-wheel-metadata/
|
|
40
|
+
share/python-wheels/
|
|
41
|
+
*.egg-info/
|
|
42
|
+
.installed.cfg
|
|
43
|
+
*.egg
|
|
44
|
+
MANIFEST
|
|
45
|
+
cache/
|
|
46
|
+
node_modules/
|
|
47
|
+
|
|
48
|
+
# Logs
|
|
49
|
+
*.log
|
|
50
|
+
# Desktop Service Store
|
|
51
|
+
*.DS_Store
|
|
52
|
+
|
|
53
|
+
# Local env files
|
|
54
|
+
.env
|
|
55
|
+
/http_service/docker-compose.override.yml
|
|
56
|
+
|
|
57
|
+
# Local files
|
|
58
|
+
*.local.*
|
|
59
|
+
|
|
60
|
+
# Claude Code local state
|
|
61
|
+
.claude/worktrees/
|
|
62
|
+
.claude/.cc-writes/
|
bugbug-0.0.650/PKG-INFO
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: bugbug
|
|
3
|
+
Version: 0.0.650
|
|
4
|
+
Summary: ML tools for Mozilla projects
|
|
5
|
+
Author-email: Marco Castelluccio <mcastelluccio@mozilla.com>
|
|
6
|
+
License-Expression: MPL-2.0
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
|
|
9
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
|
+
Requires-Python: >=3.14
|
|
12
|
+
Requires-Dist: amqp<5.5.0,>=5.3.1
|
|
13
|
+
Requires-Dist: async-lru<2.5,>=2.3
|
|
14
|
+
Requires-Dist: beautifulsoup4<4.16.0,>=4.14.3
|
|
15
|
+
Requires-Dist: boto3<1.44.0,>=1.42.78
|
|
16
|
+
Requires-Dist: httpx~=0.28.1
|
|
17
|
+
Requires-Dist: imbalanced-learn~=0.14.1
|
|
18
|
+
Requires-Dist: langchain-anthropic<1.8.0,>=1.3.4
|
|
19
|
+
Requires-Dist: langchain-classic~=1.0.3
|
|
20
|
+
Requires-Dist: langchain-community~=0.4.1
|
|
21
|
+
Requires-Dist: langchain-google-genai<4.4.0,>=4.2.1
|
|
22
|
+
Requires-Dist: langchain-mistralai~=1.1.2
|
|
23
|
+
Requires-Dist: langchain-openai<1.8.0,>=1.1.14
|
|
24
|
+
Requires-Dist: langchain~=1.3.15
|
|
25
|
+
Requires-Dist: langgraph>=1.2.5
|
|
26
|
+
Requires-Dist: libmozdata>=0.2.20
|
|
27
|
+
Requires-Dist: lmdb<2.4.0,>=1.8.1
|
|
28
|
+
Requires-Dist: lxml-html-clean~=0.4.4
|
|
29
|
+
Requires-Dist: markdown2~=2.5.5
|
|
30
|
+
Requires-Dist: matplotlib<3.12.0,>=3.10.8
|
|
31
|
+
Requires-Dist: mercurial~=7.2
|
|
32
|
+
Requires-Dist: microannotate~=0.0.24
|
|
33
|
+
Requires-Dist: mozci~=2.4.10
|
|
34
|
+
Requires-Dist: numpy~=2.4.3
|
|
35
|
+
Requires-Dist: orjson<3.14.0,>=3.11.7
|
|
36
|
+
Requires-Dist: ortools~=9.15.6755
|
|
37
|
+
Requires-Dist: pandas~=3.0.1
|
|
38
|
+
Requires-Dist: psutil~=7.2.2
|
|
39
|
+
Requires-Dist: pyopenssl>=0.14
|
|
40
|
+
Requires-Dist: python-dateutil~=2.9.0.post0
|
|
41
|
+
Requires-Dist: python-hglib~=2.6.2
|
|
42
|
+
Requires-Dist: qdrant-client~=1.15.1
|
|
43
|
+
Requires-Dist: ratelimit~=2.2.1
|
|
44
|
+
Requires-Dist: requests-html~=0.10.0
|
|
45
|
+
Requires-Dist: requests<2.35,>=2.33
|
|
46
|
+
Requires-Dist: rs-parsepatch~=0.4.6
|
|
47
|
+
Requires-Dist: scikit-learn~=1.7.2
|
|
48
|
+
Requires-Dist: scipy<1.19.0,>=1.17.1
|
|
49
|
+
Requires-Dist: searchfox>=0.20.3
|
|
50
|
+
Requires-Dist: sendgrid~=6.12.5
|
|
51
|
+
Requires-Dist: shap[plots]<0.53,>=0.51
|
|
52
|
+
Requires-Dist: tabulate~=0.10.0
|
|
53
|
+
Requires-Dist: taskcluster~=114.0
|
|
54
|
+
Requires-Dist: tenacity~=9.1.4
|
|
55
|
+
Requires-Dist: tqdm<4.71.0,>=4.67.3
|
|
56
|
+
Requires-Dist: unidiff<1.1.0,>=0.7.5
|
|
57
|
+
Requires-Dist: weave>=0.53.12
|
|
58
|
+
Requires-Dist: whatthepatch~=1.0.7
|
|
59
|
+
Requires-Dist: xgboost<3.5,>=3.2
|
|
60
|
+
Requires-Dist: zstandard~=0.25.0
|
|
61
|
+
Provides-Extra: nlp
|
|
62
|
+
Requires-Dist: spacy==3.8.16; extra == 'nlp'
|
|
63
|
+
Provides-Extra: nn
|
|
64
|
+
Provides-Extra: perf-regression-predictor
|
|
65
|
+
Requires-Dist: torch==2.14.1; extra == 'perf-regression-predictor'
|
|
66
|
+
Requires-Dist: transformers==5.20.1; extra == 'perf-regression-predictor'
|
bugbug-0.0.650/VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.0.650
|
|
@@ -54,7 +54,11 @@ class HasCrashSignature(SingleBugFeature):
|
|
|
54
54
|
|
|
55
55
|
|
|
56
56
|
class Keywords(SingleBugFeature):
|
|
57
|
-
def __init__(self, to_ignore=
|
|
57
|
+
def __init__(self, to_ignore=None, prefixes_to_ignore=None):
|
|
58
|
+
if prefixes_to_ignore is None:
|
|
59
|
+
prefixes_to_ignore = set()
|
|
60
|
+
if to_ignore is None:
|
|
61
|
+
to_ignore = set()
|
|
58
62
|
self.to_ignore = to_ignore
|
|
59
63
|
self.prefixes_to_ignore = prefixes_to_ignore
|
|
60
64
|
|
|
@@ -742,10 +746,7 @@ class IsPerformanceBug(SingleBugFeature):
|
|
|
742
746
|
return True
|
|
743
747
|
|
|
744
748
|
bug_whiteboard = bug["whiteboard"].lower()
|
|
745
|
-
|
|
746
|
-
return True
|
|
747
|
-
|
|
748
|
-
return False
|
|
749
|
+
return any(prefix in bug_whiteboard for prefix in self.whiteboard_prefixes)
|
|
749
750
|
|
|
750
751
|
|
|
751
752
|
class IsMemoryBug(SingleBugFeature):
|
|
@@ -778,10 +779,7 @@ class IsMemoryBug(SingleBugFeature):
|
|
|
778
779
|
return True
|
|
779
780
|
|
|
780
781
|
bug_whiteboard = bug["whiteboard"].lower()
|
|
781
|
-
|
|
782
|
-
return True
|
|
783
|
-
|
|
784
|
-
return False
|
|
782
|
+
return any(prefix in bug_whiteboard for prefix in self.whiteboard_prefixes)
|
|
785
783
|
|
|
786
784
|
|
|
787
785
|
class IsPowerBug(SingleBugFeature):
|
|
@@ -805,10 +803,7 @@ class IsPowerBug(SingleBugFeature):
|
|
|
805
803
|
return True
|
|
806
804
|
|
|
807
805
|
bug_whiteboard = bug["whiteboard"].lower()
|
|
808
|
-
|
|
809
|
-
return True
|
|
810
|
-
|
|
811
|
-
return False
|
|
806
|
+
return any(prefix in bug_whiteboard for prefix in self.whiteboard_prefixes)
|
|
812
807
|
|
|
813
808
|
|
|
814
809
|
class IsSecurityBug(SingleBugFeature):
|
|
@@ -823,6 +818,7 @@ class IsSecurityBug(SingleBugFeature):
|
|
|
823
818
|
self,
|
|
824
819
|
bug: bugzilla.BugDict,
|
|
825
820
|
bug_map: dict[int, bugzilla.BugDict] | None = None,
|
|
821
|
+
**kwargs,
|
|
826
822
|
) -> bool:
|
|
827
823
|
if any(
|
|
828
824
|
keyword.startswith(prefix)
|
|
@@ -832,10 +828,7 @@ class IsSecurityBug(SingleBugFeature):
|
|
|
832
828
|
return True
|
|
833
829
|
|
|
834
830
|
bug_whiteboard = bug["whiteboard"].lower()
|
|
835
|
-
|
|
836
|
-
return True
|
|
837
|
-
|
|
838
|
-
return False
|
|
831
|
+
return any(prefix in bug_whiteboard for prefix in self.whiteboard_prefixes)
|
|
839
832
|
|
|
840
833
|
|
|
841
834
|
class IsCrashBug(SingleBugFeature):
|
|
@@ -849,20 +842,18 @@ class IsCrashBug(SingleBugFeature):
|
|
|
849
842
|
self,
|
|
850
843
|
bug: bugzilla.BugDict,
|
|
851
844
|
bug_map: dict[int, bugzilla.BugDict] | None = None,
|
|
845
|
+
**kwargs,
|
|
852
846
|
) -> bool:
|
|
853
847
|
# Checking for `[@` will exclude some bugs that do not have valid
|
|
854
848
|
# signatures: https://mzl.la/46XAqRF
|
|
855
849
|
if bug.get("cf_crash_signature") and "[@" in bug["cf_crash_signature"]:
|
|
856
850
|
return True
|
|
857
851
|
|
|
858
|
-
|
|
852
|
+
return any(
|
|
859
853
|
keyword.startswith(prefix)
|
|
860
854
|
for keyword in bug["keywords"]
|
|
861
855
|
for prefix in self.keyword_prefixes
|
|
862
|
-
)
|
|
863
|
-
return True
|
|
864
|
-
|
|
865
|
-
return False
|
|
856
|
+
)
|
|
866
857
|
|
|
867
858
|
|
|
868
859
|
class BugTypes(SingleBugFeature):
|
|
@@ -885,13 +876,12 @@ class BugTypes(SingleBugFeature):
|
|
|
885
876
|
"""Infer bug types based on various bug characteristics.
|
|
886
877
|
|
|
887
878
|
Args:
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
of bug IDs to bug dictionaries. Default is None.
|
|
879
|
+
bug: A dictionary containing bug data.
|
|
880
|
+
bug_map: A mapping of bug IDs to bug dictionaries. Default is `None`.
|
|
891
881
|
|
|
892
882
|
Returns:
|
|
893
|
-
|
|
894
|
-
"performance"
|
|
883
|
+
A list of inferred bug types (e.g., `"memory"`, `"power"`,
|
|
884
|
+
`"performance"`, `"security"`, `"crash"`).
|
|
895
885
|
"""
|
|
896
886
|
return [
|
|
897
887
|
is_type.type_name
|
|
@@ -40,13 +40,13 @@ def keyword_mapping(keyword):
|
|
|
40
40
|
"bug-quality": "bmo-bug-quality",
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
return mapping
|
|
43
|
+
return mapping.get(keyword, keyword)
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
def group_mapping(group):
|
|
47
47
|
mapping = {"release-core-security": "core-security-release"}
|
|
48
48
|
|
|
49
|
-
return mapping
|
|
49
|
+
return mapping.get(group, group)
|
|
50
50
|
|
|
51
51
|
|
|
52
52
|
def cf_rank(val):
|
|
@@ -96,7 +96,7 @@ def product(product):
|
|
|
96
96
|
"Browser": "Core",
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
-
return mapping
|
|
99
|
+
return mapping.get(product, product)
|
|
100
100
|
|
|
101
101
|
|
|
102
102
|
def target_milestone(target_milestone):
|
|
@@ -105,9 +105,7 @@ def target_milestone(target_milestone):
|
|
|
105
105
|
|
|
106
106
|
mapping = {"6.2.2": "6.2.2.1", "Firefox 3.7": "Firefox 4.0"}
|
|
107
107
|
|
|
108
|
-
return (
|
|
109
|
-
mapping[target_milestone] if target_milestone in mapping else target_milestone
|
|
110
|
-
)
|
|
108
|
+
return mapping.get(target_milestone, target_milestone)
|
|
111
109
|
|
|
112
110
|
|
|
113
111
|
def null_str(val):
|
|
@@ -539,7 +537,7 @@ def rollback(bug, when=None, do_assert=False):
|
|
|
539
537
|
def assert_or_log(msg):
|
|
540
538
|
msg = f"{msg}, in bug {bug['id']}"
|
|
541
539
|
if do_assert:
|
|
542
|
-
|
|
540
|
+
raise AssertionError(msg)
|
|
543
541
|
else:
|
|
544
542
|
logger.error(msg)
|
|
545
543
|
|
|
@@ -761,11 +759,12 @@ def rollback(bug, when=None, do_assert=False):
|
|
|
761
759
|
|
|
762
760
|
continue
|
|
763
761
|
|
|
764
|
-
if
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
)
|
|
768
|
-
|
|
762
|
+
if (
|
|
763
|
+
change["added"] != "---"
|
|
764
|
+
and field not in bug
|
|
765
|
+
and not is_expected_inconsistent_field(field, last_product, bug["id"])
|
|
766
|
+
):
|
|
767
|
+
assert_or_log(f"{field} is not present")
|
|
769
768
|
|
|
770
769
|
if field in bug and isinstance(bug[field], list):
|
|
771
770
|
if change["added"]:
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import collections
|
|
7
7
|
import csv
|
|
8
|
+
import itertools
|
|
8
9
|
import math
|
|
9
10
|
import re
|
|
10
11
|
from datetime import datetime
|
|
@@ -78,6 +79,7 @@ ATTACHMENT_INCLUDE_FIELDS = [
|
|
|
78
79
|
]
|
|
79
80
|
|
|
80
81
|
COMMENT_INCLUDE_FIELDS = [
|
|
82
|
+
"_collapsed_comments",
|
|
81
83
|
"id",
|
|
82
84
|
"count",
|
|
83
85
|
"text",
|
|
@@ -196,6 +198,28 @@ def get(ids_or_query):
|
|
|
196
198
|
return new_bugs
|
|
197
199
|
|
|
198
200
|
|
|
201
|
+
def get_fields(
|
|
202
|
+
bug_ids: Iterable[int], include_fields: Iterable[str]
|
|
203
|
+
) -> dict[int, BugDict]:
|
|
204
|
+
"""Fetch only ``include_fields`` of the given bugs, keyed by id.
|
|
205
|
+
|
|
206
|
+
Unlike :func:`get`, no comments, attachments or history are requested and
|
|
207
|
+
``BUGS_DB`` is left alone.
|
|
208
|
+
"""
|
|
209
|
+
bugs: dict[int, BugDict] = {}
|
|
210
|
+
ids = sorted({int(bug_id) for bug_id in bug_ids})
|
|
211
|
+
if not ids:
|
|
212
|
+
return bugs
|
|
213
|
+
|
|
214
|
+
def bughandler(bug: BugDict) -> None:
|
|
215
|
+
bugs[int(bug["id"])] = bug
|
|
216
|
+
|
|
217
|
+
Bugzilla(
|
|
218
|
+
ids, include_fields=",".join(include_fields), bughandler=bughandler
|
|
219
|
+
).get_data().wait()
|
|
220
|
+
return bugs
|
|
221
|
+
|
|
222
|
+
|
|
199
223
|
def get_ids_between(date_from, date_to=None, security=False, resolution=None):
|
|
200
224
|
params = {
|
|
201
225
|
"f1": "creation_ts",
|
|
@@ -232,10 +256,7 @@ def download_bugs(bug_ids: Iterable[int], security: bool = False) -> list[BugDic
|
|
|
232
256
|
|
|
233
257
|
new_bug_ids = sorted(list(new_bug_ids_set))
|
|
234
258
|
|
|
235
|
-
chunks = (
|
|
236
|
-
new_bug_ids[i : (i + Bugzilla.BUGZILLA_CHUNK_SIZE)]
|
|
237
|
-
for i in range(0, len(new_bug_ids), Bugzilla.BUGZILLA_CHUNK_SIZE)
|
|
238
|
-
)
|
|
259
|
+
chunks = itertools.batched(new_bug_ids, Bugzilla.BUGZILLA_CHUNK_SIZE, strict=False)
|
|
239
260
|
|
|
240
261
|
@tenacity.retry(
|
|
241
262
|
stop=tenacity.stop_after_attempt(7),
|
|
@@ -411,7 +432,9 @@ def get_product_component_count(months: int = 12) -> dict[str, int]:
|
|
|
411
432
|
return bugs_number
|
|
412
433
|
|
|
413
434
|
|
|
414
|
-
def get_active_product_components(products=
|
|
435
|
+
def get_active_product_components(products=None) -> set[tuple[str, str]]:
|
|
436
|
+
if products is None:
|
|
437
|
+
products = []
|
|
415
438
|
active_components = set()
|
|
416
439
|
|
|
417
440
|
def product_handler(product):
|
|
@@ -469,6 +492,56 @@ def get_groups_users(group_names: list[str]) -> list[str]:
|
|
|
469
492
|
]
|
|
470
493
|
|
|
471
494
|
|
|
495
|
+
def fetch_comments(comment_ids: list[int]) -> dict[int, dict]:
|
|
496
|
+
if not comment_ids:
|
|
497
|
+
return {}
|
|
498
|
+
|
|
499
|
+
response = utils.get_session("bugzilla").get(
|
|
500
|
+
f"https://bugzilla.mozilla.org/rest/bug/comment/{comment_ids[0]}",
|
|
501
|
+
params={"comment_ids": comment_ids[1:]},
|
|
502
|
+
headers={
|
|
503
|
+
"X-Bugzilla-API-Key": Bugzilla.TOKEN,
|
|
504
|
+
"User-Agent": utils.get_user_agent(),
|
|
505
|
+
},
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
if response.status_code == 400 and len(comment_ids) > 1:
|
|
509
|
+
midpoint = len(comment_ids) // 2
|
|
510
|
+
return {
|
|
511
|
+
**fetch_comments(comment_ids[:midpoint]),
|
|
512
|
+
**fetch_comments(comment_ids[midpoint:]),
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
if response.status_code == 400 and response.json().get("code") in (110, 111):
|
|
516
|
+
return {}
|
|
517
|
+
|
|
518
|
+
response.raise_for_status()
|
|
519
|
+
return {
|
|
520
|
+
int(comment_id): comment
|
|
521
|
+
for comment_id, comment in response.json()["comments"].items()
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def get_comments(comment_ids: Iterable[int]) -> dict[int, tuple[BugDict, dict]]:
|
|
526
|
+
normalized_ids = list(dict.fromkeys(map(int, comment_ids)))
|
|
527
|
+
comments = {
|
|
528
|
+
comment_id: comment
|
|
529
|
+
for comment_ids_batch in itertools.batched(normalized_ids, 100, strict=False)
|
|
530
|
+
for comment_id, comment in fetch_comments(list(comment_ids_batch)).items()
|
|
531
|
+
}
|
|
532
|
+
bugs = get({comment["bug_id"] for comment in comments.values()})
|
|
533
|
+
|
|
534
|
+
return {
|
|
535
|
+
comment_id: (bugs[comment["bug_id"]], comment)
|
|
536
|
+
for comment_id, comment in comments.items()
|
|
537
|
+
if comment["bug_id"] in bugs
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def get_comment(comment_id: int) -> dict[int, tuple[BugDict, dict]]:
|
|
542
|
+
return get_comments([comment_id])
|
|
543
|
+
|
|
544
|
+
|
|
472
545
|
def get_revision_ids(bug: BugDict) -> list[int]:
|
|
473
546
|
revision_ids = []
|
|
474
547
|
|
|
@@ -560,8 +633,8 @@ def calculate_maintenance_effectiveness_indicator(
|
|
|
560
633
|
|
|
561
634
|
return params
|
|
562
635
|
|
|
563
|
-
for severity in MAINTENANCE_EFFECTIVENESS_SEVERITY_WEIGHTS
|
|
564
|
-
for query_type in data
|
|
636
|
+
for severity in MAINTENANCE_EFFECTIVENESS_SEVERITY_WEIGHTS:
|
|
637
|
+
for query_type in data:
|
|
565
638
|
params = build_query(severity, query_type)
|
|
566
639
|
|
|
567
640
|
r = utils.get_session("bugzilla").get(
|
|
@@ -577,10 +650,10 @@ def calculate_maintenance_effectiveness_indicator(
|
|
|
577
650
|
data[query_type][severity] = r.json()["bug_count"]
|
|
578
651
|
|
|
579
652
|
# Calculate number of bugs without severity set.
|
|
580
|
-
for query_type in data
|
|
653
|
+
for query_type in data:
|
|
581
654
|
data[query_type]["--"] = data[query_type]["--"] - sum(
|
|
582
655
|
data[query_type][s]
|
|
583
|
-
for s in MAINTENANCE_EFFECTIVENESS_SEVERITY_WEIGHTS
|
|
656
|
+
for s in MAINTENANCE_EFFECTIVENESS_SEVERITY_WEIGHTS
|
|
584
657
|
if s != "--"
|
|
585
658
|
)
|
|
586
659
|
|
|
@@ -591,7 +664,7 @@ def calculate_maintenance_effectiveness_indicator(
|
|
|
591
664
|
print("Before applying weights:")
|
|
592
665
|
print(data)
|
|
593
666
|
|
|
594
|
-
for query_type in data
|
|
667
|
+
for query_type in data:
|
|
595
668
|
# Apply weights.
|
|
596
669
|
for (
|
|
597
670
|
severity,
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
3
|
+
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
|
4
|
+
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
|
|
6
|
+
"""Which platforms build the files of mozilla-central, according to the builds of Firefox's CI.
|
|
7
|
+
|
|
8
|
+
Searchfox's indexing jobs build Firefox for each platform. Their index lists the source files
|
|
9
|
+
(including headers) the build compiled, and the build's chrome map the files it packaged (JS, CSS,
|
|
10
|
+
...): a file is built on the platforms whose builds list it. A file no build lists (a file added
|
|
11
|
+
since the builds, tests, docs, Rust or Java files, ...) is built on the platforms that build files
|
|
12
|
+
in its closest ancestor directory, or on all platforms if there's none.
|
|
13
|
+
|
|
14
|
+
A patch only touching files built on some platforms can only break those (e.g. widget/cocoa only
|
|
15
|
+
on macOS, gfx/layers/d3d11 only on Windows).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import io
|
|
19
|
+
import logging
|
|
20
|
+
import os
|
|
21
|
+
import zipfile
|
|
22
|
+
from typing import Collection, Iterable, Iterator, Mapping
|
|
23
|
+
|
|
24
|
+
import requests
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
# The OS families, named as test_scheduling.config_platform_family names them.
|
|
29
|
+
PLATFORMS = ("android", "linux", "macosx", "windows")
|
|
30
|
+
ALL_PLATFORMS = frozenset(PLATFORMS)
|
|
31
|
+
|
|
32
|
+
# The Searchfox jobs of each platform (gecko.v2.mozilla-central.latest.firefox.<job>), for the
|
|
33
|
+
# architectures tests run on.
|
|
34
|
+
SEARCHFOX_JOBS = {
|
|
35
|
+
"android": ("android-aarch64-searchfox-debug",),
|
|
36
|
+
"linux": ("linux64-searchfox-debug",),
|
|
37
|
+
"macosx": ("macosx64-searchfox-debug", "macosx64-aarch64-searchfox-debug"),
|
|
38
|
+
"windows": ("win64-searchfox-debug",),
|
|
39
|
+
}
|
|
40
|
+
# Platforms tested on an architecture their jobs don't build (most Android tests run on x86_64
|
|
41
|
+
# emulators, the job builds aarch64): a file built on other platforms (e.g. x86 SIMD code) isn't
|
|
42
|
+
# excluded if they build files in its directory.
|
|
43
|
+
INCOMPLETE_PLATFORMS = frozenset({"android"})
|
|
44
|
+
# Directories of code that some platforms' jobs don't build, although their tests run it: the Android
|
|
45
|
+
# job doesn't build gtests (4 files in gtest directories, against ~800 for the desktop jobs).
|
|
46
|
+
UNBUILT_DIRECTORIES = {"android": frozenset({"gtest"})}
|
|
47
|
+
|
|
48
|
+
ARTIFACT_URL = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.{job}/artifacts/public/build/{artifact}"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _RemoteFile(io.RawIOBase):
|
|
52
|
+
"""A file read with HTTP range requests, to list the files of a remote zip without downloading it."""
|
|
53
|
+
|
|
54
|
+
def __init__(self, url: str) -> None:
|
|
55
|
+
response = requests.head(url, allow_redirects=True, timeout=60)
|
|
56
|
+
response.raise_for_status()
|
|
57
|
+
self.url = response.url
|
|
58
|
+
self.size = int(response.headers["Content-Length"])
|
|
59
|
+
self.position = 0
|
|
60
|
+
|
|
61
|
+
def readable(self) -> bool:
|
|
62
|
+
return True
|
|
63
|
+
|
|
64
|
+
def seekable(self) -> bool:
|
|
65
|
+
return True
|
|
66
|
+
|
|
67
|
+
def tell(self) -> int:
|
|
68
|
+
return self.position
|
|
69
|
+
|
|
70
|
+
def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
|
|
71
|
+
start = {io.SEEK_SET: 0, io.SEEK_CUR: self.position, io.SEEK_END: self.size}
|
|
72
|
+
self.position = start[whence] + offset
|
|
73
|
+
return self.position
|
|
74
|
+
|
|
75
|
+
def readinto(self, buffer) -> int:
|
|
76
|
+
end = min(self.size, self.position + len(buffer))
|
|
77
|
+
if end <= self.position:
|
|
78
|
+
return 0
|
|
79
|
+
response = requests.get(
|
|
80
|
+
self.url,
|
|
81
|
+
headers={
|
|
82
|
+
"Range": f"bytes={self.position}-{end - 1}",
|
|
83
|
+
"Accept-Encoding": "identity",
|
|
84
|
+
},
|
|
85
|
+
timeout=60,
|
|
86
|
+
)
|
|
87
|
+
response.raise_for_status()
|
|
88
|
+
data = response.content
|
|
89
|
+
buffer[: len(data)] = data
|
|
90
|
+
self.position += len(data)
|
|
91
|
+
return len(data)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _job_files(job: str) -> set[str]:
|
|
95
|
+
"""The files compiled and packaged by a Searchfox job's build."""
|
|
96
|
+
url = ARTIFACT_URL.format(job=job, artifact="target.mozsearch-index.zip")
|
|
97
|
+
with zipfile.ZipFile(_RemoteFile(url)) as index:
|
|
98
|
+
files = {
|
|
99
|
+
name
|
|
100
|
+
for name in index.namelist()
|
|
101
|
+
if not name.endswith("/") and not name.startswith("__GENERATED__/")
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
response = requests.get(
|
|
105
|
+
ARTIFACT_URL.format(job=job, artifact="chrome-map.json"), timeout=60
|
|
106
|
+
)
|
|
107
|
+
response.raise_for_status()
|
|
108
|
+
# [chrome URLs, overrides, {installed path: [source path, ...]}, build config]: only the files
|
|
109
|
+
# packaged in the application, not test files installed for the harnesses (e.g. crashtests).
|
|
110
|
+
for installed, sources in response.json()[2].items():
|
|
111
|
+
if (
|
|
112
|
+
installed.startswith("dist/bin/")
|
|
113
|
+
and sources
|
|
114
|
+
and isinstance(sources[0], str)
|
|
115
|
+
and "*" not in sources[0]
|
|
116
|
+
):
|
|
117
|
+
files.add(sources[0])
|
|
118
|
+
|
|
119
|
+
return files
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def retrieve_built_files() -> dict[str, list[str]]:
|
|
123
|
+
"""The files built on each platform. Platforms whose files can't be retrieved are left out."""
|
|
124
|
+
built = {}
|
|
125
|
+
for platform, jobs in SEARCHFOX_JOBS.items():
|
|
126
|
+
try:
|
|
127
|
+
built[platform] = sorted(set().union(*(_job_files(job) for job in jobs)))
|
|
128
|
+
except Exception:
|
|
129
|
+
logger.exception("Failed to retrieve the files built on %s", platform)
|
|
130
|
+
return built
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _ancestors(path: str) -> Iterator[str]:
|
|
134
|
+
directory = os.path.dirname(path)
|
|
135
|
+
while directory:
|
|
136
|
+
yield directory
|
|
137
|
+
directory = os.path.dirname(directory)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class BuildPlatforms:
|
|
141
|
+
"""The platforms building files, from the files built on each platform (see retrieve_built_files)."""
|
|
142
|
+
|
|
143
|
+
def __init__(self, built: Mapping[str, Iterable[str]]) -> None:
|
|
144
|
+
self.files = {
|
|
145
|
+
platform: frozenset(files)
|
|
146
|
+
for platform, files in built.items()
|
|
147
|
+
if platform in ALL_PLATFORMS and files
|
|
148
|
+
}
|
|
149
|
+
self.directories = {
|
|
150
|
+
platform: frozenset(d for f in files for d in _ancestors(f))
|
|
151
|
+
for platform, files in self.files.items()
|
|
152
|
+
}
|
|
153
|
+
# The types of the files the builds list: the directory of a file of another type (e.g. the
|
|
154
|
+
# inputs of code generation, like WebIDL or Glean YAML files, or build files) doesn't tell
|
|
155
|
+
# which platforms use it.
|
|
156
|
+
self.types = frozenset(
|
|
157
|
+
os.path.splitext(f)[1] for files in self.files.values() for f in files
|
|
158
|
+
)
|
|
159
|
+
# The platforms whose files are unknown might build anything.
|
|
160
|
+
self.unknown = ALL_PLATFORMS - set(self.files)
|
|
161
|
+
|
|
162
|
+
def platforms(self, path: str) -> frozenset[str]:
|
|
163
|
+
return self._platforms(path) | self.unknown | self._unbuilt(path)
|
|
164
|
+
|
|
165
|
+
def _platforms(self, path: str) -> frozenset[str]:
|
|
166
|
+
listed = {p for p, files in self.files.items() if path in files}
|
|
167
|
+
if listed:
|
|
168
|
+
directory = os.path.dirname(path)
|
|
169
|
+
listed |= {
|
|
170
|
+
p
|
|
171
|
+
for p in INCOMPLETE_PLATFORMS & set(self.directories)
|
|
172
|
+
if directory in self.directories[p]
|
|
173
|
+
}
|
|
174
|
+
return frozenset(listed)
|
|
175
|
+
|
|
176
|
+
if os.path.splitext(path)[1] not in self.types:
|
|
177
|
+
return ALL_PLATFORMS
|
|
178
|
+
|
|
179
|
+
for directory in _ancestors(path):
|
|
180
|
+
near = {p for p, dirs in self.directories.items() if directory in dirs}
|
|
181
|
+
if near:
|
|
182
|
+
return frozenset(near)
|
|
183
|
+
|
|
184
|
+
return ALL_PLATFORMS
|
|
185
|
+
|
|
186
|
+
def _unbuilt(self, path: str) -> frozenset[str]:
|
|
187
|
+
directories = set(path.split("/")[:-1])
|
|
188
|
+
return frozenset(
|
|
189
|
+
p for p, names in UNBUILT_DIRECTORIES.items() if directories & names
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def find_platforms_for_paths(self, paths: Collection[str]) -> frozenset[str]:
|
|
193
|
+
"""The platforms building any of `paths`."""
|
|
194
|
+
result: frozenset[str] = frozenset().union(*map(self.platforms, paths))
|
|
195
|
+
return result or ALL_PLATFORMS
|
|
@@ -42,7 +42,9 @@ def search(repo_dir, commit_hash, symbol_name):
|
|
|
42
42
|
return []
|
|
43
43
|
|
|
44
44
|
logger.error(
|
|
45
|
-
|
|
45
|
+
"Error running 'hg grep' command.\nstdout:\n %s \n\nstderr:\n %s",
|
|
46
|
+
e.stdout.decode(),
|
|
47
|
+
e.stderr.decode(),
|
|
46
48
|
)
|
|
47
49
|
raise
|
|
48
50
|
|
|
@@ -60,7 +62,9 @@ def search(repo_dir, commit_hash, symbol_name):
|
|
|
60
62
|
)
|
|
61
63
|
except subprocess.CalledProcessError as e:
|
|
62
64
|
logger.error(
|
|
63
|
-
|
|
65
|
+
"Error running 'hg cat' command.\nstdout:\n%s\n\nstderr:\n%s",
|
|
66
|
+
e.stdout.decode(),
|
|
67
|
+
e.stderr.decode(),
|
|
64
68
|
)
|
|
65
69
|
raise
|
|
66
70
|
|