simdref 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simdref/__init__.py +6 -0
- simdref/__main__.py +6 -0
- simdref/annotate.py +448 -0
- simdref/arm_instructions.py +417 -0
- simdref/cli.py +1598 -0
- simdref/display.py +963 -0
- simdref/filters.py +318 -0
- simdref/ingest.py +113 -0
- simdref/ingest_catalog.py +1172 -0
- simdref/ingest_pdf.py +188 -0
- simdref/ingest_sources.py +580 -0
- simdref/lsp.py +208 -0
- simdref/manpages.py +139 -0
- simdref/models.py +225 -0
- simdref/pdfparse/__init__.py +13 -0
- simdref/pdfparse/base.py +116 -0
- simdref/pdfparse/intel.py +614 -0
- simdref/pdfparse/registry.py +19 -0
- simdref/pdfparse/types.py +77 -0
- simdref/pdfrefs.py +95 -0
- simdref/perf.py +220 -0
- simdref/perf_sources/__init__.py +51 -0
- simdref/perf_sources/cores.py +101 -0
- simdref/perf_sources/llvm_mca.py +176 -0
- simdref/perf_sources/llvm_scheduling.py +625 -0
- simdref/perf_sources/merge.py +121 -0
- simdref/queries.py +207 -0
- simdref/riscv.py +446 -0
- simdref/search.py +288 -0
- simdref/storage.py +504 -0
- simdref/templates/__init__.py +0 -0
- simdref/templates/app.js +1590 -0
- simdref/templates/favicon.svg +5 -0
- simdref/templates/index.html +112 -0
- simdref/templates/logo.svg +12 -0
- simdref/templates/style.css +680 -0
- simdref/tui.py +2366 -0
- simdref/web.py +403 -0
- simdref-0.0.0.dist-info/METADATA +240 -0
- simdref-0.0.0.dist-info/RECORD +44 -0
- simdref-0.0.0.dist-info/WHEEL +5 -0
- simdref-0.0.0.dist-info/entry_points.txt +4 -0
- simdref-0.0.0.dist-info/licenses/LICENSE +674 -0
- simdref-0.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
"""Source acquisition for intrinsics and instruction catalogs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import io
|
|
7
|
+
import re
|
|
8
|
+
import tarfile
|
|
9
|
+
import zipfile
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from datetime import UTC, datetime
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from simdref.models import SourceVersion
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SourceUnavailableError(RuntimeError):
|
|
20
|
+
"""Raised when every candidate source for a feed is unreachable.
|
|
21
|
+
|
|
22
|
+
The installer / CI surfaces this with a hint to use the pre-built
|
|
23
|
+
release catalog via ``simdref update`` instead of ``--build``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
UOPS_XML_URL = "https://uops.info/instructions.xml"
|
|
27
|
+
INTEL_OFFLINE_ZIP_URL = "https://cdrdv2.intel.com/v1/dl/getContent/764289?fileName=Intel-Intrinsics-Guide-Offline-3.6.4.zip"
|
|
28
|
+
INTEL_INDEX_URL = "https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html"
|
|
29
|
+
INTEL_CANDIDATE_DATA_URLS = [
|
|
30
|
+
"https://www.intel.com/content/dam/develop/public/us/en/include/intrinsics-guide/files/data.js",
|
|
31
|
+
"https://www.intel.com/content/dam/develop/public/us/en/include/intrinsics-guide/files/intrinsics.json",
|
|
32
|
+
]
|
|
33
|
+
ARM_ACLE_REPO_URL = "https://github.com/ARM-software/acle"
|
|
34
|
+
ARM_ACLE_DOC_URL = "https://arm-software.github.io/acle/main/"
|
|
35
|
+
ARM_NEON_DOC_URL = "https://arm-software.github.io/acle/neon_intrinsics/advsimd.html"
|
|
36
|
+
ARM_A64_DOC_URL = "https://developer.arm.com/documentation/ddi0602/latest/Base-Instructions"
|
|
37
|
+
ARM_A64_ARCHIVE_URL = (
|
|
38
|
+
"https://developer.arm.com/-/cdn-downloads/permalink/Exploration-Tools-OS-Machine-Readable-Data/"
|
|
39
|
+
"AARCHMRS_BSD/AARCHMRS_OPENSOURCE_A_profile_FAT-2025-09_ASL0.tar.gz"
|
|
40
|
+
)
|
|
41
|
+
ARM_ACLE_ARCHIVE_URL = "https://codeload.github.com/ARM-software/acle/zip/refs/heads/main"
|
|
42
|
+
ARM_INTRINSICS_DATA_BASE_URL = "https://developer.arm.com/architectures/instruction-sets/intrinsics/data/"
|
|
43
|
+
ARM_INTRINSICS_JSON_URL = ARM_INTRINSICS_DATA_BASE_URL + "intrinsics.json"
|
|
44
|
+
ARM_INTRINSICS_OPERATIONS_JSON_URL = ARM_INTRINSICS_DATA_BASE_URL + "operations.json"
|
|
45
|
+
ARM_INTRINSICS_EXAMPLES_JSON_URL = ARM_INTRINSICS_DATA_BASE_URL + "examples.json"
|
|
46
|
+
RISCV_UNIFIED_DB_REPO_URL = "https://github.com/riscv-software-src/riscv-unified-db"
|
|
47
|
+
RISCV_RVV_INTRINSICS_REPO_URL = "https://github.com/riscv-non-isa/riscv-rvv-intrinsic-doc"
|
|
48
|
+
RISCV_UNIFIED_DB_CANDIDATE_URLS = [
|
|
49
|
+
"https://raw.githubusercontent.com/riscv-software-src/riscv-unified-db/main/build/instructions.json",
|
|
50
|
+
"https://raw.githubusercontent.com/riscv-software-src/riscv-unified-db/main/generated/instructions.json",
|
|
51
|
+
"https://raw.githubusercontent.com/riscv-software-src/riscv-unified-db/main/artifacts/instructions.json",
|
|
52
|
+
]
|
|
53
|
+
RISCV_RVV_INTRINSICS_CANDIDATE_URLS = [
|
|
54
|
+
"https://raw.githubusercontent.com/riscv-non-isa/riscv-rvv-intrinsic-doc/main/auto-generated/intrinsics.json",
|
|
55
|
+
"https://raw.githubusercontent.com/riscv-non-isa/riscv-rvv-intrinsic-doc/main/generated/intrinsics.json",
|
|
56
|
+
"https://raw.githubusercontent.com/riscv-non-isa/riscv-rvv-intrinsic-doc/main/intrinsics.json",
|
|
57
|
+
]
|
|
58
|
+
ARM_ACLE_NEON_DB_PATH = "tools/intrinsic_db/advsimd.csv"
|
|
59
|
+
ARM_ACLE_NEON_CLASSIFICATION_PATH = "tools/intrinsic_db/advsimd_classification.csv"
|
|
60
|
+
ARM_ACLE_MAIN_MD_PATH = "main/acle.md"
|
|
61
|
+
ARM_ACLE_NEON_MD_PATH = "neon_intrinsics/advsimd.md"
|
|
62
|
+
|
|
63
|
+
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
64
|
+
LOCAL_INTEL_ARCHIVES = [
|
|
65
|
+
_REPO_ROOT / "vendor" / "intel" / "Intel-Intrinsics-Guide-Offline-3.6.9.zip",
|
|
66
|
+
]
|
|
67
|
+
LOCAL_UOPS_XMLS = [
|
|
68
|
+
_REPO_ROOT / "vendor" / "uops" / "instructions.xml",
|
|
69
|
+
]
|
|
70
|
+
LOCAL_ARM_ACLE_JSONS = [
|
|
71
|
+
_REPO_ROOT / "vendor" / "arm" / "acle_intrinsics.json",
|
|
72
|
+
]
|
|
73
|
+
LOCAL_ARM_INTRINSICS_JSONS = [
|
|
74
|
+
_REPO_ROOT / "vendor" / "arm" / "intrinsics.json",
|
|
75
|
+
]
|
|
76
|
+
LOCAL_ARM_OPERATIONS_JSONS = [
|
|
77
|
+
_REPO_ROOT / "vendor" / "arm" / "operations.json",
|
|
78
|
+
]
|
|
79
|
+
LOCAL_ARM_EXAMPLES_JSONS = [
|
|
80
|
+
_REPO_ROOT / "vendor" / "arm" / "examples.json",
|
|
81
|
+
]
|
|
82
|
+
LOCAL_ARM_ACLE_ARCHIVES = [
|
|
83
|
+
_REPO_ROOT / "vendor" / "arm" / "acle-main.zip",
|
|
84
|
+
]
|
|
85
|
+
LOCAL_ARM_A64_JSONS = [
|
|
86
|
+
_REPO_ROOT / "vendor" / "arm" / "a64_instructions.json",
|
|
87
|
+
]
|
|
88
|
+
ARM_A64_ARCHIVE_CACHE = _REPO_ROOT / "vendor" / "arm" / Path(ARM_A64_ARCHIVE_URL).name
|
|
89
|
+
LOCAL_ARM_A64_ARCHIVES = [
|
|
90
|
+
ARM_A64_ARCHIVE_CACHE,
|
|
91
|
+
_REPO_ROOT / "vendor" / "arm" / "AARCHMRS_BSD.tar.gz",
|
|
92
|
+
_REPO_ROOT / "vendor" / "arm" / "aarchmrs_bsd.tar.gz",
|
|
93
|
+
_REPO_ROOT / "vendor" / "arm" / "aarchmrs-bsd.tar.gz",
|
|
94
|
+
]
|
|
95
|
+
LOCAL_RISCV_UNIFIED_DB_JSONS = [
|
|
96
|
+
_REPO_ROOT / "vendor" / "riscv" / "unified_db_bundle.json",
|
|
97
|
+
_REPO_ROOT / "vendor" / "riscv" / "unified_db_instructions.json",
|
|
98
|
+
_REPO_ROOT / "vendor" / "riscv" / "instructions.json",
|
|
99
|
+
]
|
|
100
|
+
LOCAL_RISCV_RVV_INTRINSICS_JSONS = [
|
|
101
|
+
_REPO_ROOT / "vendor" / "riscv" / "rvv_intrinsics_bundle.json",
|
|
102
|
+
_REPO_ROOT / "vendor" / "riscv" / "rvv_intrinsics.json",
|
|
103
|
+
_REPO_ROOT / "vendor" / "riscv" / "intrinsics.json",
|
|
104
|
+
]
|
|
105
|
+
LOCAL_RISCV_DOCS_JSONS = [
|
|
106
|
+
_REPO_ROOT / "vendor" / "riscv" / "docs_pages.json",
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def now_iso() -> str:
|
|
111
|
+
return datetime.now(UTC).replace(microsecond=0).isoformat()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _fetch_text(url: str) -> str:
|
|
115
|
+
with httpx.Client(follow_redirects=True, timeout=20.0) as client:
|
|
116
|
+
response = client.get(url)
|
|
117
|
+
response.raise_for_status()
|
|
118
|
+
return response.text
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _read_local_intel_archive() -> tuple[str, SourceVersion] | None:
|
|
122
|
+
for archive_path in LOCAL_INTEL_ARCHIVES:
|
|
123
|
+
if not archive_path.exists():
|
|
124
|
+
continue
|
|
125
|
+
with zipfile.ZipFile(archive_path) as zf:
|
|
126
|
+
text = zf.read("Intel Intrinsics Guide/files/data.js").decode("utf-8", "replace")
|
|
127
|
+
return text, SourceVersion(
|
|
128
|
+
source="intel-intrinsics-guide",
|
|
129
|
+
version=f"offline-package:{archive_path.stem}",
|
|
130
|
+
fetched_at=now_iso(),
|
|
131
|
+
url=str(archive_path),
|
|
132
|
+
)
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def fetch_uops_xml() -> tuple[str | Path, SourceVersion]:
|
|
137
|
+
for xml_path in LOCAL_UOPS_XMLS:
|
|
138
|
+
if xml_path.exists():
|
|
139
|
+
return xml_path, SourceVersion(
|
|
140
|
+
source="uops.info",
|
|
141
|
+
version=f"local-xml:{xml_path.name}",
|
|
142
|
+
fetched_at=now_iso(),
|
|
143
|
+
url=str(xml_path),
|
|
144
|
+
)
|
|
145
|
+
try:
|
|
146
|
+
text = _fetch_text(UOPS_XML_URL)
|
|
147
|
+
except Exception as exc:
|
|
148
|
+
raise SourceUnavailableError(
|
|
149
|
+
f"uops.info instruction XML is unreachable at {UOPS_XML_URL}"
|
|
150
|
+
) from exc
|
|
151
|
+
return text, SourceVersion(
|
|
152
|
+
source="uops.info",
|
|
153
|
+
version="live",
|
|
154
|
+
fetched_at=now_iso(),
|
|
155
|
+
url=UOPS_XML_URL,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def fetch_intel_data() -> tuple[str, SourceVersion]:
|
|
160
|
+
local_archive = _read_local_intel_archive()
|
|
161
|
+
if local_archive is not None:
|
|
162
|
+
return local_archive
|
|
163
|
+
|
|
164
|
+
for url in INTEL_CANDIDATE_DATA_URLS:
|
|
165
|
+
try:
|
|
166
|
+
text = _fetch_text(url)
|
|
167
|
+
return text, SourceVersion(
|
|
168
|
+
source="intel-intrinsics-guide",
|
|
169
|
+
version="live",
|
|
170
|
+
fetched_at=now_iso(),
|
|
171
|
+
url=url,
|
|
172
|
+
)
|
|
173
|
+
except Exception:
|
|
174
|
+
pass
|
|
175
|
+
try:
|
|
176
|
+
html = _fetch_text(INTEL_INDEX_URL)
|
|
177
|
+
matches = re.findall(r'files/[^"\']+\.(?:js|json|xml)', html)
|
|
178
|
+
for match in matches:
|
|
179
|
+
candidate = f"https://www.intel.com/content/dam/develop/public/us/en/include/intrinsics-guide/{match}"
|
|
180
|
+
try:
|
|
181
|
+
text = _fetch_text(candidate)
|
|
182
|
+
return text, SourceVersion(
|
|
183
|
+
source="intel-intrinsics-guide",
|
|
184
|
+
version="live",
|
|
185
|
+
fetched_at=now_iso(),
|
|
186
|
+
url=candidate,
|
|
187
|
+
)
|
|
188
|
+
except Exception:
|
|
189
|
+
pass
|
|
190
|
+
except Exception:
|
|
191
|
+
pass
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
with httpx.Client(follow_redirects=True, timeout=60.0) as client:
|
|
195
|
+
resp = client.get(INTEL_OFFLINE_ZIP_URL)
|
|
196
|
+
resp.raise_for_status()
|
|
197
|
+
with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
|
|
198
|
+
for name in zf.namelist():
|
|
199
|
+
if name.endswith("data.js") or name.endswith("data.json"):
|
|
200
|
+
text = zf.read(name).decode("utf-8", "replace")
|
|
201
|
+
return text, SourceVersion(
|
|
202
|
+
source="intel-intrinsics-guide",
|
|
203
|
+
version="offline-zip-download",
|
|
204
|
+
fetched_at=now_iso(),
|
|
205
|
+
url=INTEL_OFFLINE_ZIP_URL,
|
|
206
|
+
)
|
|
207
|
+
except Exception:
|
|
208
|
+
pass
|
|
209
|
+
|
|
210
|
+
raise SourceUnavailableError(
|
|
211
|
+
"Intel intrinsics data is unreachable (network + local archives all failed). "
|
|
212
|
+
"Install llvm-mca + a working network, or run `simdref update` (no --build) to "
|
|
213
|
+
"use the pre-built release catalog."
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _read_local_text(paths: list[Path], source: str, version_prefix: str) -> tuple[str, SourceVersion] | None:
|
|
218
|
+
for text_path in paths:
|
|
219
|
+
if not text_path.exists():
|
|
220
|
+
continue
|
|
221
|
+
return text_path.read_text(), SourceVersion(
|
|
222
|
+
source=source,
|
|
223
|
+
version=f"{version_prefix}:{text_path.name}",
|
|
224
|
+
fetched_at=now_iso(),
|
|
225
|
+
url=str(text_path),
|
|
226
|
+
)
|
|
227
|
+
return None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _riscv_missing_semantics_urls(payload: dict) -> list[str]:
|
|
231
|
+
instructions = payload.get("instructions") or payload.get("records") or []
|
|
232
|
+
urls: list[str] = []
|
|
233
|
+
for item in instructions:
|
|
234
|
+
if not isinstance(item, dict):
|
|
235
|
+
continue
|
|
236
|
+
sections = item.get("description") or item.get("doc_sections") or item.get("sections") or {}
|
|
237
|
+
if not isinstance(sections, dict):
|
|
238
|
+
sections = {}
|
|
239
|
+
has_description = str(sections.get("Description") or "").strip()
|
|
240
|
+
has_operation = str(sections.get("Operation") or "").strip()
|
|
241
|
+
if has_description and has_operation:
|
|
242
|
+
continue
|
|
243
|
+
url = str(item.get("url") or item.get("reference_url") or "").strip()
|
|
244
|
+
if url.startswith("https://docs.riscv.org/"):
|
|
245
|
+
urls.append(url)
|
|
246
|
+
return list(dict.fromkeys(urls))
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _augment_riscv_unified_db_payload_with_docs(text: str) -> str:
|
|
250
|
+
try:
|
|
251
|
+
payload = json.loads(text)
|
|
252
|
+
except Exception:
|
|
253
|
+
return text
|
|
254
|
+
if not isinstance(payload, dict):
|
|
255
|
+
return text
|
|
256
|
+
if payload.get("docs_pages"):
|
|
257
|
+
return text
|
|
258
|
+
|
|
259
|
+
docs_pages: dict[str, str] = {}
|
|
260
|
+
local_docs = _read_local_text(LOCAL_RISCV_DOCS_JSONS, "docs.riscv.org", "local-json")
|
|
261
|
+
if local_docs is not None:
|
|
262
|
+
try:
|
|
263
|
+
parsed = json.loads(local_docs[0])
|
|
264
|
+
if isinstance(parsed, dict):
|
|
265
|
+
docs_pages.update({
|
|
266
|
+
str(key).strip(): str(value).strip()
|
|
267
|
+
for key, value in parsed.items()
|
|
268
|
+
if str(key).strip() and str(value).strip()
|
|
269
|
+
})
|
|
270
|
+
except Exception:
|
|
271
|
+
pass
|
|
272
|
+
|
|
273
|
+
for url in _riscv_missing_semantics_urls(payload):
|
|
274
|
+
if url in docs_pages:
|
|
275
|
+
continue
|
|
276
|
+
base = url.split("#", 1)[0]
|
|
277
|
+
if base in docs_pages:
|
|
278
|
+
docs_pages[url] = docs_pages[base]
|
|
279
|
+
continue
|
|
280
|
+
try:
|
|
281
|
+
page = _fetch_text(base)
|
|
282
|
+
except Exception:
|
|
283
|
+
continue
|
|
284
|
+
if page.strip():
|
|
285
|
+
docs_pages[base] = page
|
|
286
|
+
docs_pages[url] = page
|
|
287
|
+
|
|
288
|
+
if not docs_pages:
|
|
289
|
+
return text
|
|
290
|
+
payload["docs_pages"] = docs_pages
|
|
291
|
+
return json.dumps(payload)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _arm_acle_bundle_payload(
|
|
295
|
+
intrinsics_csv: str,
|
|
296
|
+
classification_csv: str,
|
|
297
|
+
acle_markdown: str = "",
|
|
298
|
+
neon_markdown: str = "",
|
|
299
|
+
) -> str:
|
|
300
|
+
return json.dumps(
|
|
301
|
+
{
|
|
302
|
+
"format": "acle-neon-csv-v1",
|
|
303
|
+
"intrinsics_csv": intrinsics_csv,
|
|
304
|
+
"classification_csv": classification_csv,
|
|
305
|
+
"acle_markdown": acle_markdown,
|
|
306
|
+
"neon_markdown": neon_markdown,
|
|
307
|
+
}
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _arm_intrinsics_bundle_payload(intrinsics_json: str, operations_json: str, examples_json: str = "[]") -> str:
|
|
312
|
+
return json.dumps(
|
|
313
|
+
{
|
|
314
|
+
"format": "arm-intrinsics-json-v1",
|
|
315
|
+
"intrinsics_json": intrinsics_json,
|
|
316
|
+
"operations_json": operations_json,
|
|
317
|
+
"examples_json": examples_json,
|
|
318
|
+
}
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _extract_zip_text(zf: zipfile.ZipFile, suffix: str) -> str:
|
|
323
|
+
for name in zf.namelist():
|
|
324
|
+
if name.endswith(suffix):
|
|
325
|
+
return zf.read(name).decode("utf-8", "replace")
|
|
326
|
+
raise KeyError(suffix)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _extract_zip_text_by_match(zf: zipfile.ZipFile, predicate) -> tuple[str, str]:
|
|
330
|
+
for name in zf.namelist():
|
|
331
|
+
if predicate(name):
|
|
332
|
+
return zf.read(name).decode("utf-8", "replace"), name
|
|
333
|
+
raise KeyError("zip member")
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _extract_tar_text_by_match(tf: tarfile.TarFile, predicate) -> tuple[str, str]:
|
|
337
|
+
for member in tf.getmembers():
|
|
338
|
+
if not member.isfile() or not predicate(member.name):
|
|
339
|
+
continue
|
|
340
|
+
extracted = tf.extractfile(member)
|
|
341
|
+
if extracted is None:
|
|
342
|
+
continue
|
|
343
|
+
return extracted.read().decode("utf-8", "replace"), member.name
|
|
344
|
+
raise KeyError("tar member")
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _read_local_arm_acle_archive() -> tuple[str, SourceVersion] | None:
|
|
348
|
+
for archive_path in LOCAL_ARM_ACLE_ARCHIVES:
|
|
349
|
+
if not archive_path.exists():
|
|
350
|
+
continue
|
|
351
|
+
with zipfile.ZipFile(archive_path) as zf:
|
|
352
|
+
return _arm_acle_bundle_payload(
|
|
353
|
+
_extract_zip_text(zf, ARM_ACLE_NEON_DB_PATH),
|
|
354
|
+
_extract_zip_text(zf, ARM_ACLE_NEON_CLASSIFICATION_PATH),
|
|
355
|
+
_extract_zip_text(zf, ARM_ACLE_MAIN_MD_PATH),
|
|
356
|
+
_extract_zip_text(zf, ARM_ACLE_NEON_MD_PATH),
|
|
357
|
+
), SourceVersion(
|
|
358
|
+
source="arm-acle",
|
|
359
|
+
version=f"archive:{archive_path.name}",
|
|
360
|
+
fetched_at=now_iso(),
|
|
361
|
+
url=str(archive_path),
|
|
362
|
+
)
|
|
363
|
+
return None
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _read_local_arm_intrinsics_bundle() -> tuple[str, SourceVersion] | None:
|
|
367
|
+
intrinsics = _read_local_text(LOCAL_ARM_INTRINSICS_JSONS, "arm-intrinsics-site", "local-json")
|
|
368
|
+
operations = _read_local_text(LOCAL_ARM_OPERATIONS_JSONS, "arm-intrinsics-site", "local-json")
|
|
369
|
+
if intrinsics is None or operations is None:
|
|
370
|
+
return None
|
|
371
|
+
examples = _read_local_text(LOCAL_ARM_EXAMPLES_JSONS, "arm-intrinsics-site", "local-json")
|
|
372
|
+
payload = _arm_intrinsics_bundle_payload(
|
|
373
|
+
intrinsics[0],
|
|
374
|
+
operations[0],
|
|
375
|
+
examples[0] if examples is not None else "[]",
|
|
376
|
+
)
|
|
377
|
+
return payload, SourceVersion(
|
|
378
|
+
source="arm-intrinsics-site",
|
|
379
|
+
version="local-json-bundle",
|
|
380
|
+
fetched_at=now_iso(),
|
|
381
|
+
url=str(LOCAL_ARM_INTRINSICS_JSONS[0].parent),
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _looks_like_arm_instruction_json(path: str) -> bool:
|
|
386
|
+
lowered = path.casefold()
|
|
387
|
+
if not lowered.endswith(".json"):
|
|
388
|
+
return False
|
|
389
|
+
basename = lowered.rsplit("/", 1)[-1]
|
|
390
|
+
if basename == "instructions.json" and "/" not in lowered.strip("/"):
|
|
391
|
+
return True
|
|
392
|
+
if "schema" in lowered or "register" in lowered or "system" in lowered:
|
|
393
|
+
return False
|
|
394
|
+
return "instruction" in lowered and ("a64" in lowered or basename == "instructions.json")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _arm_instruction_bundle_payload(instructions_json: str) -> str:
|
|
398
|
+
return json.dumps(
|
|
399
|
+
{
|
|
400
|
+
"format": "arm-aarchmrs-instructions-v1",
|
|
401
|
+
"instructions_json": instructions_json,
|
|
402
|
+
}
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def _read_local_arm_instruction_archive() -> tuple[str, SourceVersion] | None:
|
|
407
|
+
for archive_path in LOCAL_ARM_A64_ARCHIVES:
|
|
408
|
+
if not archive_path.exists():
|
|
409
|
+
continue
|
|
410
|
+
if archive_path.suffix == ".zip":
|
|
411
|
+
with zipfile.ZipFile(archive_path) as zf:
|
|
412
|
+
payload, member_name = _extract_zip_text_by_match(zf, _looks_like_arm_instruction_json)
|
|
413
|
+
else:
|
|
414
|
+
with tarfile.open(archive_path, "r:*") as tf:
|
|
415
|
+
payload, member_name = _extract_tar_text_by_match(tf, _looks_like_arm_instruction_json)
|
|
416
|
+
return _arm_instruction_bundle_payload(payload), SourceVersion(
|
|
417
|
+
source="arm-a64",
|
|
418
|
+
version=f"archive:{archive_path.name}:{member_name}",
|
|
419
|
+
fetched_at=now_iso(),
|
|
420
|
+
url=str(archive_path),
|
|
421
|
+
)
|
|
422
|
+
return None
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def refresh_local_arm_a64_archive(
|
|
426
|
+
on_progress: Callable[[int, int | None], None] | None = None,
|
|
427
|
+
) -> Path:
|
|
428
|
+
"""Download the AARCHMRS BSD archive into ``vendor/arm/`` if not already present.
|
|
429
|
+
|
|
430
|
+
The archive is large (~hundreds of MB) and pinned to a specific
|
|
431
|
+
release via its URL. The cache filename is derived from the URL's
|
|
432
|
+
basename, so bumping ``ARM_A64_ARCHIVE_URL`` automatically triggers a
|
|
433
|
+
fresh download without requiring the user to clean ``vendor/arm/``.
|
|
434
|
+
|
|
435
|
+
``on_progress(bytes_downloaded, total_bytes)`` fires on each chunk;
|
|
436
|
+
``total_bytes`` is ``None`` if the server omits ``Content-Length``.
|
|
437
|
+
"""
|
|
438
|
+
target = ARM_A64_ARCHIVE_CACHE
|
|
439
|
+
if target.exists() and target.stat().st_size > 0:
|
|
440
|
+
return target
|
|
441
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
442
|
+
tmp = target.with_suffix(target.suffix + ".part")
|
|
443
|
+
with httpx.Client(follow_redirects=True, timeout=600.0) as client:
|
|
444
|
+
with client.stream("GET", ARM_A64_ARCHIVE_URL) as response:
|
|
445
|
+
response.raise_for_status()
|
|
446
|
+
total_header = response.headers.get("Content-Length")
|
|
447
|
+
total = int(total_header) if total_header and total_header.isdigit() else None
|
|
448
|
+
downloaded = 0
|
|
449
|
+
with tmp.open("wb") as fh:
|
|
450
|
+
for chunk in response.iter_bytes(chunk_size=1 << 20):
|
|
451
|
+
fh.write(chunk)
|
|
452
|
+
downloaded += len(chunk)
|
|
453
|
+
if on_progress is not None:
|
|
454
|
+
on_progress(downloaded, total)
|
|
455
|
+
tmp.replace(target)
|
|
456
|
+
return target
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def refresh_local_arm_intrinsics_bundle(
|
|
460
|
+
on_progress: Callable[[int, int], None] | None = None,
|
|
461
|
+
) -> list[Path]:
|
|
462
|
+
target_dir = LOCAL_ARM_INTRINSICS_JSONS[0].parent
|
|
463
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
464
|
+
downloads = [
|
|
465
|
+
(ARM_INTRINSICS_JSON_URL, LOCAL_ARM_INTRINSICS_JSONS[0]),
|
|
466
|
+
(ARM_INTRINSICS_OPERATIONS_JSON_URL, LOCAL_ARM_OPERATIONS_JSONS[0]),
|
|
467
|
+
(ARM_INTRINSICS_EXAMPLES_JSON_URL, LOCAL_ARM_EXAMPLES_JSONS[0]),
|
|
468
|
+
]
|
|
469
|
+
written: list[Path] = []
|
|
470
|
+
total = len(downloads)
|
|
471
|
+
with httpx.Client(follow_redirects=True, timeout=120.0) as client:
|
|
472
|
+
for url, path in downloads:
|
|
473
|
+
response = client.get(url)
|
|
474
|
+
response.raise_for_status()
|
|
475
|
+
path.write_text(response.text)
|
|
476
|
+
written.append(path)
|
|
477
|
+
if on_progress is not None:
|
|
478
|
+
on_progress(len(written), total)
|
|
479
|
+
return written
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def fetch_arm_acle_data() -> tuple[str, SourceVersion]:
|
|
483
|
+
local_intrinsics_bundle = _read_local_arm_intrinsics_bundle()
|
|
484
|
+
if local_intrinsics_bundle is not None:
|
|
485
|
+
return local_intrinsics_bundle
|
|
486
|
+
local_archive = _read_local_arm_acle_archive()
|
|
487
|
+
if local_archive is not None:
|
|
488
|
+
return local_archive
|
|
489
|
+
local = _read_local_text(LOCAL_ARM_ACLE_JSONS, "arm-acle", "local-json")
|
|
490
|
+
if local is not None:
|
|
491
|
+
return local
|
|
492
|
+
try:
|
|
493
|
+
payload = _arm_intrinsics_bundle_payload(
|
|
494
|
+
_fetch_text(ARM_INTRINSICS_JSON_URL),
|
|
495
|
+
_fetch_text(ARM_INTRINSICS_OPERATIONS_JSON_URL),
|
|
496
|
+
_fetch_text(ARM_INTRINSICS_EXAMPLES_JSON_URL),
|
|
497
|
+
)
|
|
498
|
+
return payload, SourceVersion(
|
|
499
|
+
source="arm-intrinsics-site",
|
|
500
|
+
version="live-json",
|
|
501
|
+
fetched_at=now_iso(),
|
|
502
|
+
url=ARM_INTRINSICS_JSON_URL,
|
|
503
|
+
)
|
|
504
|
+
except Exception:
|
|
505
|
+
pass
|
|
506
|
+
try:
|
|
507
|
+
with httpx.Client(follow_redirects=True, timeout=60.0) as client:
|
|
508
|
+
resp = client.get(ARM_ACLE_ARCHIVE_URL)
|
|
509
|
+
resp.raise_for_status()
|
|
510
|
+
with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
|
|
511
|
+
payload = _arm_acle_bundle_payload(
|
|
512
|
+
_extract_zip_text(zf, ARM_ACLE_NEON_DB_PATH),
|
|
513
|
+
_extract_zip_text(zf, ARM_ACLE_NEON_CLASSIFICATION_PATH),
|
|
514
|
+
_extract_zip_text(zf, ARM_ACLE_MAIN_MD_PATH),
|
|
515
|
+
_extract_zip_text(zf, ARM_ACLE_NEON_MD_PATH),
|
|
516
|
+
)
|
|
517
|
+
return payload, SourceVersion(
|
|
518
|
+
source="arm-acle",
|
|
519
|
+
version="archive-download",
|
|
520
|
+
fetched_at=now_iso(),
|
|
521
|
+
url=ARM_ACLE_ARCHIVE_URL,
|
|
522
|
+
)
|
|
523
|
+
except Exception:
|
|
524
|
+
pass
|
|
525
|
+
raise SourceUnavailableError("Arm ACLE intrinsics data is unreachable.")
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def fetch_arm_a64_data() -> tuple[str, SourceVersion]:
|
|
529
|
+
local = _read_local_text(LOCAL_ARM_A64_JSONS, "arm-a64", "local-json")
|
|
530
|
+
if local is not None:
|
|
531
|
+
return local
|
|
532
|
+
archive = _read_local_arm_instruction_archive()
|
|
533
|
+
if archive is not None:
|
|
534
|
+
return archive
|
|
535
|
+
try:
|
|
536
|
+
refresh_local_arm_a64_archive()
|
|
537
|
+
except Exception as exc:
|
|
538
|
+
raise SourceUnavailableError(
|
|
539
|
+
f"Arm A64 instruction data is unreachable: {exc}"
|
|
540
|
+
) from exc
|
|
541
|
+
archive = _read_local_arm_instruction_archive()
|
|
542
|
+
if archive is not None:
|
|
543
|
+
return archive
|
|
544
|
+
raise SourceUnavailableError("Arm A64 instruction data is unreachable.")
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def fetch_riscv_unified_db_data() -> tuple[str, SourceVersion]:
|
|
548
|
+
local = _read_local_text(LOCAL_RISCV_UNIFIED_DB_JSONS, "riscv-unified-db", "local-json")
|
|
549
|
+
if local is not None:
|
|
550
|
+
return _augment_riscv_unified_db_payload_with_docs(local[0]), local[1]
|
|
551
|
+
for url in RISCV_UNIFIED_DB_CANDIDATE_URLS:
|
|
552
|
+
try:
|
|
553
|
+
text = _fetch_text(url)
|
|
554
|
+
return _augment_riscv_unified_db_payload_with_docs(text), SourceVersion(
|
|
555
|
+
source="riscv-unified-db",
|
|
556
|
+
version="live",
|
|
557
|
+
fetched_at=now_iso(),
|
|
558
|
+
url=url,
|
|
559
|
+
)
|
|
560
|
+
except Exception:
|
|
561
|
+
pass
|
|
562
|
+
raise SourceUnavailableError("RISC-V unified-db data is unreachable.")
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def fetch_riscv_rvv_intrinsics_data() -> tuple[str, SourceVersion]:
|
|
566
|
+
local = _read_local_text(LOCAL_RISCV_RVV_INTRINSICS_JSONS, "rvv-intrinsic-doc", "local-json")
|
|
567
|
+
if local is not None:
|
|
568
|
+
return local
|
|
569
|
+
for url in RISCV_RVV_INTRINSICS_CANDIDATE_URLS:
|
|
570
|
+
try:
|
|
571
|
+
text = _fetch_text(url)
|
|
572
|
+
return text, SourceVersion(
|
|
573
|
+
source="rvv-intrinsic-doc",
|
|
574
|
+
version="live",
|
|
575
|
+
fetched_at=now_iso(),
|
|
576
|
+
url=url,
|
|
577
|
+
)
|
|
578
|
+
except Exception:
|
|
579
|
+
pass
|
|
580
|
+
raise SourceUnavailableError("RISC-V RVV intrinsics data is unreachable.")
|