quantized-lab 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. quantized/__init__.py +11 -0
  2. quantized/__main__.py +14 -0
  3. quantized/api.py +220 -0
  4. quantized/app.py +206 -0
  5. quantized/calc/__init__.py +8 -0
  6. quantized/calc/_clipfit.py +76 -0
  7. quantized/calc/_natural_neighbor.py +219 -0
  8. quantized/calc/aggregate.py +159 -0
  9. quantized/calc/backgrounds.py +353 -0
  10. quantized/calc/baseline.py +349 -0
  11. quantized/calc/batch_fit.py +148 -0
  12. quantized/calc/constants.py +27 -0
  13. quantized/calc/corrections.py +192 -0
  14. quantized/calc/crystallography.py +400 -0
  15. quantized/calc/diffusion.py +120 -0
  16. quantized/calc/electrical.py +248 -0
  17. quantized/calc/electrochemistry.py +176 -0
  18. quantized/calc/element_data.json +1 -0
  19. quantized/calc/element_data.py +59 -0
  20. quantized/calc/errors.py +246 -0
  21. quantized/calc/figure.py +417 -0
  22. quantized/calc/figure_break.py +152 -0
  23. quantized/calc/figure_categorical.py +156 -0
  24. quantized/calc/figure_corner.py +229 -0
  25. quantized/calc/figure_facets.py +127 -0
  26. quantized/calc/figure_field.py +137 -0
  27. quantized/calc/figure_hitmap.py +116 -0
  28. quantized/calc/figure_labels.py +62 -0
  29. quantized/calc/figure_map.py +287 -0
  30. quantized/calc/figure_overrides.py +159 -0
  31. quantized/calc/figure_page.py +266 -0
  32. quantized/calc/figure_scale.py +125 -0
  33. quantized/calc/figure_statplots.py +167 -0
  34. quantized/calc/figure_styles.py +131 -0
  35. quantized/calc/figure_ternary.py +239 -0
  36. quantized/calc/figure_ticks.py +217 -0
  37. quantized/calc/fit_autoguess.py +156 -0
  38. quantized/calc/fit_bootstrap.py +163 -0
  39. quantized/calc/fit_bumps.py +258 -0
  40. quantized/calc/fit_constraints.py +114 -0
  41. quantized/calc/fit_equation.py +264 -0
  42. quantized/calc/fit_findxy.py +80 -0
  43. quantized/calc/fit_models.py +195 -0
  44. quantized/calc/fit_models_special.py +189 -0
  45. quantized/calc/fit_odr.py +99 -0
  46. quantized/calc/fit_scan.py +243 -0
  47. quantized/calc/fit_stats.py +215 -0
  48. quantized/calc/fitting.py +199 -0
  49. quantized/calc/formula.py +90 -0
  50. quantized/calc/global_curve_fit.py +305 -0
  51. quantized/calc/global_fit.py +181 -0
  52. quantized/calc/interp2d.py +260 -0
  53. quantized/calc/linecut.py +269 -0
  54. quantized/calc/magnetic.py +414 -0
  55. quantized/calc/magnetometry.py +464 -0
  56. quantized/calc/map.py +228 -0
  57. quantized/calc/mcmc.py +177 -0
  58. quantized/calc/optics.py +228 -0
  59. quantized/calc/pawley.py +251 -0
  60. quantized/calc/peak_batch.py +128 -0
  61. quantized/calc/peak_fit.py +259 -0
  62. quantized/calc/peak_integrate.py +104 -0
  63. quantized/calc/peak_multifit.py +260 -0
  64. quantized/calc/peak_track.py +134 -0
  65. quantized/calc/peaks.py +298 -0
  66. quantized/calc/peakshapes.py +85 -0
  67. quantized/calc/plotting.py +147 -0
  68. quantized/calc/processing.py +232 -0
  69. quantized/calc/qspace.py +48 -0
  70. quantized/calc/reductions.py +155 -0
  71. quantized/calc/reductions_fft.py +383 -0
  72. quantized/calc/refl_sld_presets.json +1 -0
  73. quantized/calc/reflectivity.py +80 -0
  74. quantized/calc/registry.py +303 -0
  75. quantized/calc/relaxation.py +119 -0
  76. quantized/calc/report.py +253 -0
  77. quantized/calc/report_emit.py +227 -0
  78. quantized/calc/resample.py +142 -0
  79. quantized/calc/rsm.py +91 -0
  80. quantized/calc/rsm_analyze.py +245 -0
  81. quantized/calc/semiconductor.py +488 -0
  82. quantized/calc/sld.py +131 -0
  83. quantized/calc/sld_formula.py +138 -0
  84. quantized/calc/spectral.py +357 -0
  85. quantized/calc/statplots.py +214 -0
  86. quantized/calc/stats.py +399 -0
  87. quantized/calc/stats_anova2.py +202 -0
  88. quantized/calc/stats_anova_ext.py +338 -0
  89. quantized/calc/stats_dist.py +196 -0
  90. quantized/calc/stats_glm.py +245 -0
  91. quantized/calc/stats_multivar.py +289 -0
  92. quantized/calc/stats_roc.py +157 -0
  93. quantized/calc/stats_survival.py +261 -0
  94. quantized/calc/stats_tests.py +380 -0
  95. quantized/calc/substrates.py +181 -0
  96. quantized/calc/superconductor.py +359 -0
  97. quantized/calc/surface_fit.py +290 -0
  98. quantized/calc/surface_models.py +156 -0
  99. quantized/calc/thermal.py +119 -0
  100. quantized/calc/thin_film.py +425 -0
  101. quantized/calc/unit_convert.py +259 -0
  102. quantized/calc/units.py +80 -0
  103. quantized/calc/vacuum.py +290 -0
  104. quantized/calc/xray.py +169 -0
  105. quantized/cli.py +214 -0
  106. quantized/datastruct.py +153 -0
  107. quantized/io/__init__.py +11 -0
  108. quantized/io/_hdf5_layout.py +308 -0
  109. quantized/io/_jcamp_asdf.py +135 -0
  110. quantized/io/_xrdml_scan.py +291 -0
  111. quantized/io/base.py +82 -0
  112. quantized/io/bruker_brml.py +177 -0
  113. quantized/io/bruker_raw.py +158 -0
  114. quantized/io/cif.py +266 -0
  115. quantized/io/consolidated.py +122 -0
  116. quantized/io/delimited.py +222 -0
  117. quantized/io/excel.py +135 -0
  118. quantized/io/hdf5.py +192 -0
  119. quantized/io/import_filters.py +178 -0
  120. quantized/io/import_preview.py +262 -0
  121. quantized/io/jcamp.py +179 -0
  122. quantized/io/lakeshore.py +163 -0
  123. quantized/io/ncnr.py +278 -0
  124. quantized/io/netcdf.py +195 -0
  125. quantized/io/opus.py +231 -0
  126. quantized/io/origin.py +346 -0
  127. quantized/io/origin_com.py +194 -0
  128. quantized/io/origin_project/__init__.py +221 -0
  129. quantized/io/origin_project/annotation_marks.py +288 -0
  130. quantized/io/origin_project/container.py +262 -0
  131. quantized/io/origin_project/curve_style_color.py +359 -0
  132. quantized/io/origin_project/figure_geometry.py +108 -0
  133. quantized/io/origin_project/figure_layers.py +333 -0
  134. quantized/io/origin_project/figure_text.py +258 -0
  135. quantized/io/origin_project/figures.py +210 -0
  136. quantized/io/origin_project/figures_opju.py +440 -0
  137. quantized/io/origin_project/notes.py +302 -0
  138. quantized/io/origin_project/opj.py +459 -0
  139. quantized/io/origin_project/opj_curves.py +297 -0
  140. quantized/io/origin_project/opj_shapes.py +148 -0
  141. quantized/io/origin_project/opju.py +146 -0
  142. quantized/io/origin_project/opju_axis_real_form.py +418 -0
  143. quantized/io/origin_project/opju_axis_specimen_form.py +167 -0
  144. quantized/io/origin_project/opju_codec.py +370 -0
  145. quantized/io/origin_project/opju_curves.py +497 -0
  146. quantized/io/origin_project/opju_curves_allcols.py +258 -0
  147. quantized/io/origin_project/opju_figure_curves.py +302 -0
  148. quantized/io/origin_project/opju_figure_text.py +245 -0
  149. quantized/io/origin_project/opju_reports.py +129 -0
  150. quantized/io/origin_project/origin_richtext.py +145 -0
  151. quantized/io/origin_project/preview.py +132 -0
  152. quantized/io/origin_project/templates.py +314 -0
  153. quantized/io/origin_project/tree.py +379 -0
  154. quantized/io/origin_project/tree_opju.py +228 -0
  155. quantized/io/origin_project/windows.py +238 -0
  156. quantized/io/origin_project/windows_opju.py +393 -0
  157. quantized/io/origin_project/writer.py +156 -0
  158. quantized/io/origin_project/writer_blocks.py +282 -0
  159. quantized/io/qd.py +380 -0
  160. quantized/io/refl1d.py +132 -0
  161. quantized/io/registry.py +210 -0
  162. quantized/io/report_export.py +347 -0
  163. quantized/io/rigaku.py +100 -0
  164. quantized/io/sims.py +398 -0
  165. quantized/io/spc.py +311 -0
  166. quantized/io/xrd_csv.py +308 -0
  167. quantized/io/xrdml.py +394 -0
  168. quantized/jobs.py +173 -0
  169. quantized/plugins/__init__.py +50 -0
  170. quantized/plugins/contract.py +111 -0
  171. quantized/plugins/loader.py +394 -0
  172. quantized/plugins/steps.py +90 -0
  173. quantized/routes/__init__.py +7 -0
  174. quantized/routes/_bookcache.py +62 -0
  175. quantized/routes/_export_common.py +27 -0
  176. quantized/routes/_payload.py +58 -0
  177. quantized/routes/_uploadcache.py +59 -0
  178. quantized/routes/aggregate.py +46 -0
  179. quantized/routes/baseline.py +210 -0
  180. quantized/routes/books.py +117 -0
  181. quantized/routes/calc.py +56 -0
  182. quantized/routes/corrections.py +78 -0
  183. quantized/routes/crystallography.py +80 -0
  184. quantized/routes/diffusion.py +58 -0
  185. quantized/routes/electrical.py +101 -0
  186. quantized/routes/electrochemistry.py +83 -0
  187. quantized/routes/export.py +280 -0
  188. quantized/routes/export_facets.py +83 -0
  189. quantized/routes/export_figures.py +471 -0
  190. quantized/routes/export_page.py +125 -0
  191. quantized/routes/fitting.py +379 -0
  192. quantized/routes/fitting_bumps.py +97 -0
  193. quantized/routes/import_template.py +97 -0
  194. quantized/routes/import_wizard.py +150 -0
  195. quantized/routes/jobs_api.py +59 -0
  196. quantized/routes/magnetic.py +135 -0
  197. quantized/routes/magnetometry.py +133 -0
  198. quantized/routes/optics.py +98 -0
  199. quantized/routes/parsers.py +281 -0
  200. quantized/routes/peaks.py +184 -0
  201. quantized/routes/plot.py +103 -0
  202. quantized/routes/reductions.py +121 -0
  203. quantized/routes/reference.py +58 -0
  204. quantized/routes/reflectivity.py +91 -0
  205. quantized/routes/report_export.py +119 -0
  206. quantized/routes/rsm.py +136 -0
  207. quantized/routes/samples.py +32 -0
  208. quantized/routes/semiconductor.py +207 -0
  209. quantized/routes/sld.py +42 -0
  210. quantized/routes/spectral.py +54 -0
  211. quantized/routes/statplots.py +99 -0
  212. quantized/routes/stats.py +418 -0
  213. quantized/routes/stats_design.py +321 -0
  214. quantized/routes/substrates.py +46 -0
  215. quantized/routes/superconductor.py +139 -0
  216. quantized/routes/thermal.py +57 -0
  217. quantized/routes/thin_film.py +153 -0
  218. quantized/routes/vacuum.py +113 -0
  219. quantized/routes/xray.py +32 -0
  220. quantized/samples/demo_vsm.csv +42 -0
  221. quantized/server_launch.py +251 -0
  222. quantized/web/assets/JetBrainsMono-Bold-CUogYd9I.woff2 +0 -0
  223. quantized/web/assets/JetBrainsMono-Regular-CA-Os4ii.woff2 +0 -0
  224. quantized/web/assets/index-BHmmCL-x.js +27 -0
  225. quantized/web/assets/index-BiZzN7J6.css +1 -0
  226. quantized/web/index.html +13 -0
  227. quantized/web/loading.html +69 -0
  228. quantized_lab-0.8.0.dist-info/METADATA +122 -0
  229. quantized_lab-0.8.0.dist-info/RECORD +233 -0
  230. quantized_lab-0.8.0.dist-info/WHEEL +4 -0
  231. quantized_lab-0.8.0.dist-info/entry_points.txt +4 -0
  232. quantized_lab-0.8.0.dist-info/licenses/LICENSE +201 -0
  233. quantized_lab-0.8.0.dist-info/licenses/NOTICE +11 -0
@@ -0,0 +1,379 @@
1
+ """Origin *Project Explorer* folder tree — which folder each window lives in.
2
+
3
+ ``.opj`` (CPYA) — SOLVED and validated general (not sample-tuned): the tail
4
+ of the file (after the datasets+windows block stream) is::
5
+
6
+ <params: name\\n f64 \\n, repeated, terminated by a lone 0x00>
7
+ NULL-block, <project-record block (size varies)>
8
+ <notes: name-block + content-block, repeated, terminated by a NULL block>
9
+ <blk4 scalar> <blk16 ids>
10
+ <folder> -- the root folder, recursively
11
+
12
+ folder := <hdr32: 32-byte block, zeros + 2 f64 dates>
13
+ NULL
14
+ <name-block: NUL-terminated folder name, any bytes/length>
15
+ <bare u32 LE == 2><0x0A> -- fixed marker, no payload
16
+ <attrs block> <storage block> -- sizes vary; skipped as-is
17
+ <nwin: u32>
18
+ {NULL <8-byte (u32 flags, u32 ordinal)> NULL} * nwin
19
+ <nsub: u32>
20
+ {folder} * nsub -- recursion; no root closer
21
+
22
+ Windows are referenced by their **0-based ordinal into the file's window-
23
+ header stream order** (every worksheet AND graph window counts, in the
24
+ order their header block appears) — not by name, not by offset. This is a
25
+ real recursive parse: nesting depth, per-folder window/subfolder counts,
26
+ and folder names are never hard-coded, only the block *shapes* are (and
27
+ those are self-describing — each block carries its own byte length).
28
+ Validated byte-exact (0 mismatches) against the full window->folder-path
29
+ mapping COM reports for 7 structurally diverse real projects (611 windows
30
+ total): trivial (``SuperlatticeFits``, 6 sibling folders), root-level
31
+ windows mixed with folders (``MnN_Diffusion_PNR``), 4-5 levels of nesting
32
+ with duplicate folder names at different parents and empty intermediate
33
+ folders (``PNR``, ``XMCD``), and both container sub-versions (CPYA
34
+ 4.3227 ``XRD``/4.3380 ``Moke``).
35
+
36
+ Window-name enumeration is its own small hazard: a window's short name can
37
+ legitimately start with a digit (e.g. ``"30nmADPNR"``), and a purely
38
+ byte-shape-based scan can also collide with an ordinary 10-byte-record data
39
+ block that happens to start ``00 00`` and look printable for a few bytes.
40
+ Requiring the extracted name to fully match a plain identifier charset
41
+ (``[A-Za-z0-9][A-Za-z0-9_-]*``) resolves both: real Origin window names use
42
+ only that charset (verified across every project in the corpus), while the
43
+ rare data-block collisions always contain a space or punctuation the regex
44
+ rejects. This was found and fixed against ``MnN_Diffusion_PNR.opj`` (6
45
+ digit-led graph names were silently dropped by an alpha-only heuristic,
46
+ shifting every later ordinal by an accumulating offset) and ``hc2convert.opj``
47
+ /``XMCD.opj`` (single-character false-positive collisions inside the
48
+ datasets section).
49
+
50
+ The ``.opju`` (CPYUA) folder tree — which reuses this module's shared
51
+ ``_FolderNode`` + ``_flatten`` — lives in the sibling ``tree_opju.py``
52
+ (:func:`~quantized.io.origin_project.tree_opju.opju_folder_paths`), split out
53
+ to keep each module under the size ceiling.
54
+ """
55
+
56
+ from __future__ import annotations
57
+
58
+ import re
59
+ import struct
60
+ from dataclasses import dataclass, field
61
+ from datetime import datetime, timedelta
62
+
63
+ from quantized.io.origin_project.container import walk_blocks
64
+
65
+ __all__ = ["opj_folder_paths", "opj_project_dates"]
66
+
67
+ # A plain Origin window (page) short name: verified against every window name
68
+ # in a 12-file corpus (.opj + .opju) via live COM enumeration — always
69
+ # `[A-Za-z0-9_-]`, never a space or other punctuation (folder names, unlike
70
+ # window names, CAN have spaces/parens/unicode — see `_read_cstring`).
71
+ # Length 1+ (a window renamed to a single letter is legal in Origin; a
72
+ # 2-char minimum would skip it and shift every later window's ordinal — the
73
+ # same accumulating-offset bug class the digit-led fix addressed).
74
+ _WINDOW_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,62}")
75
+
76
+
77
+ class _TailParseError(ValueError):
78
+ """The tail doesn't match the expected grammar at this byte position.
79
+
80
+ Always caught at the top level (:func:`_parse_opj_tree`) — a folder-tree
81
+ parse failure must never crash the caller or fall back to a guess, only
82
+ to an empty mapping (every book then defaults to ``origin_folder_path:
83
+ []``, same as a project that never used folders at all).
84
+ """
85
+
86
+
87
+ @dataclass
88
+ class _FolderNode:
89
+ name: str
90
+ windows: list[int] = field(default_factory=list)
91
+ subfolders: list[_FolderNode] = field(default_factory=list)
92
+
93
+
94
+ def _enumerate_window_names(b: bytes) -> list[str]:
95
+ """Every window (worksheet or graph) header name, in file stream order.
96
+
97
+ A window header block is >=150 bytes, starts ``00 00``, and carries a
98
+ NUL-terminated name at offset 2 (see ``windows.py``'s ``_is_window_header``
99
+ for the ``.opj`` reader's own, looser version of this check — this is a
100
+ stricter, purpose-built variant: see the module docstring for why).
101
+ """
102
+ names: list[str] = []
103
+ for size, payload in walk_blocks(b):
104
+ if size == 0 or len(payload) < 150 or payload[0] or payload[1]:
105
+ continue
106
+ end = payload.find(b"\x00", 2, 66)
107
+ if end <= 2:
108
+ continue
109
+ raw = payload[2:end]
110
+ if not all(0x20 <= c < 0x7F for c in raw):
111
+ continue
112
+ name = raw.decode("latin1")
113
+ if _WINDOW_NAME_RE.fullmatch(name):
114
+ names.append(name)
115
+ return names
116
+
117
+
118
+ def _read_block(b: bytes, p: int) -> tuple[int, bytes, int]:
119
+ """One ``<u32 size LE><0x0A><payload><0x0A>`` block (or a 5-byte ``size==0``
120
+ spacer); returns ``(size, payload, next_pos)``. Raises on any framing
121
+ mismatch or out-of-bounds read — the caller always treats that as "stop,
122
+ don't guess"."""
123
+ if p + 5 > len(b):
124
+ raise _TailParseError(f"truncated at {p}")
125
+ size = int.from_bytes(b[p : p + 4], "little")
126
+ if b[p + 4] != 0x0A:
127
+ raise _TailParseError(f"bad block framing at {p}")
128
+ if size == 0:
129
+ return 0, b"", p + 5
130
+ end = p + 5 + size
131
+ if end >= len(b) or b[end] != 0x0A:
132
+ raise _TailParseError(f"bad block closing framing at {p} (size={size})")
133
+ return size, b[p + 5 : end], end + 1
134
+
135
+
136
+ def _find_tail_start(b: bytes) -> int:
137
+ """Byte position right after the datasets+windows block stream ends.
138
+
139
+ ``walk_blocks`` already stops the instant the ``<u32><0x0A>`` framing
140
+ breaks (see ``container.py``) — that first break is exactly where the
141
+ free-text "Parameters" section begins (plan-item 34's stream model).
142
+ """
143
+ nl = b.find(b"\n")
144
+ if nl < 0:
145
+ raise _TailParseError("no header line")
146
+ pos = nl + 1
147
+ for size, _payload in walk_blocks(b):
148
+ pos += 5 if size == 0 else 5 + size + 1
149
+ return pos
150
+
151
+
152
+ def _skip_params(b: bytes, p: int) -> int:
153
+ """The free-text ``<name>\\n<f64:8 bytes>\\n`` run (any count, including
154
+ zero), terminated by a lone non-printable byte (a bare ``0x00``) whose
155
+ line is consumed through its own trailing newline."""
156
+ n = len(b)
157
+ while True:
158
+ e = b.find(b"\n", p)
159
+ if e < 0:
160
+ raise _TailParseError("params section never terminates")
161
+ line = b[p:e]
162
+ if not line or not all(0x20 <= c < 0x7F for c in line):
163
+ return e + 1
164
+ p = e + 1 + 8
165
+ if p >= n or b[p] != 0x0A:
166
+ raise _TailParseError(f"expected newline after param value at {p}")
167
+ p += 1
168
+
169
+
170
+ def _skip_project_record(b: bytes, p: int) -> int:
171
+ """A NULL block, then one opaque "project record" block (size varies —
172
+ 88 bytes is the common case, but not asserted; see the module docstring:
173
+ only the *shape*, never a specific byte count, is trusted)."""
174
+ size, _payload, p = _read_block(b, p)
175
+ if size != 0:
176
+ raise _TailParseError("expected NULL block before the project record")
177
+ _size, _payload, p = _read_block(b, p)
178
+ return p
179
+
180
+
181
+ # The project record's two f64 Julian-date fields (created/modified — they
182
+ # match the results log's own JDN timestamps, validation_log.md §"tail"):
183
+ # payload offsets 32/40 of the CPYA-4.3380 88-byte record. The 4.3227
184
+ # variant is 80 bytes with NO value in the Julian range at any offset
185
+ # (measured on XMCD.opj) — fail-closed there.
186
+ _PROJECT_DATE_OFFSETS = (32, 40)
187
+ _JD_MIN, _JD_MAX = 2_378_497.0, 2_524_593.0 # ~1800-01-01 .. ~2200-01-01
188
+ _JD_UNIX_EPOCH = 2_440_587.5 # JD of 1970-01-01T00:00:00
189
+
190
+
191
+ def _jd_to_iso(jd: float) -> str:
192
+ """Julian date -> naive ISO-8601 (Origin stamps local wall-clock time)."""
193
+ epoch = datetime(1970, 1, 1)
194
+ return (epoch + timedelta(days=jd - _JD_UNIX_EPOCH)).isoformat(timespec="seconds")
195
+
196
+
197
+ def opj_project_dates(b: bytes) -> dict[str, str] | None:
198
+ """The project's created/modified timestamps from the ``.opj`` tail's
199
+ project record, as naive ISO-8601 strings, or ``None`` when the tail
200
+ doesn't parse or the fields aren't plausible Julian dates (the 4.3227
201
+ container variant, truncated files, CPYUA bytes) — never guessed.
202
+ """
203
+ try:
204
+ p = _find_tail_start(b)
205
+ p = _skip_params(b, p)
206
+ size, _payload, p = _read_block(b, p)
207
+ if size != 0:
208
+ return None
209
+ size, payload, _p = _read_block(b, p)
210
+ except (_TailParseError, struct.error):
211
+ return None
212
+ if len(payload) < _PROJECT_DATE_OFFSETS[-1] + 8:
213
+ return None
214
+ created, modified = (
215
+ float(struct.unpack_from("<d", payload, off)[0]) for off in _PROJECT_DATE_OFFSETS
216
+ )
217
+ if not all(_JD_MIN <= v <= _JD_MAX for v in (created, modified)):
218
+ return None
219
+ return {"created": _jd_to_iso(created), "modified": _jd_to_iso(modified)}
220
+
221
+
222
+ def _skip_notes(b: bytes, p: int) -> int:
223
+ """Zero or more ``name-block + content-block`` note pairs, terminated by
224
+ a NULL block (the count is never hard-coded — real projects range from
225
+ zero notes to one ``ResultsLog``; the loop reads however many there
226
+ are)."""
227
+ while True:
228
+ size, _payload, p2 = _read_block(b, p)
229
+ if size == 0:
230
+ return p2
231
+ p = p2
232
+ _size2, _content, p2 = _read_block(b, p)
233
+ p = p2
234
+
235
+
236
+ def _read_cstring_block(b: bytes, p: int) -> tuple[str, int]:
237
+ """A folder name-block's text, decoded latin1 (byte-preserving, like every
238
+ other name decode in this reader family -- ``windows.py``'s ``_cstring``).
239
+ Any byte value round-trips exactly; genuine multi-byte Unicode (CJK,
240
+ emoji) folder names are not specially handled and may come back as
241
+ mojibake rather than crash or being guessed at -- an inherited
242
+ limitation, not one introduced here."""
243
+ size, payload, p = _read_block(b, p)
244
+ if size == 0:
245
+ raise _TailParseError("expected a name block, got NULL")
246
+ return payload.rstrip(b"\x00").decode("latin1", errors="replace"), p
247
+
248
+
249
+ def _parse_folder(b: bytes, p: int) -> tuple[_FolderNode, int]:
250
+ """One ``folder`` record (module docstring), recursing into ``nsub``
251
+ children. Depth and per-folder window/subfolder counts are read from the
252
+ file, not assumed — this is what makes the parse general rather than a
253
+ scan tuned to any one sample's shape."""
254
+ size, _payload, p = _read_block(b, p)
255
+ if size != 32:
256
+ raise _TailParseError(f"expected the 32-byte folder header at {p}")
257
+ size, _payload, p = _read_block(b, p)
258
+ if size != 0:
259
+ raise _TailParseError("expected NULL after the folder header")
260
+ name, p = _read_cstring_block(b, p)
261
+
262
+ if p + 5 > len(b):
263
+ raise _TailParseError(f"truncated bare-u32 marker at {p}")
264
+ marker = int.from_bytes(b[p : p + 4], "little")
265
+ if marker != 2 or b[p + 4] != 0x0A:
266
+ raise _TailParseError(f"expected the bare u32==2 marker at {p}")
267
+ p += 5
268
+
269
+ _size, _payload, p = _read_block(b, p) # attrs block (size varies)
270
+ _size, _payload, p = _read_block(b, p) # storage block (size varies)
271
+
272
+ size, payload, p = _read_block(b, p)
273
+ if size != 4:
274
+ raise _TailParseError(f"expected the window-count scalar at {p}")
275
+ nwin = int.from_bytes(payload, "little")
276
+
277
+ ordinals: list[int] = []
278
+ for _ in range(nwin):
279
+ s0, _p0, p = _read_block(b, p)
280
+ if s0 != 0:
281
+ raise _TailParseError("expected NULL before a window entry")
282
+ s1, payload1, p = _read_block(b, p)
283
+ if s1 != 8:
284
+ raise _TailParseError(f"expected an 8-byte window entry at {p}")
285
+ _flags, ordinal = struct.unpack("<II", payload1)
286
+ ordinals.append(ordinal)
287
+ s2, _p2, p = _read_block(b, p)
288
+ if s2 != 0:
289
+ raise _TailParseError("expected NULL after a window entry")
290
+
291
+ size, payload, p = _read_block(b, p)
292
+ if size != 4:
293
+ raise _TailParseError(f"expected the subfolder-count scalar at {p}")
294
+ nsub = int.from_bytes(payload, "little")
295
+
296
+ subfolders: list[_FolderNode] = []
297
+ for _ in range(nsub):
298
+ child, p = _parse_folder(b, p)
299
+ subfolders.append(child)
300
+
301
+ return _FolderNode(name=name, windows=ordinals, subfolders=subfolders), p
302
+
303
+
304
+ def _parse_opj_tree(b: bytes) -> _FolderNode | None:
305
+ try:
306
+ p = _find_tail_start(b)
307
+ p = _skip_params(b, p)
308
+ p = _skip_project_record(b, p)
309
+ p = _skip_notes(b, p)
310
+ _size, _payload, p = _read_block(b, p) # leading scalar (unused)
311
+ _size, _payload, p = _read_block(b, p) # id blob (unused)
312
+ root, _p = _parse_folder(b, p)
313
+ return root
314
+ except (_TailParseError, UnicodeDecodeError, struct.error, RecursionError):
315
+ # RecursionError: a pathologically deep folder chain — degrade to flat.
316
+ return None
317
+
318
+
319
+ def _flatten(
320
+ node: _FolderNode,
321
+ names: list[str],
322
+ path: tuple[str, ...],
323
+ out: dict[str, list[str]],
324
+ ) -> None:
325
+ # Iterative pre-order (explicit stack) so a pathologically deep chain can't
326
+ # hit the recursion limit and break the callers' never-raises contract.
327
+ # Push reversed → siblings pop left-to-right (keeps first-write-wins order).
328
+ stack: list[tuple[_FolderNode, tuple[str, ...]]] = [(node, path)]
329
+ while stack:
330
+ cur, cur_path = stack.pop()
331
+ for ordinal in cur.windows:
332
+ if 0 <= ordinal < len(names):
333
+ out.setdefault(names[ordinal], list(cur_path))
334
+ for sub in reversed(cur.subfolders):
335
+ stack.append((sub, cur_path + (sub.name,)))
336
+
337
+
338
+ def opj_folder_paths(b: bytes) -> dict[str, list[str]]:
339
+ """Map every window's short name to its Project Explorer folder path.
340
+
341
+ The path is root-exclusive (a window sitting directly in the project's
342
+ root folder maps to ``[]``) and lists ancestor folder names from the
343
+ root down. A window that never appears in any folder's window list (a
344
+ "stray" window some real projects have, e.g. an auto-generated report
345
+ table never dragged into a folder) is simply absent from the returned
346
+ mapping — the caller treats that the same as "no folder tree at all":
347
+ default to ``[]``, never guess. Returns ``{}`` when the tail can't be
348
+ parsed (older/corrupt/unexpected containers) — never raises.
349
+ """
350
+ root = _parse_opj_tree(b)
351
+ if root is None:
352
+ return {}
353
+ names = _enumerate_window_names(b)
354
+ # Fail-closed cross-check: the tree's ordinals index into the enumerated
355
+ # name list positionally, so an ordinal past the end PROVES the name
356
+ # enumeration missed at least one window header — every later name would
357
+ # be attributed to the wrong window. Degrade to "no tree" rather than
358
+ # ship misattributed folder paths.
359
+ max_ordinal = max(
360
+ (o for node in _iter_nodes(root) for o in node.windows), default=-1
361
+ )
362
+ if max_ordinal >= len(names):
363
+ return {}
364
+ out: dict[str, list[str]] = {}
365
+ _flatten(root, names, (), out)
366
+ return out
367
+
368
+
369
+ def _iter_nodes(root: _FolderNode) -> list[_FolderNode]:
370
+ """Every folder node, iteratively (same no-recursion rationale as
371
+ ``_flatten``)."""
372
+ out: list[_FolderNode] = []
373
+ stack: list[_FolderNode] = [root]
374
+ while stack:
375
+ cur = stack.pop()
376
+ out.append(cur)
377
+ stack.extend(cur.subfolders)
378
+ return out
379
+
@@ -0,0 +1,228 @@
1
+ """``.opju`` (CPYUA) Project Explorer folder tree decode.
2
+
3
+ Split out of ``tree.py`` (the ``.opj`` decoder) to stay under the module
4
+ ceiling; it reuses that module's shared ``_FolderNode`` + ``_flatten``.
5
+
6
+ SOLVED for both known container sub-versions (4.3811, what OriginPro 2026b
7
+ writes, and the older 4.3380 corpus). The CPYUA tail reuses ``.opj``'s
8
+ name-block framing but its own everything-else: each folder record is
9
+ ``<name-block> <attrs/storage> <2*nwin> [00] {window-entry} <2*nsub>
10
+ {SEP <16 date bytes> subfolder}``, where a window entry is ``80 01 85 00``
11
+ (ordinal 0) or ``80 04 81 <len> <ordinal LE> 80 00`` (ordinal >= 1), and the
12
+ 0-based window-stream ordinal indexes the windows (Origin writes headers
13
+ grouped by folder, so members precede root-level ones). The two versions
14
+ differ only in *where* the count sits (4.3811: right after the ``0A 02 75 62
15
+ 0A`` "ub" attrs; 4.3380: after a ``<OriginStorage/>`` block) and the SEP bytes
16
+ (``80 12 8d 10`` / ``80 16 03 00 00 01 8a 10``), so one parser handles both: it
17
+ locates the count by scanning for ``0A <2*nwin> [00] <entries>`` validated by a
18
+ clean entry run, matches either SEP, and enumerates windows by the ``0A``-framed
19
+ page header ``0A [00] 80 <type> <namelen+2> 00 00 <name> <hi>`` — the leading
20
+ ``0A`` separates a true page header from a dataset-curve record (a graph's
21
+ ``FitLine``), ``namelen >= 2`` rejects 1-byte coincidental matches, and the type
22
+ byte is NOT used (unstable across files). The tree is rebuilt from *preorder +
23
+ per-folder child count* (a structural invariant: arbitrary depth, empty folders,
24
+ duplicate/unicode/spaced names). Validated byte-exact vs live COM on all 5
25
+ corpus files (incl. the 39-book ``Hc2 data`` with report-table windows and
26
+ nested folders) plus 11 controlled 4.3811 specimens. Fail-closed: an ordinal
27
+ past the window list, an inconsistent child count, or any framing mismatch
28
+ returns ``{}`` (flat import). Ordinals are read as 1 byte (>255 windows degrades
29
+ to flat); a 4.3380 empty *leaf* folder is recovered only via its introducing
30
+ SEP.
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import re
36
+ import struct
37
+
38
+ from quantized.io.origin_project.tree import _flatten, _FolderNode
39
+
40
+ __all__ = ["opju_folder_paths"]
41
+
42
+ # The subfolder separator precedes every child folder record: 4.3811 uses the
43
+ # short form, 4.3380 the long one. Either shape also marks "leaf" when it lands
44
+ # immediately after a folder's window entries (see _opju_peek_nsub).
45
+ _OPJU_SEPS = (b"\x80\x12\x8d\x10", b"\x80\x16\x03\x00\x00\x01\x8a\x10")
46
+ # Window/page header: 0A [00] 80 <type> <namelen+2> 00 00 <name> <hi-bit byte>.
47
+ # The leading 0A is the record framing that tells a true page header apart from
48
+ # a dataset-curve record (a graph's "FitLine" fit result lacks it). The name
49
+ # span is driven by the <namelen+2> length prefix itself — NOT by a regex
50
+ # length window (the old {1,39} bound was a corpus maximum: a >40-char or
51
+ # 1-char window name was silently skipped, and a skipped page header doesn't
52
+ # just lose that window — every column record inside its span is attributed
53
+ # to the PREVIOUS page, i.e. the wrong book; 2026-07-06 genericity audit).
54
+ # The name charset + the trailing hi-bit byte remain the validity gates.
55
+ # Order = the 0-based ordinal the folder tree references (Origin writes
56
+ # headers grouped by folder, depth-first).
57
+ _OPJU_WIN_HEAD_RE = re.compile(rb"\x0a\x00?\x80.(.)\x00\x00", re.DOTALL)
58
+ _OPJU_WIN_NAME_RE = re.compile(rb"[A-Za-z0-9][A-Za-z0-9_-]*\Z")
59
+
60
+
61
+ def iter_opju_windows(b: bytes) -> list[tuple[int, str]]:
62
+ """Every page header as ``(match_offset, name)``, file-stream order,
63
+ first occurrence per name. Shared by the folder tree (ordinal order)
64
+ and ``opju_figure_curves.opju_pages`` (span scoping)."""
65
+ out: list[tuple[int, str]] = []
66
+ seen: set[str] = set()
67
+ for m in _OPJU_WIN_HEAD_RE.finditer(b):
68
+ nlen = m.group(1)[0] - 2
69
+ if nlen < 1:
70
+ continue
71
+ raw = b[m.end() : m.end() + nlen]
72
+ if len(raw) < nlen or not _OPJU_WIN_NAME_RE.fullmatch(raw):
73
+ continue
74
+ trail = b[m.end() + nlen : m.end() + nlen + 1]
75
+ if not trail or trail[0] < 0x80:
76
+ continue
77
+ name = raw.decode("latin1")
78
+ if name not in seen:
79
+ seen.add(name)
80
+ out.append((m.start(), name))
81
+ return out
82
+
83
+
84
+ def _enumerate_opju_windows(b: bytes) -> list[str]:
85
+ """Window (worksheet/graph/table) names in file-stream = ordinal order."""
86
+ return [name for _pos, name in iter_opju_windows(b)]
87
+
88
+
89
+ def _opju_read_entries(b: bytes, q: int, nwin: int, hi: int) -> tuple[list[int] | None, int]:
90
+ """Parse ``nwin`` window entries at ``q``; ``(None, q)`` on the first byte
91
+ that isn't a valid entry — so a candidate count position whose bytes don't
92
+ form a clean entry run is rejected rather than mis-read."""
93
+ ords: list[int] = []
94
+ n = len(b)
95
+ for k in range(nwin):
96
+ if b[q : q + 3] == b"\x80\x01\x85": # ordinal 0 (short form)
97
+ ords.append(0)
98
+ q += 4
99
+ elif b[q : q + 3] == b"\x80\x04\x81": # ordinal >= 1 (1-byte tagged value)
100
+ ln = b[q + 3]
101
+ ords.append(int.from_bytes(b[q + 4 : q + 4 + ln], "little"))
102
+ q += 4 + ln + 2
103
+ else:
104
+ return None, q
105
+ if k < nwin - 1 and q < n and b[q] == 0x00: # inter-entry separator
106
+ q += 1
107
+ if q > hi:
108
+ return None, q
109
+ return ords, q
110
+
111
+
112
+ def _opju_peek_nsub(b: bytes, p: int) -> tuple[int, bool]:
113
+ """Subfolder count at ``p``: a separator here => leaf (0, it's an
114
+ ancestor's); ``<2*nsub>`` then a separator => that many children. The bool
115
+ reports whether a separator was seen (used to accept a 0-window count)."""
116
+ for sep in _OPJU_SEPS:
117
+ if b[p : p + len(sep)] == sep:
118
+ return 0, True
119
+ for sep in _OPJU_SEPS:
120
+ if p < len(b) and b[p] % 2 == 0 and b[p + 1 : p + 1 + len(sep)] == sep:
121
+ return b[p] // 2, True
122
+ return 0, False
123
+
124
+
125
+ def _parse_opju_folder_seq(b: bytes) -> list[tuple[str, list[int], int]]:
126
+ """Preorder ``(name, [ordinals], subfolder_count)`` per folder record.
127
+
128
+ Each folder's window count is found by scanning for ``0A <2*nwin> [00]
129
+ <entries>`` validated by a clean entry run — 4.3811 puts it just after the
130
+ "ub" attrs, 4.3380 after a ``<OriginStorage/>`` block, so no fixed anchor
131
+ works. A folder with no entries is accepted only when a subfolder separator
132
+ precedes its name block (a real empty folder is always introduced by one),
133
+ which keeps stray data that looks like a name block from becoming a phantom
134
+ folder. Returns ``[]`` when nothing parses (caller degrades to flat).
135
+ """
136
+ blocks: list[tuple[int, int, str]] = []
137
+ n = len(b)
138
+ i = 0
139
+ while i < n - 8:
140
+ size = int.from_bytes(b[i : i + 4], "little")
141
+ if (
142
+ 1 <= size <= 64
143
+ and b[i + 4] == 0x0A
144
+ and b[i + 4 + size] == 0x00
145
+ and b[i + 5 + size] == 0x0A
146
+ ):
147
+ blocks.append((i, i + 5 + size + 1, b[i + 5 : i + 4 + size].decode("latin1")))
148
+ i = i + 5 + size
149
+ continue
150
+ i += 1
151
+ seq: list[tuple[str, list[int], int]] = []
152
+ for bi, (start, name_end, name) in enumerate(blocks):
153
+ hi = blocks[bi + 1][0] if bi + 1 < len(blocks) else n
154
+ found: tuple[str, list[int], int] | None = None
155
+ p = name_end
156
+ while p < hi - 1:
157
+ if b[p] == 0x0A and b[p + 1] % 2 == 0:
158
+ nwin = b[p + 1] // 2
159
+ q = p + 2
160
+ if q < n and b[q] == 0x00: # optional separator before entries
161
+ q += 1
162
+ ords, qend = _opju_read_entries(b, q, nwin, hi)
163
+ if ords is not None:
164
+ nsub, has_sep = _opju_peek_nsub(b, qend)
165
+ if nwin > 0 or has_sep:
166
+ found = (name, ords, nsub)
167
+ break
168
+ p += 1
169
+ if found is None and any(sep in b[max(0, start - 28) : start] for sep in _OPJU_SEPS):
170
+ found = (name, [], 0) # SEP-introduced empty folder
171
+ if found is not None:
172
+ seq.append(found)
173
+ return seq
174
+
175
+
176
+ def _reconstruct_opju(seq: list[tuple[str, list[int], int]]) -> _FolderNode | None:
177
+ """Rebuild the folder tree from the preorder + per-folder child counts.
178
+
179
+ Returns ``None`` if the sequence isn't a consistent tree (every non-root
180
+ folder must be exactly one folder's child) — fail-closed, never a guess.
181
+ """
182
+ if sum(nsub for _n, _o, nsub in seq) != len(seq) - 1:
183
+ return None
184
+ nodes = [_FolderNode(name=nm, windows=list(ords)) for nm, ords, _ns in seq]
185
+ stack: list[list[int]] = [[0, seq[0][2]]]
186
+ idx = 1
187
+ while idx < len(seq) and stack:
188
+ frame = stack[-1]
189
+ if frame[1] <= 0:
190
+ stack.pop()
191
+ continue
192
+ frame[1] -= 1
193
+ nodes[frame[0]].subfolders.append(nodes[idx])
194
+ stack.append([idx, seq[idx][2]])
195
+ idx += 1
196
+ if idx != len(seq):
197
+ return None
198
+ return nodes[0]
199
+
200
+
201
+ def opju_folder_paths(b: bytes) -> dict[str, list[str]]:
202
+ """Map every ``.opju`` window's short name to its Project Explorer path.
203
+
204
+ Root-exclusive, same contract as :func:`quantized.io.origin_project.tree.
205
+ opj_folder_paths` (a book in the project root maps to ``[]``; strays are
206
+ absent). Decodes the CPYUA folder tree for both container sub-versions
207
+ (4.3811 and 4.3380 — see the module docstring); other/newer containers and
208
+ any framing or consistency mismatch return ``{}`` so the import degrades to
209
+ a flat project folder rather than a guess. Never raises.
210
+ """
211
+ try:
212
+ seq = _parse_opju_folder_seq(b)
213
+ if not seq:
214
+ return {}
215
+ root = _reconstruct_opju(seq)
216
+ if root is None:
217
+ return {}
218
+ names = _enumerate_opju_windows(b)
219
+ max_ord = max((o for _n, ords, _ns in seq for o in ords), default=-1)
220
+ # every referenced ordinal must resolve — otherwise the window
221
+ # enumeration is incomplete and any mapping would be wrong.
222
+ if max_ord >= len(names):
223
+ return {}
224
+ out: dict[str, list[str]] = {}
225
+ _flatten(root, names, (), out)
226
+ return out
227
+ except (IndexError, UnicodeDecodeError, struct.error, ValueError, RecursionError):
228
+ return {}