quantized-lab 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. quantized/__init__.py +11 -0
  2. quantized/__main__.py +14 -0
  3. quantized/api.py +220 -0
  4. quantized/app.py +206 -0
  5. quantized/calc/__init__.py +8 -0
  6. quantized/calc/_clipfit.py +76 -0
  7. quantized/calc/_natural_neighbor.py +219 -0
  8. quantized/calc/aggregate.py +159 -0
  9. quantized/calc/backgrounds.py +353 -0
  10. quantized/calc/baseline.py +349 -0
  11. quantized/calc/batch_fit.py +148 -0
  12. quantized/calc/constants.py +27 -0
  13. quantized/calc/corrections.py +192 -0
  14. quantized/calc/crystallography.py +400 -0
  15. quantized/calc/diffusion.py +120 -0
  16. quantized/calc/electrical.py +248 -0
  17. quantized/calc/electrochemistry.py +176 -0
  18. quantized/calc/element_data.json +1 -0
  19. quantized/calc/element_data.py +59 -0
  20. quantized/calc/errors.py +246 -0
  21. quantized/calc/figure.py +417 -0
  22. quantized/calc/figure_break.py +152 -0
  23. quantized/calc/figure_categorical.py +156 -0
  24. quantized/calc/figure_corner.py +229 -0
  25. quantized/calc/figure_facets.py +127 -0
  26. quantized/calc/figure_field.py +137 -0
  27. quantized/calc/figure_hitmap.py +116 -0
  28. quantized/calc/figure_labels.py +62 -0
  29. quantized/calc/figure_map.py +287 -0
  30. quantized/calc/figure_overrides.py +159 -0
  31. quantized/calc/figure_page.py +266 -0
  32. quantized/calc/figure_scale.py +125 -0
  33. quantized/calc/figure_statplots.py +167 -0
  34. quantized/calc/figure_styles.py +131 -0
  35. quantized/calc/figure_ternary.py +239 -0
  36. quantized/calc/figure_ticks.py +217 -0
  37. quantized/calc/fit_autoguess.py +156 -0
  38. quantized/calc/fit_bootstrap.py +163 -0
  39. quantized/calc/fit_bumps.py +258 -0
  40. quantized/calc/fit_constraints.py +114 -0
  41. quantized/calc/fit_equation.py +264 -0
  42. quantized/calc/fit_findxy.py +80 -0
  43. quantized/calc/fit_models.py +195 -0
  44. quantized/calc/fit_models_special.py +189 -0
  45. quantized/calc/fit_odr.py +99 -0
  46. quantized/calc/fit_scan.py +243 -0
  47. quantized/calc/fit_stats.py +215 -0
  48. quantized/calc/fitting.py +199 -0
  49. quantized/calc/formula.py +90 -0
  50. quantized/calc/global_curve_fit.py +305 -0
  51. quantized/calc/global_fit.py +181 -0
  52. quantized/calc/interp2d.py +260 -0
  53. quantized/calc/linecut.py +269 -0
  54. quantized/calc/magnetic.py +414 -0
  55. quantized/calc/magnetometry.py +464 -0
  56. quantized/calc/map.py +228 -0
  57. quantized/calc/mcmc.py +177 -0
  58. quantized/calc/optics.py +228 -0
  59. quantized/calc/pawley.py +251 -0
  60. quantized/calc/peak_batch.py +128 -0
  61. quantized/calc/peak_fit.py +259 -0
  62. quantized/calc/peak_integrate.py +104 -0
  63. quantized/calc/peak_multifit.py +260 -0
  64. quantized/calc/peak_track.py +134 -0
  65. quantized/calc/peaks.py +298 -0
  66. quantized/calc/peakshapes.py +85 -0
  67. quantized/calc/plotting.py +147 -0
  68. quantized/calc/processing.py +232 -0
  69. quantized/calc/qspace.py +48 -0
  70. quantized/calc/reductions.py +155 -0
  71. quantized/calc/reductions_fft.py +383 -0
  72. quantized/calc/refl_sld_presets.json +1 -0
  73. quantized/calc/reflectivity.py +80 -0
  74. quantized/calc/registry.py +303 -0
  75. quantized/calc/relaxation.py +119 -0
  76. quantized/calc/report.py +253 -0
  77. quantized/calc/report_emit.py +227 -0
  78. quantized/calc/resample.py +142 -0
  79. quantized/calc/rsm.py +91 -0
  80. quantized/calc/rsm_analyze.py +245 -0
  81. quantized/calc/semiconductor.py +488 -0
  82. quantized/calc/sld.py +131 -0
  83. quantized/calc/sld_formula.py +138 -0
  84. quantized/calc/spectral.py +357 -0
  85. quantized/calc/statplots.py +214 -0
  86. quantized/calc/stats.py +399 -0
  87. quantized/calc/stats_anova2.py +202 -0
  88. quantized/calc/stats_anova_ext.py +338 -0
  89. quantized/calc/stats_dist.py +196 -0
  90. quantized/calc/stats_glm.py +245 -0
  91. quantized/calc/stats_multivar.py +289 -0
  92. quantized/calc/stats_roc.py +157 -0
  93. quantized/calc/stats_survival.py +261 -0
  94. quantized/calc/stats_tests.py +380 -0
  95. quantized/calc/substrates.py +181 -0
  96. quantized/calc/superconductor.py +359 -0
  97. quantized/calc/surface_fit.py +290 -0
  98. quantized/calc/surface_models.py +156 -0
  99. quantized/calc/thermal.py +119 -0
  100. quantized/calc/thin_film.py +425 -0
  101. quantized/calc/unit_convert.py +259 -0
  102. quantized/calc/units.py +80 -0
  103. quantized/calc/vacuum.py +290 -0
  104. quantized/calc/xray.py +169 -0
  105. quantized/cli.py +214 -0
  106. quantized/datastruct.py +153 -0
  107. quantized/io/__init__.py +11 -0
  108. quantized/io/_hdf5_layout.py +308 -0
  109. quantized/io/_jcamp_asdf.py +135 -0
  110. quantized/io/_xrdml_scan.py +291 -0
  111. quantized/io/base.py +82 -0
  112. quantized/io/bruker_brml.py +177 -0
  113. quantized/io/bruker_raw.py +158 -0
  114. quantized/io/cif.py +266 -0
  115. quantized/io/consolidated.py +122 -0
  116. quantized/io/delimited.py +222 -0
  117. quantized/io/excel.py +135 -0
  118. quantized/io/hdf5.py +192 -0
  119. quantized/io/import_filters.py +178 -0
  120. quantized/io/import_preview.py +262 -0
  121. quantized/io/jcamp.py +179 -0
  122. quantized/io/lakeshore.py +163 -0
  123. quantized/io/ncnr.py +278 -0
  124. quantized/io/netcdf.py +195 -0
  125. quantized/io/opus.py +231 -0
  126. quantized/io/origin.py +346 -0
  127. quantized/io/origin_com.py +194 -0
  128. quantized/io/origin_project/__init__.py +221 -0
  129. quantized/io/origin_project/annotation_marks.py +288 -0
  130. quantized/io/origin_project/container.py +262 -0
  131. quantized/io/origin_project/curve_style_color.py +359 -0
  132. quantized/io/origin_project/figure_geometry.py +108 -0
  133. quantized/io/origin_project/figure_layers.py +333 -0
  134. quantized/io/origin_project/figure_text.py +258 -0
  135. quantized/io/origin_project/figures.py +210 -0
  136. quantized/io/origin_project/figures_opju.py +440 -0
  137. quantized/io/origin_project/notes.py +302 -0
  138. quantized/io/origin_project/opj.py +459 -0
  139. quantized/io/origin_project/opj_curves.py +297 -0
  140. quantized/io/origin_project/opj_shapes.py +148 -0
  141. quantized/io/origin_project/opju.py +146 -0
  142. quantized/io/origin_project/opju_axis_real_form.py +418 -0
  143. quantized/io/origin_project/opju_axis_specimen_form.py +167 -0
  144. quantized/io/origin_project/opju_codec.py +370 -0
  145. quantized/io/origin_project/opju_curves.py +497 -0
  146. quantized/io/origin_project/opju_curves_allcols.py +258 -0
  147. quantized/io/origin_project/opju_figure_curves.py +302 -0
  148. quantized/io/origin_project/opju_figure_text.py +245 -0
  149. quantized/io/origin_project/opju_reports.py +129 -0
  150. quantized/io/origin_project/origin_richtext.py +145 -0
  151. quantized/io/origin_project/preview.py +132 -0
  152. quantized/io/origin_project/templates.py +314 -0
  153. quantized/io/origin_project/tree.py +379 -0
  154. quantized/io/origin_project/tree_opju.py +228 -0
  155. quantized/io/origin_project/windows.py +238 -0
  156. quantized/io/origin_project/windows_opju.py +393 -0
  157. quantized/io/origin_project/writer.py +156 -0
  158. quantized/io/origin_project/writer_blocks.py +282 -0
  159. quantized/io/qd.py +380 -0
  160. quantized/io/refl1d.py +132 -0
  161. quantized/io/registry.py +210 -0
  162. quantized/io/report_export.py +347 -0
  163. quantized/io/rigaku.py +100 -0
  164. quantized/io/sims.py +398 -0
  165. quantized/io/spc.py +311 -0
  166. quantized/io/xrd_csv.py +308 -0
  167. quantized/io/xrdml.py +394 -0
  168. quantized/jobs.py +173 -0
  169. quantized/plugins/__init__.py +50 -0
  170. quantized/plugins/contract.py +111 -0
  171. quantized/plugins/loader.py +394 -0
  172. quantized/plugins/steps.py +90 -0
  173. quantized/routes/__init__.py +7 -0
  174. quantized/routes/_bookcache.py +62 -0
  175. quantized/routes/_export_common.py +27 -0
  176. quantized/routes/_payload.py +58 -0
  177. quantized/routes/_uploadcache.py +59 -0
  178. quantized/routes/aggregate.py +46 -0
  179. quantized/routes/baseline.py +210 -0
  180. quantized/routes/books.py +117 -0
  181. quantized/routes/calc.py +56 -0
  182. quantized/routes/corrections.py +78 -0
  183. quantized/routes/crystallography.py +80 -0
  184. quantized/routes/diffusion.py +58 -0
  185. quantized/routes/electrical.py +101 -0
  186. quantized/routes/electrochemistry.py +83 -0
  187. quantized/routes/export.py +280 -0
  188. quantized/routes/export_facets.py +83 -0
  189. quantized/routes/export_figures.py +471 -0
  190. quantized/routes/export_page.py +125 -0
  191. quantized/routes/fitting.py +379 -0
  192. quantized/routes/fitting_bumps.py +97 -0
  193. quantized/routes/import_template.py +97 -0
  194. quantized/routes/import_wizard.py +150 -0
  195. quantized/routes/jobs_api.py +59 -0
  196. quantized/routes/magnetic.py +135 -0
  197. quantized/routes/magnetometry.py +133 -0
  198. quantized/routes/optics.py +98 -0
  199. quantized/routes/parsers.py +281 -0
  200. quantized/routes/peaks.py +184 -0
  201. quantized/routes/plot.py +103 -0
  202. quantized/routes/reductions.py +121 -0
  203. quantized/routes/reference.py +58 -0
  204. quantized/routes/reflectivity.py +91 -0
  205. quantized/routes/report_export.py +119 -0
  206. quantized/routes/rsm.py +136 -0
  207. quantized/routes/samples.py +32 -0
  208. quantized/routes/semiconductor.py +207 -0
  209. quantized/routes/sld.py +42 -0
  210. quantized/routes/spectral.py +54 -0
  211. quantized/routes/statplots.py +99 -0
  212. quantized/routes/stats.py +418 -0
  213. quantized/routes/stats_design.py +321 -0
  214. quantized/routes/substrates.py +46 -0
  215. quantized/routes/superconductor.py +139 -0
  216. quantized/routes/thermal.py +57 -0
  217. quantized/routes/thin_film.py +153 -0
  218. quantized/routes/vacuum.py +113 -0
  219. quantized/routes/xray.py +32 -0
  220. quantized/samples/demo_vsm.csv +42 -0
  221. quantized/server_launch.py +251 -0
  222. quantized/web/assets/JetBrainsMono-Bold-CUogYd9I.woff2 +0 -0
  223. quantized/web/assets/JetBrainsMono-Regular-CA-Os4ii.woff2 +0 -0
  224. quantized/web/assets/index-BHmmCL-x.js +27 -0
  225. quantized/web/assets/index-BiZzN7J6.css +1 -0
  226. quantized/web/index.html +13 -0
  227. quantized/web/loading.html +69 -0
  228. quantized_lab-0.8.0.dist-info/METADATA +122 -0
  229. quantized_lab-0.8.0.dist-info/RECORD +233 -0
  230. quantized_lab-0.8.0.dist-info/WHEEL +4 -0
  231. quantized_lab-0.8.0.dist-info/entry_points.txt +4 -0
  232. quantized_lab-0.8.0.dist-info/licenses/LICENSE +201 -0
  233. quantized_lab-0.8.0.dist-info/licenses/NOTICE +11 -0
@@ -0,0 +1,238 @@
1
+ """Extract per-column metadata from the ``.opj`` windows section.
2
+
3
+ Worksheet *window definitions* live in the same block stream as the column
4
+ data, after the datasets section. Each worksheet window opens with a header
5
+ block (``00 00 <BookShort> 00 …``, long name ending at the ``@${`` storage
6
+ marker) and contains, per column, a **property block** (≥500 B; designation at
7
+ ``0x11``, NUL-terminated short name at ``0x12``) immediately followed by a
8
+ **label-text block** (``LongName\\r\\nUnit\\r\\nComment``, cut at ``@${``).
9
+ Column ``S`` of book ``B`` maps to the dataset named ``"B_S"``.
10
+
11
+ A property block's byte 0x06 varies with the column's storage flavour --
12
+ ``0x09`` for a plain sheet-1 column, ``0x0B`` for a formula/derived column
13
+ *and* for every column of an auto-generated report sheet (FitLinear/FitNL's
14
+ "Parameters"/"Notes"/etc. sheets) -- so it does not distinguish sheet
15
+ identity; both values are accepted (plan item: report-sheet leak fix).
16
+
17
+ **Multi-sheet books** (a workbook with a report/curve sheet appended after
18
+ the real data, e.g. Origin's FitLinear auto-adds "FitLinear1"/
19
+ "FitLinearCurve1" siblings to the sheet it fits) restart column lettering at
20
+ "A" for each extra sheet, and *also* reuse the very same 0x0B property-block
21
+ shape as a real formula column -- so neither the storage-flavour byte nor a
22
+ short-name repeat is a reliable, early sheet-boundary signal on its own (a
23
+ report sheet's own property/label blocks can front-run a repeat detection
24
+ by many columns). The real signal is a fixed 365-byte **sheet/layer
25
+ sub-header** block carrying ``Pd<Name>\\0`` at a constant offset (0xD0) --
26
+ one appears at the very start of every worksheet sheet (and every graph
27
+ layer, tagged ``Pd1``/``Pd2``/…) inside a window's block span. The *second*
28
+ one seen since the enclosing window header is the true start of sheet 2+;
29
+ everything from there on (property blocks, labels, formulas) is excluded.
30
+ Validated against Moke.opj's ``Book4`` (Sheet1 / FitLinear1 /
31
+ FitLinearCurve1 — three ``Pd...`` markers, one per sheet, exactly bracketing
32
+ each sheet's real column-property-block run) and every single-sheet corpus
33
+ book (exactly one marker, at the window's very start). Older-format (CPYA
34
+ 4.3227) files carry no such marker at all; the short-name-repeat guard is
35
+ kept as a fallback for that case (unverified whether any 4.3227 file in the
36
+ corpus has a real multi-sheet book -- none observed to date).
37
+
38
+ Byte layout validated against the local corpus — see
39
+ ``docs/origin_re/opj_windows_section.md`` (plan item 1).
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import re
45
+ from dataclasses import dataclass, field
46
+
47
+ from quantized.io.origin_project.container import walk_blocks
48
+
49
+ __all__ = ["BookMeta", "ColumnMeta", "window_metadata"]
50
+
51
+ # Origin's published plot-designation enum (format fact).
52
+ _DESIGNATION = {
53
+ 0: "Y",
54
+ 1: "disregard",
55
+ 2: "Y-error",
56
+ 3: "X",
57
+ 4: "label",
58
+ 5: "Z",
59
+ 6: "X-error",
60
+ }
61
+
62
+
63
+ @dataclass(frozen=True)
64
+ class ColumnMeta:
65
+ short: str
66
+ designation: str
67
+ long_name: str = ""
68
+ unit: str = ""
69
+ comment: str = ""
70
+
71
+
72
+ @dataclass
73
+ class BookMeta:
74
+ short: str
75
+ long_name: str
76
+ columns: dict[str, ColumnMeta] = field(default_factory=dict)
77
+
78
+
79
+ def _cstring(payload: bytes, start: int, limit: int = 48) -> str | None:
80
+ """Printable NUL-terminated ASCII at ``start``, or None."""
81
+ end = payload.find(b"\x00", start, start + limit)
82
+ if end <= start:
83
+ return None
84
+ raw = payload[start:end]
85
+ if not all(0x20 <= c < 0x7F for c in raw):
86
+ return None
87
+ return raw.decode("latin1")
88
+
89
+
90
+ def _is_window_header(payload: bytes) -> str | None:
91
+ """A window-header block starts ``00 00 <Name> 00``; return the name.
92
+
93
+ Digit-led names are real (``tree.py``'s module docstring documents
94
+ ``30nmADPNR`` and six digit-led graph names in the corpus, COM-verified),
95
+ so the first-char gate is ``isalnum``, not ``isalpha`` — the old
96
+ alpha-first check silently hid digit-led graph windows from
97
+ ``figures.py`` AND let their internal blocks poison
98
+ ``opj_curves.column_id_map`` (a digit-led graph was not recognized as a
99
+ graph, so its 519-byte curve records were scanned as column storage)."""
100
+ if len(payload) < 150 or payload[0] or payload[1]:
101
+ return None
102
+ name = _cstring(payload, 2, 64)
103
+ return name if name and name[0].isalnum() else None
104
+
105
+
106
+ def _book_long_name(payload: bytes, short: str) -> str:
107
+ """The display title, stored in the header tail.
108
+
109
+ v4.3380 headers end the title at an ``@${…}<OriginStorage>`` marker;
110
+ v4.3227 headers have no storage blob — the title is simply the last
111
+ printable run after the fixed-format region (offset > 0x60).
112
+ """
113
+ anchor = payload.find(b"@${")
114
+ if anchor >= 0:
115
+ start = payload.rfind(b"\x00", 0, anchor) + 1
116
+ raw = payload[start:anchor]
117
+ if raw and all(0x20 <= c < 0x7F for c in raw):
118
+ return raw.decode("latin1")
119
+ return short
120
+ runs = [m for m in re.finditer(rb"[\x20-\x7e]{2,}", payload) if m.start() > 0x60]
121
+ if runs:
122
+ cand = runs[-1].group().decode("latin1")
123
+ if cand != short:
124
+ return cand
125
+ return short
126
+
127
+
128
+ def _is_column_block(payload: bytes) -> bool:
129
+ """Robust, version-independent column-property block detector.
130
+
131
+ Byte 0x06 is ``0x09`` for a plain (never-recalculated) sheet-1 column and
132
+ ``0x0B`` for a formula/derived column or any report-sheet column -- both
133
+ are real column-property blocks, so both are accepted (a Moke.opj Book4
134
+ Sheet1 measurement had 14/15 columns at ``0x09``; only its one formula
135
+ column, "Normalized1", was ``0x0B`` -- the old ``==0x0B``-only check
136
+ dropped those 14 silently instead of mapping them).
137
+ """
138
+ return (
139
+ len(payload) >= 500
140
+ and payload[0x06] in (0x09, 0x0B)
141
+ and payload[0x25] in (0x21, 0x30) # 0x30 observed on Y-error columns
142
+ and _cstring(payload, 0x12, 8) is not None
143
+ )
144
+
145
+
146
+ _SHEET_HEADER_SIZE = 365
147
+ _SHEET_NAME_OFFSET = 0xD0 # 208: fixed offset of the "Pd<Name>" marker
148
+
149
+
150
+ def _is_sheet_header(payload: bytes) -> str | None:
151
+ """A per-sheet (or per-graph-layer) sub-header: the real sheet-boundary
152
+ signal (see module docstring). Always exactly 365 B, carrying
153
+ NUL-terminated ``Pd<Name>`` at a fixed offset -- never observed to
154
+ collide with a window-header or column-property block's size/shape
155
+ across the local corpus (window headers range 195-359 B in every
156
+ corpus file; a column-property block is always >=500 B)."""
157
+ if len(payload) != _SHEET_HEADER_SIZE:
158
+ return None
159
+ if payload[_SHEET_NAME_OFFSET : _SHEET_NAME_OFFSET + 2] != b"Pd":
160
+ return None
161
+ return _cstring(payload, _SHEET_NAME_OFFSET + 2, 40)
162
+
163
+
164
+ def _label_rows(payload: bytes) -> tuple[str, str, str]:
165
+ """Parse a label-text block: LongName / Unit / Comment (missing → "")."""
166
+ text = payload.split(b"\x00", 1)[0]
167
+ cut = text.find(b"@${")
168
+ if cut >= 0:
169
+ text = text[:cut]
170
+ rows = text.decode("latin1", errors="replace").split("\r\n")
171
+ rows += ["", "", ""]
172
+ return rows[0], rows[1], rows[2]
173
+
174
+
175
+ def window_metadata(b: bytes) -> dict[str, BookMeta]:
176
+ """Map book short name → :class:`BookMeta` with per-column names/units.
177
+
178
+ Only primary-sheet columns are mapped for now (multi-sheet ``@N`` datasets
179
+ are plan item 5). Graph windows contain no column blocks and drop out
180
+ naturally.
181
+
182
+ A report/curve sheet appended after the primary sheet (Origin's
183
+ FitLinear/FitNL auto-add, e.g. "FitLinear1"/"FitLinearCurve1") must never
184
+ reach ``current.columns`` — that dict is the *primary-sheet* map. The
185
+ ``Pd<Name>`` sheet sub-header (`_is_sheet_header`) is the precise
186
+ boundary: the second one seen since the enclosing window header closes
187
+ collection immediately, before that sheet's first property block. The
188
+ short-name-repeat check stays as a fallback for the older container
189
+ version that carries no such marker (plan item 5's original guard).
190
+ """
191
+ books: dict[str, BookMeta] = {}
192
+ current: BookMeta | None = None
193
+ pending: tuple[str, str] | None = None # (short, designation) awaiting label
194
+ closed = False # set once the current window moves past its primary sheet
195
+ sheets_seen = 0 # count of Pd<Name> markers since the current window header
196
+
197
+ def commit(long_name: str = "", unit: str = "", comment: str = "") -> None:
198
+ nonlocal pending, closed
199
+ if pending is not None and current is not None and not closed:
200
+ short, desig = pending
201
+ if short in current.columns:
202
+ # A repeated short means sheet 2+ started (every sheet restarts
203
+ # at column A); only the primary sheet maps to plain
204
+ # "<Book>_<Col>" datasets, so stop collecting (plan item 5;
205
+ # fallback for containers with no Pd<Name> marker).
206
+ closed = True
207
+ else:
208
+ current.columns[short] = ColumnMeta(short, desig, long_name, unit, comment)
209
+ pending = None
210
+
211
+ for size, payload in walk_blocks(b):
212
+ if size == 0:
213
+ continue
214
+ header_name = _is_window_header(payload)
215
+ sheet_name = _is_sheet_header(payload) if header_name is None else None
216
+ is_col = (
217
+ _is_column_block(payload) if header_name is None and sheet_name is None else False
218
+ )
219
+ if pending is not None:
220
+ if header_name is None and sheet_name is None and not is_col and size < 500:
221
+ commit(*_label_rows(payload)) # the label block for the pending column
222
+ continue
223
+ commit() # structural block follows — column had no label text
224
+ if header_name is not None:
225
+ current = books.setdefault(
226
+ header_name, BookMeta(header_name, _book_long_name(payload, header_name))
227
+ )
228
+ closed = False
229
+ sheets_seen = 0
230
+ elif sheet_name is not None:
231
+ sheets_seen += 1
232
+ if sheets_seen > 1: # the 2nd+ sheet/layer marker: stop mapping columns
233
+ closed = True
234
+ elif current is not None and is_col and not closed:
235
+ short = _cstring(payload, 0x12, 8) or "?"
236
+ pending = (short, _DESIGNATION.get(payload[0x11], "Y"))
237
+ commit()
238
+ return books
@@ -0,0 +1,393 @@
1
+ """Extract per-column long-names/units from the ``.opju`` (CPYUA) windows section.
2
+
3
+ The CPYUA container's worksheet windows section is not `.opj`'s CPY block
4
+ stream (see ``docs/origin_re/opj_windows_section.md``) — it uses its own
5
+ tag/length framing that this module does not fully parse. What IS pinned,
6
+ validated against Origin's own exported ground truth across five real
7
+ corpus files (XAS, RockingCurve, UnpolPlots, "Fixed Lambdas SI", plus the
8
+ ``rosetta_*`` specimens — 145/145 names, units, and comments matched):
9
+
10
+ * Every worksheet column carries a 2-byte **plot-designation marker**:
11
+ ``21 51`` for X, ``21 61`` for Y, ``30 61`` for Y-error — the same
12
+ marker-byte + display-code convention `.opj` uses at property-block
13
+ offsets 0x25/0x26 (see ``opj_windows_section.md`` sec 4.1), reused inside
14
+ CPYUA's own framing.
15
+ * A fixed-shape run (default column-format doubles) follows every marker,
16
+ then an OPTIONAL length-prefixed embedded blob (``<len><tag=0x01><bytes>
17
+ <NUL>``) carrying that column's ``ColumnInfo``/``ImportFile`` storage
18
+ (only present for imported-file columns — a long file path, possibly
19
+ using Origin's internal string back-reference shorthand which this
20
+ module never tries to decode), then the REAL label record in the SAME
21
+ ``<len:u8><tag:u8><text><NUL>`` shape (``len`` counts tag+text+NUL).
22
+ ``text`` splits on ``\\r\\n`` into long_name/unit/comment (0-3 rows); a
23
+ zero-length text (tag ``0x01``, no ``\\r\\n``) means "no label" (e.g. an
24
+ unlabeled Y column) and some columns carry only a bare long name with no
25
+ ``\\r\\n`` at all (single-row form).
26
+ * Every column emits its own marker (+ optional label) record, in true
27
+ sheet column order (A, B, C, ...) — INCLUDING columns that never decode
28
+ as data (e.g. a blank/text column between two decoded numeric ones).
29
+ That is why association is by ORDINAL POSITION within one book's
30
+ contiguous marker run, mapped through Origin's column lettering
31
+ (A, B, ... Z, AA, AB, ... — wide measurement sheets run well past Z),
32
+ rather than by parsing an internal short-name field — no such field was
33
+ pinned for CPYUA (unlike `.opj`'s offset-0x12 short name). Each book is
34
+ anchored INDEPENDENTLY (not via a forward-only cursor): a project's book
35
+ windows are not in decoded-book order, so a monotonic cursor drops every
36
+ book whose window sits before an already-anchored one (the Hc2 project
37
+ interleaves 30+ of them). Anchors are exact, so independent search is safe.
38
+ * Each book's own marker run is anchored via ONE OF: (a) the embedded
39
+ ``ColumnInfo``/``ImportFile`` path's filename, alnum-stripped and matched
40
+ against the book's known short name (handles Origin's habit of dropping
41
+ underscores when deriving a book short name from an imported filename,
42
+ e.g. ``bl11_YIGPy_032.dat`` -> book ``bl11YIGPy032``); or (b) a
43
+ ``<len=namelen+2> 00 00 <name>`` window/book-header reference that
44
+ appears even for books never imported from a file (manually-typed
45
+ sheets, e.g. the ``rosetta_*`` specimens).
46
+
47
+ Positional guessing is NOT used to *detect* a label: every accepted record
48
+ matches the exact ``<len><tag><text><NUL>`` byte count PLUS a character-
49
+ class + known-internal-token filter (rejects embedded blob fragments like
50
+ a truncated ``ResultsLog``/``OriginStorage`` token, which the length-prefix
51
+ match alone can accidentally land inside). Association across a book's
52
+ columns IS positional, but only after that book's boundary is independently
53
+ confirmed by anchor (a) or (b) above — never by scanning the whole file for
54
+ ASCII runs. When no anchor is found, or the marker run doesn't cover every
55
+ column ``opju_codec.scan_columns`` actually decoded for that book, the book
56
+ is left out of the result entirely (A/B/C fallback stays in force) rather
57
+ than guessed at.
58
+
59
+ See ``docs/origin_re/opju_container.md`` for the full byte-level trail.
60
+ """
61
+
62
+ from __future__ import annotations
63
+
64
+ import re
65
+ from collections.abc import Mapping, Sequence
66
+
67
+ from quantized.io.origin_project.opju_codec import tail_start
68
+ from quantized.io.origin_project.windows import BookMeta, ColumnMeta
69
+
70
+ __all__ = ["opju_window_metadata"]
71
+
72
+ _X_MARK = b"\x21\x51"
73
+ _Y_MARK = b"\x21\x61"
74
+ _YERR_MARK = b"\x30\x61"
75
+ # Label / Z / X-error markers, pinned 2026-07-06 by the designations.opju
76
+ # by-construction specimen (one column per designation): Label = `21 41`;
77
+ # Z and X-error SHARE `20 61` and are told apart by the record's own
78
+ # designation code byte (`82 02 <code>`: 1=label 2=Y-error 3=X 4=Z
79
+ # 5=X-error; a plain Y stores no code field at all). Before these, a book
80
+ # containing ANY such column failed the cover-check and silently lost its
81
+ # ENTIRE metadata run (names, units, and the X designation) — the audit's
82
+ # whole-book blast radius (#10).
83
+ _LABEL_MARK = b"\x21\x41"
84
+ _ZX_MARK = b"\x20\x61" # Z or X-error; resolved via the code byte
85
+ _DESIGNATION = {
86
+ "X": "X",
87
+ "Y": "Y",
88
+ "Y-error": "Y-error",
89
+ "label": "label",
90
+ "Z": "Z",
91
+ "X-error": "X-error",
92
+ }
93
+
94
+ # A backslash-delimited filename token, as embedded in a column's ImportFile path.
95
+ _FILENAME_RE = re.compile(rb"\\([\w \-]{1,60}?\.[A-Za-z0-9]{2,5})(?=[^\w.]|$)")
96
+ # A plausible bare long-name (no unit/comment row): starts with a letter, then
97
+ # the punctuation real scientific column names use -- parens/brackets/colons/
98
+ # underscores (e.g. "Nb Hc2 (T)", "NiBi_3::R", "temperature: T1",
99
+ # "multi[0]:iterator"). Still excludes path/markup chars (backslash, <, >), which
100
+ # the _JUNK_MARKERS filter rejects separately, so a length-prefix coincidence
101
+ # landing inside a storage blob is still caught. Checked on the DECODED string
102
+ # (see _is_single_row_label) so non-ASCII scientific glyphs -- degree, Angstrom,
103
+ # micro, ohm, Greek, sub/superscripts -- are accepted; the .opju container is
104
+ # UTF-8, so "tilt 45<deg>" arrives as bytes c2 b0 and a byte-only ASCII regex
105
+ # both rejected it and (via latin-1) mojibaked it (2026-07-06 genericity audit).
106
+ _LABEL_PUNCT = frozenset(" _:()[]./+%#*,-")
107
+ # Text containing any of these is inside an embedded storage blob, not a real label.
108
+ _JUNK_MARKERS = (b"\\", b"OriginStorage", b"ColumnInfo", b"ImportFile", b"<", b">")
109
+ # Single-row candidates that are themselves a fragment of one of these internal
110
+ # tokens (e.g. a length-prefix match landing mid-string, "esultsLog") are noise.
111
+ _KNOWN_TOKENS = (
112
+ b"ResultsLog",
113
+ b"OriginStorage",
114
+ b"ColumnInfo",
115
+ b"ImportFile",
116
+ b"TREE",
117
+ b"Organizer",
118
+ b"IMGEXP",
119
+ b"AXISTYPE",
120
+ b"Script",
121
+ b"History",
122
+ b"PConst",
123
+ )
124
+ _MAX_GAP = 600 # max byte spacing between one column-property record and the next
125
+
126
+
127
+ def _zx_designation(b: bytes, mark_pos: int) -> str:
128
+ """Resolve the shared ``20 61`` marker to Z or X-error via the record's
129
+ own designation code (``82 02 <code>`` a few bytes earlier: 4=Z,
130
+ 5=X-error). An unresolvable record reads as "Z" (either way the column
131
+ is a non-plottable auxiliary; the marker still COUNTS the column, which
132
+ is what protects the book's metadata run)."""
133
+ j = b.rfind(b"\x82\x02", max(0, mark_pos - 16), mark_pos)
134
+ if j >= 0 and j + 2 < len(b) and b[j + 2] == 5:
135
+ return "X-error"
136
+ return "Z"
137
+
138
+
139
+ def _decode_cell_text(raw: bytes) -> str | None:
140
+ """Decode a column-label cell, or ``None`` if it isn't text.
141
+
142
+ The ``.opju`` container stores text as UTF-8 (degree = ``c2 b0``, micro =
143
+ ``c2 b5``, Greek mu = ``ce bc``); fall back to latin-1 for the rare cell
144
+ that is valid latin-1 but not UTF-8. Reject any cell holding a C0 control
145
+ other than TAB/CR/LF — that is a binary/format block, never a label."""
146
+ for enc in ("utf-8", "latin1"):
147
+ try:
148
+ s = raw.decode(enc)
149
+ except UnicodeDecodeError:
150
+ continue
151
+ if any(ord(c) < 0x20 and c not in "\t\r\n" for c in s):
152
+ return None
153
+ return s
154
+ return None
155
+
156
+
157
+ def _is_single_row_label(s: str) -> bool:
158
+ """Whether ``s`` is a plausible bare long-name (see ``_LABEL_PUNCT``).
159
+
160
+ Unicode-aware: the first char is any letter (incl. Greek/Cyrillic) and the
161
+ rest are alphanumerics, the scientific punctuation set, or ANY non-ASCII
162
+ printable (``ord >= 0x80``) — the glyphs real column names use. Markup/path
163
+ chars (``\\``, ``<``, ``>``) are ASCII and not in the set, so they still
164
+ fail here on top of the ``_JUNK_MARKERS`` pre-filter."""
165
+ if not s or not s[0].isalpha() or len(s) > 60:
166
+ return False
167
+ return all(c.isalnum() or c in _LABEL_PUNCT or ord(c) >= 0x80 for c in s)
168
+
169
+
170
+ def _strip_alnum(s: bytes) -> bytes:
171
+ return re.sub(rb"[^A-Za-z0-9]", b"", s)
172
+
173
+
174
+ def _excel_col(i: int) -> str:
175
+ """0-based column index -> Origin/Excel column letter (0->A, 25->Z, 26->AA...)."""
176
+ letters = ""
177
+ i += 1
178
+ while i:
179
+ i, rem = divmod(i - 1, 26)
180
+ letters = chr(ord("A") + rem) + letters
181
+ return letters
182
+
183
+
184
+ def _filename_anchors(b: bytes, start: int) -> list[tuple[int, bytes, bytes]]:
185
+ """``(position, alnum-stripped-basename, raw-filename)`` per backslash token."""
186
+ out: list[tuple[int, bytes, bytes]] = []
187
+ for m in _FILENAME_RE.finditer(b):
188
+ if m.start() < start:
189
+ continue
190
+ raw = m.group(1)
191
+ base = raw.rsplit(b".", 1)[0]
192
+ out.append((m.start(), _strip_alnum(base), raw))
193
+ return out
194
+
195
+
196
+ def _book_anchor(
197
+ b: bytes, short: str, search_from: int, filename_hits: list[tuple[int, bytes, bytes]]
198
+ ) -> tuple[int, bytes | None] | None:
199
+ """First ``(position, raw_filename_or_None)`` identifying this book's window section."""
200
+ short_b = short.encode("latin1")
201
+ for pos, stripped, raw in filename_hits:
202
+ if pos >= search_from and stripped == short_b:
203
+ return pos, raw
204
+ header_len = len(short_b) + 2
205
+ if header_len < 256:
206
+ idx = b.find(bytes([header_len]) + b"\x00\x00" + short_b, search_from)
207
+ if idx >= 0:
208
+ return idx, None
209
+ if len(short) >= 4: # storage-tail reference; too ambiguous for short names alone
210
+ storage_len = len(short_b) + 1
211
+ if storage_len < 256:
212
+ idx = b.find(bytes([storage_len]) + short_b + b"\x00", search_from)
213
+ if idx >= 0:
214
+ return idx, None
215
+ return None
216
+
217
+
218
+ def _gap_text_span(b: bytes, start: int, end: int) -> int:
219
+ """Longest run of consecutive text bytes in ``[start, end)``.
220
+
221
+ A text byte is printable ASCII / TAB / CR / LF, or a UTF-8 high byte
222
+ (>= 0x80 — a label glyph). Used only to WIDEN the inter-marker gap
223
+ allowance by the label content actually present, so a long column comment
224
+ (a long contiguous printable run) can't split a book's metadata run. A
225
+ real book boundary is a multi-KB BINARY window header — no comparable
226
+ printable run — so this never merges two books. Framing-agnostic on
227
+ purpose: the comment's length prefix is a multi-byte varint we don't parse
228
+ here, and we don't need to."""
229
+
230
+ def _is_text(c: int) -> bool:
231
+ return c in (0x09, 0x0A, 0x0D) or 0x20 <= c <= 0x7E or c >= 0x80
232
+
233
+ best = run = 0
234
+ for c in b[start : min(end, len(b))]:
235
+ run = run + 1 if _is_text(c) else 0
236
+ if run > best:
237
+ best = run
238
+ return best
239
+
240
+
241
+ def _parse_label_record(b: bytes, p: int) -> bytes | None:
242
+ """One label record at ``p``: ``<LEB128 length><chunks>`` where each chunk
243
+ is ``<len:1><data>``, non-final chunks are exactly 127 bytes, and the
244
+ concatenated data is ``<text><NUL>``.
245
+
246
+ The common short label is the single-chunk case (the old fixed-shape
247
+ read). A >127-byte label — a long column comment — stores a 2-byte
248
+ varint length and 127-byte chunking; the old single-byte read silently
249
+ dropped that column's WHOLE label (long-name included), the §13.2 #13
250
+ residual, pinned by the ``long_comment.opju`` specimen (record:
251
+ varint ``e8 05`` = 744, chunks 127×5 + 103, text 737)."""
252
+ length = b[p]
253
+ off = 1
254
+ if length & 0x80: # LEB128 continuation bit -> 2-byte varint
255
+ if p + 2 > len(b):
256
+ return None
257
+ length = (length & 0x7F) | (b[p + 1] << 7)
258
+ off = 2
259
+ if length <= 0x7F or length > 4096: # a real 2-byte varint is >127
260
+ return None
261
+ elif length < 2:
262
+ return None
263
+ end_rec = p + off + length
264
+ if end_rec > len(b):
265
+ return None
266
+ payload = b[p + off : end_rec]
267
+ data = bytearray()
268
+ i = 0
269
+ while i < len(payload):
270
+ clen = payload[i]
271
+ last = i + 1 + clen == len(payload)
272
+ if clen == 0 or (not last and clen != 0x7F) or i + 1 + clen > len(payload):
273
+ return None
274
+ data += payload[i + 1 : i + 1 + clen]
275
+ i += 1 + clen
276
+ if not data or data[-1] != 0:
277
+ return None
278
+ return bytes(data[:-1])
279
+
280
+
281
+ def _find_label(b: bytes, start: int, end: int) -> str | None:
282
+ """First label-shaped record in ``[start, end)``.
283
+
284
+ Prefers a ``\\r\\n``-split multi-row label (long_name/unit/comment); falls
285
+ back to a single-row (long-name-only) record when no multi-row match
286
+ exists in range.
287
+ """
288
+ multi = single = None
289
+ limit = min(end, len(b) - 2)
290
+ for p in range(start, limit):
291
+ text = _parse_label_record(b, p)
292
+ if text is None or not text:
293
+ continue
294
+ if any(marker in text for marker in _JUNK_MARKERS):
295
+ continue
296
+ decoded = _decode_cell_text(text)
297
+ if decoded is None:
298
+ continue
299
+ if "\r\n" in decoded:
300
+ multi = decoded
301
+ break
302
+ if single is None and _is_single_row_label(decoded):
303
+ if len(text) >= 4 and any(text in token for token in _KNOWN_TOKENS):
304
+ continue
305
+ single = decoded
306
+ return multi if multi is not None else single
307
+
308
+
309
+ def opju_window_metadata(
310
+ b: bytes, book_columns: Mapping[str, Sequence[str]]
311
+ ) -> dict[str, BookMeta]:
312
+ """Map book short name -> :class:`~quantized.io.origin_project.windows.BookMeta`.
313
+
314
+ ``book_columns`` is the already-decoded ``{book: [col_letter, ...]}``
315
+ ordering (see ``opj._group``) — this function never invents a column
316
+ that ``opju_codec.scan_columns`` didn't independently decode; it only
317
+ attaches a label to it when the book's marker run can be confirmed.
318
+ """
319
+ start = tail_start(b)
320
+ markers = sorted(
321
+ [(m.start(), "X") for m in re.finditer(_X_MARK, b) if m.start() >= start]
322
+ + [(m.start(), "Y") for m in re.finditer(_Y_MARK, b) if m.start() >= start]
323
+ + [(m.start(), "Y-error") for m in re.finditer(_YERR_MARK, b) if m.start() >= start]
324
+ + [(m.start(), "label") for m in re.finditer(_LABEL_MARK, b) if m.start() >= start]
325
+ + [
326
+ (m.start(), _zx_designation(b, m.start()))
327
+ for m in re.finditer(_ZX_MARK, b)
328
+ if m.start() >= start
329
+ ]
330
+ )
331
+ filename_hits = _filename_anchors(b, start)
332
+ books: dict[str, BookMeta] = {}
333
+ # Anchor each book INDEPENDENTLY from the tail start (not a forward-only
334
+ # cursor): a project's book windows are not laid out in decoded-book order
335
+ # (the Hc2 project interleaves them), so a monotonic cursor drops every book
336
+ # whose window section sits before an already-anchored one. Anchors are exact
337
+ # (length-prefixed name / filename basename), so independent search cannot
338
+ # cross-match a different book.
339
+ for book, cols in book_columns.items():
340
+ if not cols:
341
+ continue
342
+ anchored = _book_anchor(b, book, start, filename_hits)
343
+ if anchored is None:
344
+ continue
345
+ anchor, raw_filename = anchored
346
+ candidates = [t for t in markers if t[0] > anchor]
347
+ x_positions = [i for i, t in enumerate(candidates) if t[1] == "X"]
348
+ letter_map: dict[str, tuple[int, str]] | None = None
349
+ run: list[tuple[int, str]] = []
350
+ # The run bound is DERIVED from the column count (the old fixed
351
+ # _MAX_RUN=128 cap silently dropped ALL metadata of a >128-column
352
+ # book — names, units, and the X designation; 2026-07-06 genericity
353
+ # audit). The >_MAX_GAP break remains the real book-boundary signal;
354
+ # len(cols) + slack is only the runaway backstop.
355
+ max_run = len(cols) + 32
356
+ for xi in x_positions:
357
+ attempt = [candidates[xi]]
358
+ j = xi + 1
359
+ while j < len(candidates) and len(attempt) < max_run:
360
+ # The gap to the NEXT column marker is set by how much label
361
+ # content THIS column carries: long_name + unit + comment, and
362
+ # a comment can run to hundreds of chars. So the allowance is
363
+ # _MAX_GAP plus the length of the label record that actually
364
+ # sits in the gap (structural), not a fixed byte cap that a
365
+ # long comment silently overruns -- which used to drop the
366
+ # whole book's metadata (2026-07-06 genericity audit). The
367
+ # real book boundary is a multi-KB window header, far past any
368
+ # single label record, so this never merges two books.
369
+ gap = candidates[j][0] - attempt[-1][0]
370
+ allow = _MAX_GAP + _gap_text_span(b, attempt[-1][0] + 2, candidates[j][0])
371
+ if gap > allow:
372
+ break
373
+ attempt.append(candidates[j])
374
+ j += 1
375
+ if len(attempt) < len(cols):
376
+ continue
377
+ candidate_map = {_excel_col(i): attempt[i] for i in range(len(attempt))}
378
+ if all(c in candidate_map for c in cols):
379
+ letter_map, run = candidate_map, attempt
380
+ break
381
+ if letter_map is None:
382
+ continue
383
+ long_name = raw_filename.decode("latin1", errors="replace") if raw_filename else book
384
+ columns: dict[str, ColumnMeta] = {}
385
+ for col in cols:
386
+ pos, desig = letter_map[col]
387
+ ordinal = run.index(letter_map[col])
388
+ next_pos = run[ordinal + 1][0] if ordinal + 1 < len(run) else pos + 3000
389
+ label = _find_label(b, pos + 2, min(next_pos, pos + 3000))
390
+ rows = (label.split("\r\n") if label else []) + ["", "", ""]
391
+ columns[col] = ColumnMeta(col, _DESIGNATION.get(desig, "Y"), rows[0], rows[1], rows[2])
392
+ books[book] = BookMeta(book, long_name, columns)
393
+ return books