quantized-lab 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quantized/__init__.py +11 -0
- quantized/__main__.py +14 -0
- quantized/api.py +220 -0
- quantized/app.py +206 -0
- quantized/calc/__init__.py +8 -0
- quantized/calc/_clipfit.py +76 -0
- quantized/calc/_natural_neighbor.py +219 -0
- quantized/calc/aggregate.py +159 -0
- quantized/calc/backgrounds.py +353 -0
- quantized/calc/baseline.py +349 -0
- quantized/calc/batch_fit.py +148 -0
- quantized/calc/constants.py +27 -0
- quantized/calc/corrections.py +192 -0
- quantized/calc/crystallography.py +400 -0
- quantized/calc/diffusion.py +120 -0
- quantized/calc/electrical.py +248 -0
- quantized/calc/electrochemistry.py +176 -0
- quantized/calc/element_data.json +1 -0
- quantized/calc/element_data.py +59 -0
- quantized/calc/errors.py +246 -0
- quantized/calc/figure.py +417 -0
- quantized/calc/figure_break.py +152 -0
- quantized/calc/figure_categorical.py +156 -0
- quantized/calc/figure_corner.py +229 -0
- quantized/calc/figure_facets.py +127 -0
- quantized/calc/figure_field.py +137 -0
- quantized/calc/figure_hitmap.py +116 -0
- quantized/calc/figure_labels.py +62 -0
- quantized/calc/figure_map.py +287 -0
- quantized/calc/figure_overrides.py +159 -0
- quantized/calc/figure_page.py +266 -0
- quantized/calc/figure_scale.py +125 -0
- quantized/calc/figure_statplots.py +167 -0
- quantized/calc/figure_styles.py +131 -0
- quantized/calc/figure_ternary.py +239 -0
- quantized/calc/figure_ticks.py +217 -0
- quantized/calc/fit_autoguess.py +156 -0
- quantized/calc/fit_bootstrap.py +163 -0
- quantized/calc/fit_bumps.py +258 -0
- quantized/calc/fit_constraints.py +114 -0
- quantized/calc/fit_equation.py +264 -0
- quantized/calc/fit_findxy.py +80 -0
- quantized/calc/fit_models.py +195 -0
- quantized/calc/fit_models_special.py +189 -0
- quantized/calc/fit_odr.py +99 -0
- quantized/calc/fit_scan.py +243 -0
- quantized/calc/fit_stats.py +215 -0
- quantized/calc/fitting.py +199 -0
- quantized/calc/formula.py +90 -0
- quantized/calc/global_curve_fit.py +305 -0
- quantized/calc/global_fit.py +181 -0
- quantized/calc/interp2d.py +260 -0
- quantized/calc/linecut.py +269 -0
- quantized/calc/magnetic.py +414 -0
- quantized/calc/magnetometry.py +464 -0
- quantized/calc/map.py +228 -0
- quantized/calc/mcmc.py +177 -0
- quantized/calc/optics.py +228 -0
- quantized/calc/pawley.py +251 -0
- quantized/calc/peak_batch.py +128 -0
- quantized/calc/peak_fit.py +259 -0
- quantized/calc/peak_integrate.py +104 -0
- quantized/calc/peak_multifit.py +260 -0
- quantized/calc/peak_track.py +134 -0
- quantized/calc/peaks.py +298 -0
- quantized/calc/peakshapes.py +85 -0
- quantized/calc/plotting.py +147 -0
- quantized/calc/processing.py +232 -0
- quantized/calc/qspace.py +48 -0
- quantized/calc/reductions.py +155 -0
- quantized/calc/reductions_fft.py +383 -0
- quantized/calc/refl_sld_presets.json +1 -0
- quantized/calc/reflectivity.py +80 -0
- quantized/calc/registry.py +303 -0
- quantized/calc/relaxation.py +119 -0
- quantized/calc/report.py +253 -0
- quantized/calc/report_emit.py +227 -0
- quantized/calc/resample.py +142 -0
- quantized/calc/rsm.py +91 -0
- quantized/calc/rsm_analyze.py +245 -0
- quantized/calc/semiconductor.py +488 -0
- quantized/calc/sld.py +131 -0
- quantized/calc/sld_formula.py +138 -0
- quantized/calc/spectral.py +357 -0
- quantized/calc/statplots.py +214 -0
- quantized/calc/stats.py +399 -0
- quantized/calc/stats_anova2.py +202 -0
- quantized/calc/stats_anova_ext.py +338 -0
- quantized/calc/stats_dist.py +196 -0
- quantized/calc/stats_glm.py +245 -0
- quantized/calc/stats_multivar.py +289 -0
- quantized/calc/stats_roc.py +157 -0
- quantized/calc/stats_survival.py +261 -0
- quantized/calc/stats_tests.py +380 -0
- quantized/calc/substrates.py +181 -0
- quantized/calc/superconductor.py +359 -0
- quantized/calc/surface_fit.py +290 -0
- quantized/calc/surface_models.py +156 -0
- quantized/calc/thermal.py +119 -0
- quantized/calc/thin_film.py +425 -0
- quantized/calc/unit_convert.py +259 -0
- quantized/calc/units.py +80 -0
- quantized/calc/vacuum.py +290 -0
- quantized/calc/xray.py +169 -0
- quantized/cli.py +214 -0
- quantized/datastruct.py +153 -0
- quantized/io/__init__.py +11 -0
- quantized/io/_hdf5_layout.py +308 -0
- quantized/io/_jcamp_asdf.py +135 -0
- quantized/io/_xrdml_scan.py +291 -0
- quantized/io/base.py +82 -0
- quantized/io/bruker_brml.py +177 -0
- quantized/io/bruker_raw.py +158 -0
- quantized/io/cif.py +266 -0
- quantized/io/consolidated.py +122 -0
- quantized/io/delimited.py +222 -0
- quantized/io/excel.py +135 -0
- quantized/io/hdf5.py +192 -0
- quantized/io/import_filters.py +178 -0
- quantized/io/import_preview.py +262 -0
- quantized/io/jcamp.py +179 -0
- quantized/io/lakeshore.py +163 -0
- quantized/io/ncnr.py +278 -0
- quantized/io/netcdf.py +195 -0
- quantized/io/opus.py +231 -0
- quantized/io/origin.py +346 -0
- quantized/io/origin_com.py +194 -0
- quantized/io/origin_project/__init__.py +221 -0
- quantized/io/origin_project/annotation_marks.py +288 -0
- quantized/io/origin_project/container.py +262 -0
- quantized/io/origin_project/curve_style_color.py +359 -0
- quantized/io/origin_project/figure_geometry.py +108 -0
- quantized/io/origin_project/figure_layers.py +333 -0
- quantized/io/origin_project/figure_text.py +258 -0
- quantized/io/origin_project/figures.py +210 -0
- quantized/io/origin_project/figures_opju.py +440 -0
- quantized/io/origin_project/notes.py +302 -0
- quantized/io/origin_project/opj.py +459 -0
- quantized/io/origin_project/opj_curves.py +297 -0
- quantized/io/origin_project/opj_shapes.py +148 -0
- quantized/io/origin_project/opju.py +146 -0
- quantized/io/origin_project/opju_axis_real_form.py +418 -0
- quantized/io/origin_project/opju_axis_specimen_form.py +167 -0
- quantized/io/origin_project/opju_codec.py +370 -0
- quantized/io/origin_project/opju_curves.py +497 -0
- quantized/io/origin_project/opju_curves_allcols.py +258 -0
- quantized/io/origin_project/opju_figure_curves.py +302 -0
- quantized/io/origin_project/opju_figure_text.py +245 -0
- quantized/io/origin_project/opju_reports.py +129 -0
- quantized/io/origin_project/origin_richtext.py +145 -0
- quantized/io/origin_project/preview.py +132 -0
- quantized/io/origin_project/templates.py +314 -0
- quantized/io/origin_project/tree.py +379 -0
- quantized/io/origin_project/tree_opju.py +228 -0
- quantized/io/origin_project/windows.py +238 -0
- quantized/io/origin_project/windows_opju.py +393 -0
- quantized/io/origin_project/writer.py +156 -0
- quantized/io/origin_project/writer_blocks.py +282 -0
- quantized/io/qd.py +380 -0
- quantized/io/refl1d.py +132 -0
- quantized/io/registry.py +210 -0
- quantized/io/report_export.py +347 -0
- quantized/io/rigaku.py +100 -0
- quantized/io/sims.py +398 -0
- quantized/io/spc.py +311 -0
- quantized/io/xrd_csv.py +308 -0
- quantized/io/xrdml.py +394 -0
- quantized/jobs.py +173 -0
- quantized/plugins/__init__.py +50 -0
- quantized/plugins/contract.py +111 -0
- quantized/plugins/loader.py +394 -0
- quantized/plugins/steps.py +90 -0
- quantized/routes/__init__.py +7 -0
- quantized/routes/_bookcache.py +62 -0
- quantized/routes/_export_common.py +27 -0
- quantized/routes/_payload.py +58 -0
- quantized/routes/_uploadcache.py +59 -0
- quantized/routes/aggregate.py +46 -0
- quantized/routes/baseline.py +210 -0
- quantized/routes/books.py +117 -0
- quantized/routes/calc.py +56 -0
- quantized/routes/corrections.py +78 -0
- quantized/routes/crystallography.py +80 -0
- quantized/routes/diffusion.py +58 -0
- quantized/routes/electrical.py +101 -0
- quantized/routes/electrochemistry.py +83 -0
- quantized/routes/export.py +280 -0
- quantized/routes/export_facets.py +83 -0
- quantized/routes/export_figures.py +471 -0
- quantized/routes/export_page.py +125 -0
- quantized/routes/fitting.py +379 -0
- quantized/routes/fitting_bumps.py +97 -0
- quantized/routes/import_template.py +97 -0
- quantized/routes/import_wizard.py +150 -0
- quantized/routes/jobs_api.py +59 -0
- quantized/routes/magnetic.py +135 -0
- quantized/routes/magnetometry.py +133 -0
- quantized/routes/optics.py +98 -0
- quantized/routes/parsers.py +281 -0
- quantized/routes/peaks.py +184 -0
- quantized/routes/plot.py +103 -0
- quantized/routes/reductions.py +121 -0
- quantized/routes/reference.py +58 -0
- quantized/routes/reflectivity.py +91 -0
- quantized/routes/report_export.py +119 -0
- quantized/routes/rsm.py +136 -0
- quantized/routes/samples.py +32 -0
- quantized/routes/semiconductor.py +207 -0
- quantized/routes/sld.py +42 -0
- quantized/routes/spectral.py +54 -0
- quantized/routes/statplots.py +99 -0
- quantized/routes/stats.py +418 -0
- quantized/routes/stats_design.py +321 -0
- quantized/routes/substrates.py +46 -0
- quantized/routes/superconductor.py +139 -0
- quantized/routes/thermal.py +57 -0
- quantized/routes/thin_film.py +153 -0
- quantized/routes/vacuum.py +113 -0
- quantized/routes/xray.py +32 -0
- quantized/samples/demo_vsm.csv +42 -0
- quantized/server_launch.py +251 -0
- quantized/web/assets/JetBrainsMono-Bold-CUogYd9I.woff2 +0 -0
- quantized/web/assets/JetBrainsMono-Regular-CA-Os4ii.woff2 +0 -0
- quantized/web/assets/index-BHmmCL-x.js +27 -0
- quantized/web/assets/index-BiZzN7J6.css +1 -0
- quantized/web/index.html +13 -0
- quantized/web/loading.html +69 -0
- quantized_lab-0.8.0.dist-info/METADATA +122 -0
- quantized_lab-0.8.0.dist-info/RECORD +233 -0
- quantized_lab-0.8.0.dist-info/WHEEL +4 -0
- quantized_lab-0.8.0.dist-info/entry_points.txt +4 -0
- quantized_lab-0.8.0.dist-info/licenses/LICENSE +201 -0
- quantized_lab-0.8.0.dist-info/licenses/NOTICE +11 -0
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""Extract per-column metadata from the ``.opj`` windows section.
|
|
2
|
+
|
|
3
|
+
Worksheet *window definitions* live in the same block stream as the column
|
|
4
|
+
data, after the datasets section. Each worksheet window opens with a header
|
|
5
|
+
block (``00 00 <BookShort> 00 …``, long name ending at the ``@${`` storage
|
|
6
|
+
marker) and contains, per column, a **property block** (≥500 B; designation at
|
|
7
|
+
``0x11``, NUL-terminated short name at ``0x12``) immediately followed by a
|
|
8
|
+
**label-text block** (``LongName\\r\\nUnit\\r\\nComment``, cut at ``@${``).
|
|
9
|
+
Column ``S`` of book ``B`` maps to the dataset named ``"B_S"``.
|
|
10
|
+
|
|
11
|
+
A property block's byte 0x06 varies with the column's storage flavour --
|
|
12
|
+
``0x09`` for a plain sheet-1 column, ``0x0B`` for a formula/derived column
|
|
13
|
+
*and* for every column of an auto-generated report sheet (FitLinear/FitNL's
|
|
14
|
+
"Parameters"/"Notes"/etc. sheets) -- so it does not distinguish sheet
|
|
15
|
+
identity; both values are accepted (plan item: report-sheet leak fix).
|
|
16
|
+
|
|
17
|
+
**Multi-sheet books** (a workbook with a report/curve sheet appended after
|
|
18
|
+
the real data, e.g. Origin's FitLinear auto-adds "FitLinear1"/
|
|
19
|
+
"FitLinearCurve1" siblings to the sheet it fits) restart column lettering at
|
|
20
|
+
"A" for each extra sheet, and *also* reuse the very same 0x0B property-block
|
|
21
|
+
shape as a real formula column -- so neither the storage-flavour byte nor a
|
|
22
|
+
short-name repeat is a reliable, early sheet-boundary signal on its own (a
|
|
23
|
+
report sheet's own property/label blocks can front-run a repeat detection
|
|
24
|
+
by many columns). The real signal is a fixed 365-byte **sheet/layer
|
|
25
|
+
sub-header** block carrying ``Pd<Name>\\0`` at a constant offset (0xD0) --
|
|
26
|
+
one appears at the very start of every worksheet sheet (and every graph
|
|
27
|
+
layer, tagged ``Pd1``/``Pd2``/…) inside a window's block span. The *second*
|
|
28
|
+
one seen since the enclosing window header is the true start of sheet 2+;
|
|
29
|
+
everything from there on (property blocks, labels, formulas) is excluded.
|
|
30
|
+
Validated against Moke.opj's ``Book4`` (Sheet1 / FitLinear1 /
|
|
31
|
+
FitLinearCurve1 — three ``Pd...`` markers, one per sheet, exactly bracketing
|
|
32
|
+
each sheet's real column-property-block run) and every single-sheet corpus
|
|
33
|
+
book (exactly one marker, at the window's very start). Older-format (CPYA
|
|
34
|
+
4.3227) files carry no such marker at all; the short-name-repeat guard is
|
|
35
|
+
kept as a fallback for that case (unverified whether any 4.3227 file in the
|
|
36
|
+
corpus has a real multi-sheet book -- none observed to date).
|
|
37
|
+
|
|
38
|
+
Byte layout validated against the local corpus — see
|
|
39
|
+
``docs/origin_re/opj_windows_section.md`` (plan item 1).
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import re
|
|
45
|
+
from dataclasses import dataclass, field
|
|
46
|
+
|
|
47
|
+
from quantized.io.origin_project.container import walk_blocks
|
|
48
|
+
|
|
49
|
+
__all__ = ["BookMeta", "ColumnMeta", "window_metadata"]
|
|
50
|
+
|
|
51
|
+
# Origin's published plot-designation enum (format fact).
|
|
52
|
+
_DESIGNATION = {
|
|
53
|
+
0: "Y",
|
|
54
|
+
1: "disregard",
|
|
55
|
+
2: "Y-error",
|
|
56
|
+
3: "X",
|
|
57
|
+
4: "label",
|
|
58
|
+
5: "Z",
|
|
59
|
+
6: "X-error",
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class ColumnMeta:
|
|
65
|
+
short: str
|
|
66
|
+
designation: str
|
|
67
|
+
long_name: str = ""
|
|
68
|
+
unit: str = ""
|
|
69
|
+
comment: str = ""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class BookMeta:
|
|
74
|
+
short: str
|
|
75
|
+
long_name: str
|
|
76
|
+
columns: dict[str, ColumnMeta] = field(default_factory=dict)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _cstring(payload: bytes, start: int, limit: int = 48) -> str | None:
|
|
80
|
+
"""Printable NUL-terminated ASCII at ``start``, or None."""
|
|
81
|
+
end = payload.find(b"\x00", start, start + limit)
|
|
82
|
+
if end <= start:
|
|
83
|
+
return None
|
|
84
|
+
raw = payload[start:end]
|
|
85
|
+
if not all(0x20 <= c < 0x7F for c in raw):
|
|
86
|
+
return None
|
|
87
|
+
return raw.decode("latin1")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _is_window_header(payload: bytes) -> str | None:
|
|
91
|
+
"""A window-header block starts ``00 00 <Name> 00``; return the name.
|
|
92
|
+
|
|
93
|
+
Digit-led names are real (``tree.py``'s module docstring documents
|
|
94
|
+
``30nmADPNR`` and six digit-led graph names in the corpus, COM-verified),
|
|
95
|
+
so the first-char gate is ``isalnum``, not ``isalpha`` — the old
|
|
96
|
+
alpha-first check silently hid digit-led graph windows from
|
|
97
|
+
``figures.py`` AND let their internal blocks poison
|
|
98
|
+
``opj_curves.column_id_map`` (a digit-led graph was not recognized as a
|
|
99
|
+
graph, so its 519-byte curve records were scanned as column storage)."""
|
|
100
|
+
if len(payload) < 150 or payload[0] or payload[1]:
|
|
101
|
+
return None
|
|
102
|
+
name = _cstring(payload, 2, 64)
|
|
103
|
+
return name if name and name[0].isalnum() else None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _book_long_name(payload: bytes, short: str) -> str:
|
|
107
|
+
"""The display title, stored in the header tail.
|
|
108
|
+
|
|
109
|
+
v4.3380 headers end the title at an ``@${…}<OriginStorage>`` marker;
|
|
110
|
+
v4.3227 headers have no storage blob — the title is simply the last
|
|
111
|
+
printable run after the fixed-format region (offset > 0x60).
|
|
112
|
+
"""
|
|
113
|
+
anchor = payload.find(b"@${")
|
|
114
|
+
if anchor >= 0:
|
|
115
|
+
start = payload.rfind(b"\x00", 0, anchor) + 1
|
|
116
|
+
raw = payload[start:anchor]
|
|
117
|
+
if raw and all(0x20 <= c < 0x7F for c in raw):
|
|
118
|
+
return raw.decode("latin1")
|
|
119
|
+
return short
|
|
120
|
+
runs = [m for m in re.finditer(rb"[\x20-\x7e]{2,}", payload) if m.start() > 0x60]
|
|
121
|
+
if runs:
|
|
122
|
+
cand = runs[-1].group().decode("latin1")
|
|
123
|
+
if cand != short:
|
|
124
|
+
return cand
|
|
125
|
+
return short
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _is_column_block(payload: bytes) -> bool:
|
|
129
|
+
"""Robust, version-independent column-property block detector.
|
|
130
|
+
|
|
131
|
+
Byte 0x06 is ``0x09`` for a plain (never-recalculated) sheet-1 column and
|
|
132
|
+
``0x0B`` for a formula/derived column or any report-sheet column -- both
|
|
133
|
+
are real column-property blocks, so both are accepted (a Moke.opj Book4
|
|
134
|
+
Sheet1 measurement had 14/15 columns at ``0x09``; only its one formula
|
|
135
|
+
column, "Normalized1", was ``0x0B`` -- the old ``==0x0B``-only check
|
|
136
|
+
dropped those 14 silently instead of mapping them).
|
|
137
|
+
"""
|
|
138
|
+
return (
|
|
139
|
+
len(payload) >= 500
|
|
140
|
+
and payload[0x06] in (0x09, 0x0B)
|
|
141
|
+
and payload[0x25] in (0x21, 0x30) # 0x30 observed on Y-error columns
|
|
142
|
+
and _cstring(payload, 0x12, 8) is not None
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
_SHEET_HEADER_SIZE = 365
|
|
147
|
+
_SHEET_NAME_OFFSET = 0xD0 # 208: fixed offset of the "Pd<Name>" marker
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _is_sheet_header(payload: bytes) -> str | None:
|
|
151
|
+
"""A per-sheet (or per-graph-layer) sub-header: the real sheet-boundary
|
|
152
|
+
signal (see module docstring). Always exactly 365 B, carrying
|
|
153
|
+
NUL-terminated ``Pd<Name>`` at a fixed offset -- never observed to
|
|
154
|
+
collide with a window-header or column-property block's size/shape
|
|
155
|
+
across the local corpus (window headers range 195-359 B in every
|
|
156
|
+
corpus file; a column-property block is always >=500 B)."""
|
|
157
|
+
if len(payload) != _SHEET_HEADER_SIZE:
|
|
158
|
+
return None
|
|
159
|
+
if payload[_SHEET_NAME_OFFSET : _SHEET_NAME_OFFSET + 2] != b"Pd":
|
|
160
|
+
return None
|
|
161
|
+
return _cstring(payload, _SHEET_NAME_OFFSET + 2, 40)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _label_rows(payload: bytes) -> tuple[str, str, str]:
|
|
165
|
+
"""Parse a label-text block: LongName / Unit / Comment (missing → "")."""
|
|
166
|
+
text = payload.split(b"\x00", 1)[0]
|
|
167
|
+
cut = text.find(b"@${")
|
|
168
|
+
if cut >= 0:
|
|
169
|
+
text = text[:cut]
|
|
170
|
+
rows = text.decode("latin1", errors="replace").split("\r\n")
|
|
171
|
+
rows += ["", "", ""]
|
|
172
|
+
return rows[0], rows[1], rows[2]
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def window_metadata(b: bytes) -> dict[str, BookMeta]:
|
|
176
|
+
"""Map book short name → :class:`BookMeta` with per-column names/units.
|
|
177
|
+
|
|
178
|
+
Only primary-sheet columns are mapped for now (multi-sheet ``@N`` datasets
|
|
179
|
+
are plan item 5). Graph windows contain no column blocks and drop out
|
|
180
|
+
naturally.
|
|
181
|
+
|
|
182
|
+
A report/curve sheet appended after the primary sheet (Origin's
|
|
183
|
+
FitLinear/FitNL auto-add, e.g. "FitLinear1"/"FitLinearCurve1") must never
|
|
184
|
+
reach ``current.columns`` — that dict is the *primary-sheet* map. The
|
|
185
|
+
``Pd<Name>`` sheet sub-header (`_is_sheet_header`) is the precise
|
|
186
|
+
boundary: the second one seen since the enclosing window header closes
|
|
187
|
+
collection immediately, before that sheet's first property block. The
|
|
188
|
+
short-name-repeat check stays as a fallback for the older container
|
|
189
|
+
version that carries no such marker (plan item 5's original guard).
|
|
190
|
+
"""
|
|
191
|
+
books: dict[str, BookMeta] = {}
|
|
192
|
+
current: BookMeta | None = None
|
|
193
|
+
pending: tuple[str, str] | None = None # (short, designation) awaiting label
|
|
194
|
+
closed = False # set once the current window moves past its primary sheet
|
|
195
|
+
sheets_seen = 0 # count of Pd<Name> markers since the current window header
|
|
196
|
+
|
|
197
|
+
def commit(long_name: str = "", unit: str = "", comment: str = "") -> None:
|
|
198
|
+
nonlocal pending, closed
|
|
199
|
+
if pending is not None and current is not None and not closed:
|
|
200
|
+
short, desig = pending
|
|
201
|
+
if short in current.columns:
|
|
202
|
+
# A repeated short means sheet 2+ started (every sheet restarts
|
|
203
|
+
# at column A); only the primary sheet maps to plain
|
|
204
|
+
# "<Book>_<Col>" datasets, so stop collecting (plan item 5;
|
|
205
|
+
# fallback for containers with no Pd<Name> marker).
|
|
206
|
+
closed = True
|
|
207
|
+
else:
|
|
208
|
+
current.columns[short] = ColumnMeta(short, desig, long_name, unit, comment)
|
|
209
|
+
pending = None
|
|
210
|
+
|
|
211
|
+
for size, payload in walk_blocks(b):
|
|
212
|
+
if size == 0:
|
|
213
|
+
continue
|
|
214
|
+
header_name = _is_window_header(payload)
|
|
215
|
+
sheet_name = _is_sheet_header(payload) if header_name is None else None
|
|
216
|
+
is_col = (
|
|
217
|
+
_is_column_block(payload) if header_name is None and sheet_name is None else False
|
|
218
|
+
)
|
|
219
|
+
if pending is not None:
|
|
220
|
+
if header_name is None and sheet_name is None and not is_col and size < 500:
|
|
221
|
+
commit(*_label_rows(payload)) # the label block for the pending column
|
|
222
|
+
continue
|
|
223
|
+
commit() # structural block follows — column had no label text
|
|
224
|
+
if header_name is not None:
|
|
225
|
+
current = books.setdefault(
|
|
226
|
+
header_name, BookMeta(header_name, _book_long_name(payload, header_name))
|
|
227
|
+
)
|
|
228
|
+
closed = False
|
|
229
|
+
sheets_seen = 0
|
|
230
|
+
elif sheet_name is not None:
|
|
231
|
+
sheets_seen += 1
|
|
232
|
+
if sheets_seen > 1: # the 2nd+ sheet/layer marker: stop mapping columns
|
|
233
|
+
closed = True
|
|
234
|
+
elif current is not None and is_col and not closed:
|
|
235
|
+
short = _cstring(payload, 0x12, 8) or "?"
|
|
236
|
+
pending = (short, _DESIGNATION.get(payload[0x11], "Y"))
|
|
237
|
+
commit()
|
|
238
|
+
return books
|
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""Extract per-column long-names/units from the ``.opju`` (CPYUA) windows section.
|
|
2
|
+
|
|
3
|
+
The CPYUA container's worksheet windows section is not `.opj`'s CPY block
|
|
4
|
+
stream (see ``docs/origin_re/opj_windows_section.md``) — it uses its own
|
|
5
|
+
tag/length framing that this module does not fully parse. What IS pinned,
|
|
6
|
+
validated against Origin's own exported ground truth across five real
|
|
7
|
+
corpus files (XAS, RockingCurve, UnpolPlots, "Fixed Lambdas SI", plus the
|
|
8
|
+
``rosetta_*`` specimens — 145/145 names, units, and comments matched):
|
|
9
|
+
|
|
10
|
+
* Every worksheet column carries a 2-byte **plot-designation marker**:
|
|
11
|
+
``21 51`` for X, ``21 61`` for Y, ``30 61`` for Y-error — the same
|
|
12
|
+
marker-byte + display-code convention `.opj` uses at property-block
|
|
13
|
+
offsets 0x25/0x26 (see ``opj_windows_section.md`` sec 4.1), reused inside
|
|
14
|
+
CPYUA's own framing.
|
|
15
|
+
* A fixed-shape run (default column-format doubles) follows every marker,
|
|
16
|
+
then an OPTIONAL length-prefixed embedded blob (``<len><tag=0x01><bytes>
|
|
17
|
+
<NUL>``) carrying that column's ``ColumnInfo``/``ImportFile`` storage
|
|
18
|
+
(only present for imported-file columns — a long file path, possibly
|
|
19
|
+
using Origin's internal string back-reference shorthand which this
|
|
20
|
+
module never tries to decode), then the REAL label record in the SAME
|
|
21
|
+
``<len:u8><tag:u8><text><NUL>`` shape (``len`` counts tag+text+NUL).
|
|
22
|
+
``text`` splits on ``\\r\\n`` into long_name/unit/comment (0-3 rows); a
|
|
23
|
+
zero-length text (tag ``0x01``, no ``\\r\\n``) means "no label" (e.g. an
|
|
24
|
+
unlabeled Y column) and some columns carry only a bare long name with no
|
|
25
|
+
``\\r\\n`` at all (single-row form).
|
|
26
|
+
* Every column emits its own marker (+ optional label) record, in true
|
|
27
|
+
sheet column order (A, B, C, ...) — INCLUDING columns that never decode
|
|
28
|
+
as data (e.g. a blank/text column between two decoded numeric ones).
|
|
29
|
+
That is why association is by ORDINAL POSITION within one book's
|
|
30
|
+
contiguous marker run, mapped through Origin's column lettering
|
|
31
|
+
(A, B, ... Z, AA, AB, ... — wide measurement sheets run well past Z),
|
|
32
|
+
rather than by parsing an internal short-name field — no such field was
|
|
33
|
+
pinned for CPYUA (unlike `.opj`'s offset-0x12 short name). Each book is
|
|
34
|
+
anchored INDEPENDENTLY (not via a forward-only cursor): a project's book
|
|
35
|
+
windows are not in decoded-book order, so a monotonic cursor drops every
|
|
36
|
+
book whose window sits before an already-anchored one (the Hc2 project
|
|
37
|
+
interleaves 30+ of them). Anchors are exact, so independent search is safe.
|
|
38
|
+
* Each book's own marker run is anchored via ONE OF: (a) the embedded
|
|
39
|
+
``ColumnInfo``/``ImportFile`` path's filename, alnum-stripped and matched
|
|
40
|
+
against the book's known short name (handles Origin's habit of dropping
|
|
41
|
+
underscores when deriving a book short name from an imported filename,
|
|
42
|
+
e.g. ``bl11_YIGPy_032.dat`` -> book ``bl11YIGPy032``); or (b) a
|
|
43
|
+
``<len=namelen+2> 00 00 <name>`` window/book-header reference that
|
|
44
|
+
appears even for books never imported from a file (manually-typed
|
|
45
|
+
sheets, e.g. the ``rosetta_*`` specimens).
|
|
46
|
+
|
|
47
|
+
Positional guessing is NOT used to *detect* a label: every accepted record
|
|
48
|
+
matches the exact ``<len><tag><text><NUL>`` byte count PLUS a character-
|
|
49
|
+
class + known-internal-token filter (rejects embedded blob fragments like
|
|
50
|
+
a truncated ``ResultsLog``/``OriginStorage`` token, which the length-prefix
|
|
51
|
+
match alone can accidentally land inside). Association across a book's
|
|
52
|
+
columns IS positional, but only after that book's boundary is independently
|
|
53
|
+
confirmed by anchor (a) or (b) above — never by scanning the whole file for
|
|
54
|
+
ASCII runs. When no anchor is found, or the marker run doesn't cover every
|
|
55
|
+
column ``opju_codec.scan_columns`` actually decoded for that book, the book
|
|
56
|
+
is left out of the result entirely (A/B/C fallback stays in force) rather
|
|
57
|
+
than guessed at.
|
|
58
|
+
|
|
59
|
+
See ``docs/origin_re/opju_container.md`` for the full byte-level trail.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
from __future__ import annotations
|
|
63
|
+
|
|
64
|
+
import re
|
|
65
|
+
from collections.abc import Mapping, Sequence
|
|
66
|
+
|
|
67
|
+
from quantized.io.origin_project.opju_codec import tail_start
|
|
68
|
+
from quantized.io.origin_project.windows import BookMeta, ColumnMeta
|
|
69
|
+
|
|
70
|
+
__all__ = ["opju_window_metadata"]
|
|
71
|
+
|
|
72
|
+
_X_MARK = b"\x21\x51"
|
|
73
|
+
_Y_MARK = b"\x21\x61"
|
|
74
|
+
_YERR_MARK = b"\x30\x61"
|
|
75
|
+
# Label / Z / X-error markers, pinned 2026-07-06 by the designations.opju
|
|
76
|
+
# by-construction specimen (one column per designation): Label = `21 41`;
|
|
77
|
+
# Z and X-error SHARE `20 61` and are told apart by the record's own
|
|
78
|
+
# designation code byte (`82 02 <code>`: 1=label 2=Y-error 3=X 4=Z
|
|
79
|
+
# 5=X-error; a plain Y stores no code field at all). Before these, a book
|
|
80
|
+
# containing ANY such column failed the cover-check and silently lost its
|
|
81
|
+
# ENTIRE metadata run (names, units, and the X designation) — the audit's
|
|
82
|
+
# whole-book blast radius (#10).
|
|
83
|
+
_LABEL_MARK = b"\x21\x41"
|
|
84
|
+
_ZX_MARK = b"\x20\x61" # Z or X-error; resolved via the code byte
|
|
85
|
+
_DESIGNATION = {
|
|
86
|
+
"X": "X",
|
|
87
|
+
"Y": "Y",
|
|
88
|
+
"Y-error": "Y-error",
|
|
89
|
+
"label": "label",
|
|
90
|
+
"Z": "Z",
|
|
91
|
+
"X-error": "X-error",
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
# A backslash-delimited filename token, as embedded in a column's ImportFile path.
|
|
95
|
+
_FILENAME_RE = re.compile(rb"\\([\w \-]{1,60}?\.[A-Za-z0-9]{2,5})(?=[^\w.]|$)")
|
|
96
|
+
# A plausible bare long-name (no unit/comment row): starts with a letter, then
|
|
97
|
+
# the punctuation real scientific column names use -- parens/brackets/colons/
|
|
98
|
+
# underscores (e.g. "Nb Hc2 (T)", "NiBi_3::R", "temperature: T1",
|
|
99
|
+
# "multi[0]:iterator"). Still excludes path/markup chars (backslash, <, >), which
|
|
100
|
+
# the _JUNK_MARKERS filter rejects separately, so a length-prefix coincidence
|
|
101
|
+
# landing inside a storage blob is still caught. Checked on the DECODED string
|
|
102
|
+
# (see _is_single_row_label) so non-ASCII scientific glyphs -- degree, Angstrom,
|
|
103
|
+
# micro, ohm, Greek, sub/superscripts -- are accepted; the .opju container is
|
|
104
|
+
# UTF-8, so "tilt 45<deg>" arrives as bytes c2 b0 and a byte-only ASCII regex
|
|
105
|
+
# both rejected it and (via latin-1) mojibaked it (2026-07-06 genericity audit).
|
|
106
|
+
_LABEL_PUNCT = frozenset(" _:()[]./+%#*,-")
|
|
107
|
+
# Text containing any of these is inside an embedded storage blob, not a real label.
|
|
108
|
+
_JUNK_MARKERS = (b"\\", b"OriginStorage", b"ColumnInfo", b"ImportFile", b"<", b">")
|
|
109
|
+
# Single-row candidates that are themselves a fragment of one of these internal
|
|
110
|
+
# tokens (e.g. a length-prefix match landing mid-string, "esultsLog") are noise.
|
|
111
|
+
_KNOWN_TOKENS = (
|
|
112
|
+
b"ResultsLog",
|
|
113
|
+
b"OriginStorage",
|
|
114
|
+
b"ColumnInfo",
|
|
115
|
+
b"ImportFile",
|
|
116
|
+
b"TREE",
|
|
117
|
+
b"Organizer",
|
|
118
|
+
b"IMGEXP",
|
|
119
|
+
b"AXISTYPE",
|
|
120
|
+
b"Script",
|
|
121
|
+
b"History",
|
|
122
|
+
b"PConst",
|
|
123
|
+
)
|
|
124
|
+
_MAX_GAP = 600 # max byte spacing between one column-property record and the next
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _zx_designation(b: bytes, mark_pos: int) -> str:
|
|
128
|
+
"""Resolve the shared ``20 61`` marker to Z or X-error via the record's
|
|
129
|
+
own designation code (``82 02 <code>`` a few bytes earlier: 4=Z,
|
|
130
|
+
5=X-error). An unresolvable record reads as "Z" (either way the column
|
|
131
|
+
is a non-plottable auxiliary; the marker still COUNTS the column, which
|
|
132
|
+
is what protects the book's metadata run)."""
|
|
133
|
+
j = b.rfind(b"\x82\x02", max(0, mark_pos - 16), mark_pos)
|
|
134
|
+
if j >= 0 and j + 2 < len(b) and b[j + 2] == 5:
|
|
135
|
+
return "X-error"
|
|
136
|
+
return "Z"
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _decode_cell_text(raw: bytes) -> str | None:
|
|
140
|
+
"""Decode a column-label cell, or ``None`` if it isn't text.
|
|
141
|
+
|
|
142
|
+
The ``.opju`` container stores text as UTF-8 (degree = ``c2 b0``, micro =
|
|
143
|
+
``c2 b5``, Greek mu = ``ce bc``); fall back to latin-1 for the rare cell
|
|
144
|
+
that is valid latin-1 but not UTF-8. Reject any cell holding a C0 control
|
|
145
|
+
other than TAB/CR/LF — that is a binary/format block, never a label."""
|
|
146
|
+
for enc in ("utf-8", "latin1"):
|
|
147
|
+
try:
|
|
148
|
+
s = raw.decode(enc)
|
|
149
|
+
except UnicodeDecodeError:
|
|
150
|
+
continue
|
|
151
|
+
if any(ord(c) < 0x20 and c not in "\t\r\n" for c in s):
|
|
152
|
+
return None
|
|
153
|
+
return s
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _is_single_row_label(s: str) -> bool:
|
|
158
|
+
"""Whether ``s`` is a plausible bare long-name (see ``_LABEL_PUNCT``).
|
|
159
|
+
|
|
160
|
+
Unicode-aware: the first char is any letter (incl. Greek/Cyrillic) and the
|
|
161
|
+
rest are alphanumerics, the scientific punctuation set, or ANY non-ASCII
|
|
162
|
+
printable (``ord >= 0x80``) — the glyphs real column names use. Markup/path
|
|
163
|
+
chars (``\\``, ``<``, ``>``) are ASCII and not in the set, so they still
|
|
164
|
+
fail here on top of the ``_JUNK_MARKERS`` pre-filter."""
|
|
165
|
+
if not s or not s[0].isalpha() or len(s) > 60:
|
|
166
|
+
return False
|
|
167
|
+
return all(c.isalnum() or c in _LABEL_PUNCT or ord(c) >= 0x80 for c in s)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _strip_alnum(s: bytes) -> bytes:
|
|
171
|
+
return re.sub(rb"[^A-Za-z0-9]", b"", s)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _excel_col(i: int) -> str:
|
|
175
|
+
"""0-based column index -> Origin/Excel column letter (0->A, 25->Z, 26->AA...)."""
|
|
176
|
+
letters = ""
|
|
177
|
+
i += 1
|
|
178
|
+
while i:
|
|
179
|
+
i, rem = divmod(i - 1, 26)
|
|
180
|
+
letters = chr(ord("A") + rem) + letters
|
|
181
|
+
return letters
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _filename_anchors(b: bytes, start: int) -> list[tuple[int, bytes, bytes]]:
|
|
185
|
+
"""``(position, alnum-stripped-basename, raw-filename)`` per backslash token."""
|
|
186
|
+
out: list[tuple[int, bytes, bytes]] = []
|
|
187
|
+
for m in _FILENAME_RE.finditer(b):
|
|
188
|
+
if m.start() < start:
|
|
189
|
+
continue
|
|
190
|
+
raw = m.group(1)
|
|
191
|
+
base = raw.rsplit(b".", 1)[0]
|
|
192
|
+
out.append((m.start(), _strip_alnum(base), raw))
|
|
193
|
+
return out
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _book_anchor(
|
|
197
|
+
b: bytes, short: str, search_from: int, filename_hits: list[tuple[int, bytes, bytes]]
|
|
198
|
+
) -> tuple[int, bytes | None] | None:
|
|
199
|
+
"""First ``(position, raw_filename_or_None)`` identifying this book's window section."""
|
|
200
|
+
short_b = short.encode("latin1")
|
|
201
|
+
for pos, stripped, raw in filename_hits:
|
|
202
|
+
if pos >= search_from and stripped == short_b:
|
|
203
|
+
return pos, raw
|
|
204
|
+
header_len = len(short_b) + 2
|
|
205
|
+
if header_len < 256:
|
|
206
|
+
idx = b.find(bytes([header_len]) + b"\x00\x00" + short_b, search_from)
|
|
207
|
+
if idx >= 0:
|
|
208
|
+
return idx, None
|
|
209
|
+
if len(short) >= 4: # storage-tail reference; too ambiguous for short names alone
|
|
210
|
+
storage_len = len(short_b) + 1
|
|
211
|
+
if storage_len < 256:
|
|
212
|
+
idx = b.find(bytes([storage_len]) + short_b + b"\x00", search_from)
|
|
213
|
+
if idx >= 0:
|
|
214
|
+
return idx, None
|
|
215
|
+
return None
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _gap_text_span(b: bytes, start: int, end: int) -> int:
|
|
219
|
+
"""Longest run of consecutive text bytes in ``[start, end)``.
|
|
220
|
+
|
|
221
|
+
A text byte is printable ASCII / TAB / CR / LF, or a UTF-8 high byte
|
|
222
|
+
(>= 0x80 — a label glyph). Used only to WIDEN the inter-marker gap
|
|
223
|
+
allowance by the label content actually present, so a long column comment
|
|
224
|
+
(a long contiguous printable run) can't split a book's metadata run. A
|
|
225
|
+
real book boundary is a multi-KB BINARY window header — no comparable
|
|
226
|
+
printable run — so this never merges two books. Framing-agnostic on
|
|
227
|
+
purpose: the comment's length prefix is a multi-byte varint we don't parse
|
|
228
|
+
here, and we don't need to."""
|
|
229
|
+
|
|
230
|
+
def _is_text(c: int) -> bool:
|
|
231
|
+
return c in (0x09, 0x0A, 0x0D) or 0x20 <= c <= 0x7E or c >= 0x80
|
|
232
|
+
|
|
233
|
+
best = run = 0
|
|
234
|
+
for c in b[start : min(end, len(b))]:
|
|
235
|
+
run = run + 1 if _is_text(c) else 0
|
|
236
|
+
if run > best:
|
|
237
|
+
best = run
|
|
238
|
+
return best
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _parse_label_record(b: bytes, p: int) -> bytes | None:
|
|
242
|
+
"""One label record at ``p``: ``<LEB128 length><chunks>`` where each chunk
|
|
243
|
+
is ``<len:1><data>``, non-final chunks are exactly 127 bytes, and the
|
|
244
|
+
concatenated data is ``<text><NUL>``.
|
|
245
|
+
|
|
246
|
+
The common short label is the single-chunk case (the old fixed-shape
|
|
247
|
+
read). A >127-byte label — a long column comment — stores a 2-byte
|
|
248
|
+
varint length and 127-byte chunking; the old single-byte read silently
|
|
249
|
+
dropped that column's WHOLE label (long-name included), the §13.2 #13
|
|
250
|
+
residual, pinned by the ``long_comment.opju`` specimen (record:
|
|
251
|
+
varint ``e8 05`` = 744, chunks 127×5 + 103, text 737)."""
|
|
252
|
+
length = b[p]
|
|
253
|
+
off = 1
|
|
254
|
+
if length & 0x80: # LEB128 continuation bit -> 2-byte varint
|
|
255
|
+
if p + 2 > len(b):
|
|
256
|
+
return None
|
|
257
|
+
length = (length & 0x7F) | (b[p + 1] << 7)
|
|
258
|
+
off = 2
|
|
259
|
+
if length <= 0x7F or length > 4096: # a real 2-byte varint is >127
|
|
260
|
+
return None
|
|
261
|
+
elif length < 2:
|
|
262
|
+
return None
|
|
263
|
+
end_rec = p + off + length
|
|
264
|
+
if end_rec > len(b):
|
|
265
|
+
return None
|
|
266
|
+
payload = b[p + off : end_rec]
|
|
267
|
+
data = bytearray()
|
|
268
|
+
i = 0
|
|
269
|
+
while i < len(payload):
|
|
270
|
+
clen = payload[i]
|
|
271
|
+
last = i + 1 + clen == len(payload)
|
|
272
|
+
if clen == 0 or (not last and clen != 0x7F) or i + 1 + clen > len(payload):
|
|
273
|
+
return None
|
|
274
|
+
data += payload[i + 1 : i + 1 + clen]
|
|
275
|
+
i += 1 + clen
|
|
276
|
+
if not data or data[-1] != 0:
|
|
277
|
+
return None
|
|
278
|
+
return bytes(data[:-1])
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _find_label(b: bytes, start: int, end: int) -> str | None:
|
|
282
|
+
"""First label-shaped record in ``[start, end)``.
|
|
283
|
+
|
|
284
|
+
Prefers a ``\\r\\n``-split multi-row label (long_name/unit/comment); falls
|
|
285
|
+
back to a single-row (long-name-only) record when no multi-row match
|
|
286
|
+
exists in range.
|
|
287
|
+
"""
|
|
288
|
+
multi = single = None
|
|
289
|
+
limit = min(end, len(b) - 2)
|
|
290
|
+
for p in range(start, limit):
|
|
291
|
+
text = _parse_label_record(b, p)
|
|
292
|
+
if text is None or not text:
|
|
293
|
+
continue
|
|
294
|
+
if any(marker in text for marker in _JUNK_MARKERS):
|
|
295
|
+
continue
|
|
296
|
+
decoded = _decode_cell_text(text)
|
|
297
|
+
if decoded is None:
|
|
298
|
+
continue
|
|
299
|
+
if "\r\n" in decoded:
|
|
300
|
+
multi = decoded
|
|
301
|
+
break
|
|
302
|
+
if single is None and _is_single_row_label(decoded):
|
|
303
|
+
if len(text) >= 4 and any(text in token for token in _KNOWN_TOKENS):
|
|
304
|
+
continue
|
|
305
|
+
single = decoded
|
|
306
|
+
return multi if multi is not None else single
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def opju_window_metadata(
|
|
310
|
+
b: bytes, book_columns: Mapping[str, Sequence[str]]
|
|
311
|
+
) -> dict[str, BookMeta]:
|
|
312
|
+
"""Map book short name -> :class:`~quantized.io.origin_project.windows.BookMeta`.
|
|
313
|
+
|
|
314
|
+
``book_columns`` is the already-decoded ``{book: [col_letter, ...]}``
|
|
315
|
+
ordering (see ``opj._group``) — this function never invents a column
|
|
316
|
+
that ``opju_codec.scan_columns`` didn't independently decode; it only
|
|
317
|
+
attaches a label to it when the book's marker run can be confirmed.
|
|
318
|
+
"""
|
|
319
|
+
start = tail_start(b)
|
|
320
|
+
markers = sorted(
|
|
321
|
+
[(m.start(), "X") for m in re.finditer(_X_MARK, b) if m.start() >= start]
|
|
322
|
+
+ [(m.start(), "Y") for m in re.finditer(_Y_MARK, b) if m.start() >= start]
|
|
323
|
+
+ [(m.start(), "Y-error") for m in re.finditer(_YERR_MARK, b) if m.start() >= start]
|
|
324
|
+
+ [(m.start(), "label") for m in re.finditer(_LABEL_MARK, b) if m.start() >= start]
|
|
325
|
+
+ [
|
|
326
|
+
(m.start(), _zx_designation(b, m.start()))
|
|
327
|
+
for m in re.finditer(_ZX_MARK, b)
|
|
328
|
+
if m.start() >= start
|
|
329
|
+
]
|
|
330
|
+
)
|
|
331
|
+
filename_hits = _filename_anchors(b, start)
|
|
332
|
+
books: dict[str, BookMeta] = {}
|
|
333
|
+
# Anchor each book INDEPENDENTLY from the tail start (not a forward-only
|
|
334
|
+
# cursor): a project's book windows are not laid out in decoded-book order
|
|
335
|
+
# (the Hc2 project interleaves them), so a monotonic cursor drops every book
|
|
336
|
+
# whose window section sits before an already-anchored one. Anchors are exact
|
|
337
|
+
# (length-prefixed name / filename basename), so independent search cannot
|
|
338
|
+
# cross-match a different book.
|
|
339
|
+
for book, cols in book_columns.items():
|
|
340
|
+
if not cols:
|
|
341
|
+
continue
|
|
342
|
+
anchored = _book_anchor(b, book, start, filename_hits)
|
|
343
|
+
if anchored is None:
|
|
344
|
+
continue
|
|
345
|
+
anchor, raw_filename = anchored
|
|
346
|
+
candidates = [t for t in markers if t[0] > anchor]
|
|
347
|
+
x_positions = [i for i, t in enumerate(candidates) if t[1] == "X"]
|
|
348
|
+
letter_map: dict[str, tuple[int, str]] | None = None
|
|
349
|
+
run: list[tuple[int, str]] = []
|
|
350
|
+
# The run bound is DERIVED from the column count (the old fixed
|
|
351
|
+
# _MAX_RUN=128 cap silently dropped ALL metadata of a >128-column
|
|
352
|
+
# book — names, units, and the X designation; 2026-07-06 genericity
|
|
353
|
+
# audit). The >_MAX_GAP break remains the real book-boundary signal;
|
|
354
|
+
# len(cols) + slack is only the runaway backstop.
|
|
355
|
+
max_run = len(cols) + 32
|
|
356
|
+
for xi in x_positions:
|
|
357
|
+
attempt = [candidates[xi]]
|
|
358
|
+
j = xi + 1
|
|
359
|
+
while j < len(candidates) and len(attempt) < max_run:
|
|
360
|
+
# The gap to the NEXT column marker is set by how much label
|
|
361
|
+
# content THIS column carries: long_name + unit + comment, and
|
|
362
|
+
# a comment can run to hundreds of chars. So the allowance is
|
|
363
|
+
# _MAX_GAP plus the length of the label record that actually
|
|
364
|
+
# sits in the gap (structural), not a fixed byte cap that a
|
|
365
|
+
# long comment silently overruns -- which used to drop the
|
|
366
|
+
# whole book's metadata (2026-07-06 genericity audit). The
|
|
367
|
+
# real book boundary is a multi-KB window header, far past any
|
|
368
|
+
# single label record, so this never merges two books.
|
|
369
|
+
gap = candidates[j][0] - attempt[-1][0]
|
|
370
|
+
allow = _MAX_GAP + _gap_text_span(b, attempt[-1][0] + 2, candidates[j][0])
|
|
371
|
+
if gap > allow:
|
|
372
|
+
break
|
|
373
|
+
attempt.append(candidates[j])
|
|
374
|
+
j += 1
|
|
375
|
+
if len(attempt) < len(cols):
|
|
376
|
+
continue
|
|
377
|
+
candidate_map = {_excel_col(i): attempt[i] for i in range(len(attempt))}
|
|
378
|
+
if all(c in candidate_map for c in cols):
|
|
379
|
+
letter_map, run = candidate_map, attempt
|
|
380
|
+
break
|
|
381
|
+
if letter_map is None:
|
|
382
|
+
continue
|
|
383
|
+
long_name = raw_filename.decode("latin1", errors="replace") if raw_filename else book
|
|
384
|
+
columns: dict[str, ColumnMeta] = {}
|
|
385
|
+
for col in cols:
|
|
386
|
+
pos, desig = letter_map[col]
|
|
387
|
+
ordinal = run.index(letter_map[col])
|
|
388
|
+
next_pos = run[ordinal + 1][0] if ordinal + 1 < len(run) else pos + 3000
|
|
389
|
+
label = _find_label(b, pos + 2, min(next_pos, pos + 3000))
|
|
390
|
+
rows = (label.split("\r\n") if label else []) + ["", "", ""]
|
|
391
|
+
columns[col] = ColumnMeta(col, _DESIGNATION.get(desig, "Y"), rows[0], rows[1], rows[2])
|
|
392
|
+
books[book] = BookMeta(book, long_name, columns)
|
|
393
|
+
return books
|