gleplot 6.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
gleplot/dataio.py ADDED
@@ -0,0 +1,1403 @@
1
+ """Pure-Python delimited data file loading and resolution for gleplot.
2
+
3
+ This module has no Qt dependency so it can be imported by the core GLE
4
+ parser (``gleplot.parser``). It provides two layers:
5
+
6
+ 1. **Loading** (:func:`load_data_file`, :class:`DataTable`): parses
7
+ comma/tab/semicolon/whitespace-delimited text files (``.csv``,
8
+ ``.dat``, ``.txt``, ...) into a column-oriented :class:`DataTable`.
9
+ :func:`load_data_file` also accepts explicit, keyword-only overrides
10
+ (``delimiter``, ``header``, ``comment_chars``, ``skip_rows``) for
11
+ consumers that need to bypass detection for part or all of a file --
12
+ see its docstring. Every override defaults to "detect exactly as
13
+ before", so existing callers are unaffected.
14
+
15
+ 2. **Resolution** (:func:`resolve_data_reference`, :func:`extract_columns`,
16
+ :func:`classify_data_file`): a parser-facing layer used to turn a
17
+ ``data "file.dat" d1=c1,c2`` command referenced from a ``.gle`` script
18
+ into loaded numpy arrays, without ever raising for missing/unreadable/
19
+ malformed files -- callers get structured error information back
20
+ instead, so a broken data reference can be surfaced in an import report
21
+ rather than crashing the parse.
22
+
23
+ Format handling (loading layer)
24
+ --------------------------------
25
+ - Delimiter sniffing via :class:`csv.Sniffer` on a sample of the file,
26
+ falling back to arbitrary-whitespace splitting when sniffing fails or
27
+ the sniffed delimiter doesn't actually separate the row into multiple
28
+ fields.
29
+ - Comment lines (leading ``#`` or ``!``, ignoring leading whitespace) are
30
+ skipped entirely and never considered for header/data detection, EXCEPT
31
+ for two narrow, conservative cases (tried in this order) when the file
32
+ has no inline header row at all:
33
+
34
+ 1. **Indexed block** -- an instrument/export-style comment block with one
35
+ ``c <int> = <name>`` line per column (case-insensitive, whitespace
36
+ around ``=`` tolerated), e.g.::
37
+
38
+ ! some prose header
39
+ ! c 1 = run_id
40
+ ! c 2 = field_strength (G)
41
+ ! c 7 = err_rate (unit-1)
42
+ !
43
+ ! run_id field_strength(G) ... (aligned prose row, ignored)
44
+ 1 2.5 ...
45
+
46
+ The WHOLE comment block preceding the first data row is scanned (not
47
+ just the last line); accepted only if the discovered indices are
48
+ EXACTLY ``1..n_cols`` -- full coverage, no gaps/duplicates/out-of-range
49
+ (no partial adoption). Names may contain unicode and internal
50
+ whitespace verbatim (no tokenization: each line is matched whole).
51
+ Unambiguous, so it is tried FIRST and takes precedence over case 2
52
+ below. See :func:`_recover_indexed_comment_header`.
53
+ 2. **Last-line heuristic** -- the LAST comment line immediately preceding
54
+ the first data row (blank lines tolerated in between) is checked as a
55
+ possible comment-embedded column-name line -- e.g. a GLE-style
56
+ fit-parameter export ending in ``! x y`` right before the numeric
57
+ data. Accepted as column names only if, once the comment marker is
58
+ stripped and the remainder tokenized with the same delimiter as the
59
+ data, the token count matches the data column count AND at least one
60
+ token is a genuine non-numeric label (same rule as header detection
61
+ below). See :func:`_recover_comment_header`.
62
+
63
+ On acceptance (either case) ``DataTable.column_names`` gets the recovered
64
+ names and ``DataTable.header_source`` is set to ``"comment"``, but
65
+ ``DataTable.has_header`` stays ``False`` -- the line(s) are still
66
+ comments, so data-row indexing and any round-trip are unaffected; this
67
+ is display-name recovery only.
68
+ - Header detection: the first non-comment, non-blank row is treated as a
69
+ header if and only if at least one of its fields is a genuine
70
+ non-numeric *label* -- i.e. it fails float conversion AND is not a
71
+ missing-value token. A first data row consisting entirely of missing
72
+ tokens (e.g. ``* * *``) is therefore kept as an all-NaN DATA row, not
73
+ mistaken for a header.
74
+ - Header/column alignment: if a detected header's token count does not
75
+ match the data column count (e.g. a whitespace-split multi-word header
76
+ like ``Temperature (K) Resistance (Ohm)`` yielding 4 tokens over 2
77
+ data columns), the row is still skipped as a header, but its tokens are
78
+ not used to name columns; positional names (``col1``..``colN``) are
79
+ synthesized and a warning is recorded on ``DataTable.warnings``.
80
+ Delimited files (comma/tab/semicolon) split multi-word names correctly,
81
+ so their headers align and are used verbatim.
82
+ - Missing values: empty fields and the GLE-convention tokens ``*``,
83
+ ``?``, ``-``, ``.`` (only when they are the *entire* field, not part
84
+ of a longer token) as well as ``nan``/``NaN`` (case-insensitive) become
85
+ ``np.nan``.
86
+ - Ragged rows (inconsistent field counts) are right-padded with NaN up to
87
+ the widest row seen; a human-readable warning is recorded for each
88
+ padded row on ``DataTable.warnings``.
89
+ - Columns where every non-missing value parses as a float are numeric
90
+ (``dtype=float64``); otherwise the column is kept as-is with
91
+ ``dtype=object`` and flagged not numeric via ``DataTable.is_numeric``.
92
+ - Encoding: ``utf-8-sig`` first (tolerates a UTF-8 BOM and plain UTF-8
93
+ files alike), falling back to ``latin-1`` so no file fails to load
94
+ outright.
95
+ """
96
+
97
+ from __future__ import annotations
98
+
99
+ import csv
100
+ import io
101
+ import itertools
102
+ import re
103
+ from dataclasses import dataclass, field
104
+ from pathlib import Path
105
+ from typing import Callable, Iterator, List, Optional, Union
106
+
107
+ import numpy as np
108
+
109
+ __all__ = [
110
+ "DataTable",
111
+ "load_data_file",
112
+ "WHITESPACE_DELIMITER",
113
+ "ResolvedData",
114
+ "resolve_data_reference",
115
+ "extract_columns",
116
+ "ColumnExtractionError",
117
+ "classify_data_file",
118
+ ]
119
+
120
+ #: Standalone tokens (case-sensitive, whole-field match) treated as missing
121
+ #: data, per GLE conventions plus common conventions from other tools.
122
+ _MISSING_TOKENS = {"*", "?", "-", "."}
123
+ #: Additional case-insensitive missing tokens.
124
+ _MISSING_TOKENS_CI = {"nan", ""}
125
+
126
+ #: Delimiters tried by the Sniffer / candidate list, in preference order.
127
+ _CANDIDATE_DELIMITERS = [",", "\t", ";"]
128
+
129
+ #: ``load_data_file(delimiter=...)`` override value denoting explicit
130
+ #: whitespace-splitting, as opposed to ``delimiter=None`` (the default),
131
+ #: which means "sniff/detect". Reuses the same token
132
+ #: :attr:`DataTable.delimiter` already uses to *report* whitespace-splitting,
133
+ #: so a caller can round-trip a previously-reported delimiter straight back
134
+ #: in as an override.
135
+ WHITESPACE_DELIMITER = r"\s+"
136
+
137
+
138
+ @dataclass
139
+ class DataTable:
140
+ """A loaded delimited data file, column-oriented.
141
+
142
+ Attributes
143
+ ----------
144
+ column_names : list of str
145
+ One name per column. Synthesized as ``col1``, ``col2``, ... when
146
+ the file has no header row.
147
+ columns : list of numpy.ndarray
148
+ One array per column, same order as ``column_names``. Numeric
149
+ columns have ``dtype=float64`` (missing values are ``np.nan``);
150
+ non-numeric columns have ``dtype=object`` holding the raw string
151
+ tokens (missing values are ``None``).
152
+ is_numeric : list of bool
153
+ Per-column flag: ``True`` if the column is a numeric float array.
154
+ n_rows : int
155
+ Number of data rows (excludes the header row and comment lines).
156
+ path : pathlib.Path
157
+ Path the table was loaded from.
158
+ delimiter : str
159
+ The delimiter used to split fields. ``r"\\s+"`` denotes
160
+ whitespace-splitting rather than a literal single-character
161
+ delimiter.
162
+ has_header : bool
163
+ Whether the first non-comment row was consumed as a header.
164
+ warnings : list of str
165
+ Human-readable warnings, e.g. about ragged rows that were padded.
166
+ header_source : str or None
167
+ Where ``column_names`` came from: ``"row"`` when an inline header
168
+ row was consumed (``has_header`` is ``True``), ``"comment"`` when
169
+ no inline header row exists but names were recovered from a
170
+ trailing comment line (see :func:`load_data_file`'s comment-header
171
+ recovery; ``has_header`` stays ``False`` in this case -- the line
172
+ is still a comment, data-row indexing is unaffected), or ``None``
173
+ when ``column_names`` are the synthesized positional
174
+ ``col1``..``colN`` placeholders.
175
+ comment_lines_skipped : int
176
+ Number of non-blank lines recognized as comments (and therefore
177
+ excluded from header/data detection) -- per the built-in ``#``/``!``
178
+ convention, or per ``load_data_file``'s ``comment_chars`` override
179
+ when one was given. Counted after ``skip_rows`` leading lines (if
180
+ any) were dropped. Together with ``delimiter`` and ``has_header``,
181
+ this is enough for a caller to report "delimiter found / header
182
+ found / N comment lines skipped" without recomputing anything.
183
+ gle_missing : list of bool
184
+ Per-column flag: ``True`` if a field in that column is one of GLE's
185
+ missing-value tokens (``*``, ``?``, ``-``, ``.``), which break a
186
+ line where a ``nan`` (also loaded as NaN) is dropped and its
187
+ neighbours joined.
188
+ """
189
+
190
+ column_names: List[str]
191
+ columns: List[np.ndarray]
192
+ n_rows: int
193
+ path: Path
194
+ delimiter: str
195
+ has_header: bool
196
+ is_numeric: List[bool] = field(default_factory=list)
197
+ warnings: List[str] = field(default_factory=list)
198
+ header_source: Optional[str] = None
199
+ comment_lines_skipped: int = 0
200
+ gle_missing: List[bool] = field(default_factory=list)
201
+
202
+ @property
203
+ def n_cols(self) -> int:
204
+ """Number of columns."""
205
+ return len(self.column_names)
206
+
207
+ def numeric_column_names(self) -> List[str]:
208
+ """Names of columns that are numeric (selectable as plot data)."""
209
+ return [
210
+ name for name, numeric in zip(self.column_names, self.is_numeric) if numeric
211
+ ]
212
+
213
+ def numeric_column_indices(self) -> List[int]:
214
+ """0-based indices of columns that are numeric."""
215
+ return [i for i, numeric in enumerate(self.is_numeric) if numeric]
216
+
217
+
218
+ def _is_comment(line: str) -> bool:
219
+ stripped = line.lstrip()
220
+ return stripped.startswith("#") or stripped.startswith("!")
221
+
222
+
223
+ def _read_text(path: Union[str, Path]) -> str:
224
+ """Read file text, tolerating a UTF-8 BOM and falling back to latin-1."""
225
+ raw = Path(path).read_bytes()
226
+ try:
227
+ return raw.decode("utf-8-sig")
228
+ except UnicodeDecodeError:
229
+ return raw.decode("latin-1")
230
+
231
+
232
+ def _sniff_delimiter(sample_lines: List[str]) -> Optional[str]:
233
+ """Try csv.Sniffer on a sample; return a delimiter char or None."""
234
+ sample = "\n".join(sample_lines)
235
+ if not sample.strip():
236
+ return None
237
+ try:
238
+ dialect = csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS))
239
+ return dialect.delimiter
240
+ except csv.Error:
241
+ pass
242
+
243
+ # Sniffer failed (e.g. single column, ambiguous, or ragged rows with
244
+ # inconsistent field counts). Fall back to counting occurrences of
245
+ # each candidate delimiter across the sample lines; accept the one
246
+ # that appears on *most* non-empty lines (count > 0), even if the
247
+ # exact count varies row-to-row (ragged data) and even if a minority
248
+ # of rows lack it entirely (e.g. a single-field ragged row like a
249
+ # lone ``2`` in an otherwise comma-delimited file). A separator that
250
+ # is present on the majority of rows is a strong signal of a real
251
+ # field delimiter rather than incidental punctuation; the minority of
252
+ # rows that lack it are handled as ragged rows (right-padded with NaN)
253
+ # downstream. Prefer the delimiter present on the most rows, breaking
254
+ # ties by the most consistent per-row counts, then by candidate order.
255
+ non_empty = [ln for ln in sample_lines if ln.strip()]
256
+ if not non_empty:
257
+ return None
258
+ best_delim = None
259
+ best_key = None
260
+ for delim in _CANDIDATE_DELIMITERS:
261
+ counts = [ln.count(delim) for ln in non_empty]
262
+ present = [c for c in counts if c > 0]
263
+ # Require a strict majority of rows to contain the delimiter, so a
264
+ # stray comma in one row of a whitespace file doesn't hijack it.
265
+ if len(present) * 2 <= len(non_empty):
266
+ continue
267
+ # Consistency measured over the rows that actually have the
268
+ # delimiter; ragged single-field rows (count 0) don't count
269
+ # against it. Fewer distinct counts == more consistent.
270
+ variety = len(set(present))
271
+ # Higher presence first (negate for a min-style comparison), then
272
+ # lower variety. Candidate order breaks any remaining tie.
273
+ key = (-len(present), variety)
274
+ if best_key is None or key < best_key:
275
+ best_delim = delim
276
+ best_key = key
277
+ return best_delim
278
+
279
+
280
+ def _split_line(line: str, delimiter: Optional[str]) -> List[str]:
281
+ """Split a single line into fields using the resolved delimiter.
282
+
283
+ ``delimiter is None`` means whitespace-splitting.
284
+ """
285
+ if delimiter is None:
286
+ return line.split()
287
+ # csv module handles quoted fields correctly for comma/semicolon/tab.
288
+ reader = csv.reader(io.StringIO(line), delimiter=delimiter)
289
+ try:
290
+ return next(reader)
291
+ except StopIteration:
292
+ return []
293
+
294
+
295
+ def _normalize_token(token: str) -> Optional[str]:
296
+ """Return the token stripped, or None if it represents a missing value."""
297
+ stripped = token.strip()
298
+ if stripped in _MISSING_TOKENS:
299
+ return None
300
+ if stripped.lower() in _MISSING_TOKENS_CI:
301
+ return None
302
+ return stripped
303
+
304
+
305
+ def _try_float(token: Optional[str]) -> Optional[float]:
306
+ """Convert a normalized token to float, or None if missing/not numeric."""
307
+ if token is None:
308
+ return None
309
+ try:
310
+ return float(token)
311
+ except ValueError:
312
+ return None
313
+
314
+
315
+ def _strip_comment_marker(line: str) -> str:
316
+ """Strip a leading comment marker (``#`` or ``!``) and whitespace."""
317
+ stripped = line.lstrip()
318
+ if stripped.startswith("#") or stripped.startswith("!"):
319
+ stripped = stripped[1:]
320
+ return stripped.strip()
321
+
322
+
323
+ def _find_last_comment_before_first_data_row(raw_lines: List[str]) -> Optional[str]:
324
+ """Return the comment line immediately preceding the first data row.
325
+
326
+ "Immediately preceding" tolerates intervening blank lines (walking
327
+ backward from the first content line, skipping blanks) but stops as
328
+ soon as a non-blank, non-comment line is encountered first -- that
329
+ would mean there is no comment line directly above the first data row
330
+ (just more data / an earlier blank-separated block), so we
331
+ conservatively give up rather than risk grabbing an unrelated comment
332
+ from earlier in the file.
333
+ """
334
+ # Find the raw-line index of the first content line (same predicate
335
+ # used to build content_lines: non-blank and not a comment).
336
+ first_idx = None
337
+ for i, line in enumerate(raw_lines):
338
+ if line.strip() and not _is_comment(line):
339
+ first_idx = i
340
+ break
341
+ if first_idx is None or first_idx == 0:
342
+ return None
343
+
344
+ for i in range(first_idx - 1, -1, -1):
345
+ line = raw_lines[i]
346
+ if not line.strip():
347
+ continue
348
+ if _is_comment(line):
349
+ return line
350
+ # Hit a non-comment, non-blank line before any comment -- no
351
+ # comment line is "immediately preceding" the first data row.
352
+ return None
353
+ return None
354
+
355
+
356
+ #: Matches a comment-body line of the form ``c <int> = <name>`` (case-
357
+ #: insensitive, whitespace-tolerant around the ``=``). Group 1 is the 1-based
358
+ #: column index, group 2 is everything after ``=`` (stripped once by the
359
+ #: caller; internal whitespace in the name is preserved verbatim).
360
+ _INDEXED_COMMENT_HEADER_RE = re.compile(r"^c\s*(\d+)\s*=\s*(.*)$", re.IGNORECASE)
361
+
362
+
363
+ def _comment_block_before_first_data_row(raw_lines: List[str]) -> List[str]:
364
+ """Return every comment/blank line before the first data row, in order.
365
+
366
+ Unlike :func:`_find_last_comment_before_first_data_row` (which returns
367
+ only the single nearest comment line), this collects the WHOLE run of
368
+ comment lines preceding the first data row -- needed to scan a multi-line
369
+ ``! c N = name`` block where the column-naming lines are not necessarily
370
+ the very last comment line (a trailing ``!`` separator or an aligned
371
+ prose row -- see ``_recover_indexed_comment_header`` -- often follows
372
+ them). Blank lines are included (as empty strings) so ordering/spacing
373
+ is preserved for callers that care, but they never match the ``c N =``
374
+ pattern so they are harmless filler.
375
+
376
+ Stops (returns only what was collected so far) as soon as a non-blank,
377
+ non-comment line is reached while scanning backward from the first data
378
+ row -- i.e. only an unbroken run of comments/blanks immediately above
379
+ the data counts as "the comment block".
380
+ """
381
+ first_idx = None
382
+ for i, line in enumerate(raw_lines):
383
+ if line.strip() and not _is_comment(line):
384
+ first_idx = i
385
+ break
386
+ if first_idx is None or first_idx == 0:
387
+ return []
388
+
389
+ block: List[str] = []
390
+ for i in range(first_idx - 1, -1, -1):
391
+ line = raw_lines[i]
392
+ if not line.strip():
393
+ block.append(line)
394
+ continue
395
+ if _is_comment(line):
396
+ block.append(line)
397
+ continue
398
+ break
399
+ block.reverse()
400
+ return block
401
+
402
+
403
+ def _recover_indexed_comment_header(
404
+ raw_lines: List[str],
405
+ data_col_count: int,
406
+ ) -> Optional[List[str]]:
407
+ """Recover column names from a ``! c N = name`` comment block, if present.
408
+
409
+ Real-world shape (e.g. an instrument export with a prose preamble)::
410
+
411
+ ! some prose header
412
+ ! c 1 = run_id
413
+ ! c 2 = field_strength (G)
414
+ ...
415
+ ! c 7 = err_rate (unit-1)
416
+ !
417
+ ! run_id field_strength(G) ...
418
+ 1 2.5 ...
419
+
420
+ Scans the WHOLE comment block preceding the first data row (not just the
421
+ last line) for lines matching, case-insensitively and whitespace-
422
+ tolerantly, ``c <int> = <name>`` once the comment marker is stripped.
423
+ Everything after ``=`` becomes the column name, stripped only at the
424
+ ends (internal whitespace, and non-ASCII characters such as ``mu`` or
425
+ superscript ``-1``, are preserved verbatim -- multi-word names are fine
426
+ here since each line is matched whole, not tokenized).
427
+
428
+ This is unambiguous (each column index is explicitly named), so it takes
429
+ PRECEDENCE over the last-comment-line positional heuristic
430
+ (:func:`_recover_comment_header`) -- callers should try this function
431
+ FIRST and only fall back to the positional heuristic if it returns
432
+ ``None``.
433
+
434
+ Accepted only when the discovered indices are EXACTLY ``{1, ..., n_cols}``
435
+ -- full coverage, no gaps, no duplicates, no out-of-range indices. This
436
+ is deliberately strict (no partial/subset adoption): an incomplete or
437
+ ambiguous match is much more likely to be a coincidental line (e.g. a
438
+ sentence containing "c 2 = ...") than a genuine column-naming block, and
439
+ a wrong-but-plausible mapping silently mislabels data. Any duplicate
440
+ index, any index outside ``1..n_cols``, or missing coverage rejects the
441
+ WHOLE block (returns ``None``) rather than guessing.
442
+
443
+ A trailing aligned "prose name row" (e.g. a human-readable column
444
+ header whose whitespace-split token count does not match
445
+ ``data_col_count``) commonly follows the ``c N =`` lines in this file
446
+ shape; it is irrelevant here since this function only reads ``c N =``
447
+ lines and ignores everything else in the block.
448
+ """
449
+ block = _comment_block_before_first_data_row(raw_lines)
450
+ if not block:
451
+ return None
452
+
453
+ names_by_index: dict = {}
454
+ for line in block:
455
+ if not _is_comment(line):
456
+ continue
457
+ candidate = _strip_comment_marker(line)
458
+ if not candidate:
459
+ continue
460
+ m = _INDEXED_COMMENT_HEADER_RE.match(candidate.strip())
461
+ if not m:
462
+ continue
463
+ idx = int(m.group(1))
464
+ name = m.group(2).strip()
465
+ if not name:
466
+ continue
467
+ if idx in names_by_index:
468
+ # Duplicate index -- ambiguous, reject the whole block.
469
+ return None
470
+ names_by_index[idx] = name
471
+
472
+ if not names_by_index:
473
+ return None
474
+
475
+ found = set(names_by_index)
476
+ expected = set(range(1, data_col_count + 1))
477
+ if found != expected:
478
+ # Incomplete coverage, gap, or an out-of-range index -- reject
479
+ # rather than guess (see docstring: no partial adoption).
480
+ return None
481
+
482
+ return [names_by_index[i] for i in range(1, data_col_count + 1)]
483
+
484
+
485
+ def _recover_comment_header(
486
+ raw_lines: List[str],
487
+ delimiter: Optional[str],
488
+ data_col_count: int,
489
+ ) -> Optional[List[str]]:
490
+ """Recover column names from a trailing comment line, if it qualifies.
491
+
492
+ Only called when the file has no inline header row. Looks at the
493
+ LAST comment line immediately preceding the first data row (tolerating
494
+ intervening blank lines), strips the comment marker, and tokenizes it
495
+ with the same delimiter used for the data. The candidate is accepted
496
+ as column names ONLY IF:
497
+
498
+ - its token count equals the data column count, AND
499
+ - at least one token is a genuine non-numeric label (fails float
500
+ conversion AND is not a missing-value token) -- the same rule
501
+ ``load_data_file`` uses to decide a row is a real header rather
502
+ than data, reused here so an all-numeric comment (e.g. a stray
503
+ results line someone commented out) is never mistaken for names.
504
+
505
+ Returns ``None`` when no comment line qualifies, leaving
506
+ ``column_names``/``has_header`` untouched by this feature entirely.
507
+ """
508
+ comment_line = _find_last_comment_before_first_data_row(raw_lines)
509
+ if comment_line is None:
510
+ return None
511
+
512
+ candidate_text = _strip_comment_marker(comment_line)
513
+ if not candidate_text:
514
+ return None
515
+
516
+ tokens = _split_line(candidate_text, delimiter)
517
+ tokens = [t.strip() for t in tokens]
518
+
519
+ if len(tokens) != data_col_count:
520
+ return None
521
+
522
+ has_label = any(
523
+ _normalize_token(tok) is not None and _try_float(_normalize_token(tok)) is None
524
+ for tok in tokens
525
+ )
526
+ if not has_label:
527
+ return None
528
+
529
+ return tokens
530
+
531
+
532
+ def load_data_file(
533
+ path: Union[str, Path],
534
+ max_preview_rows: Optional[int] = None,
535
+ *,
536
+ delimiter: Optional[str] = None,
537
+ header: Optional[bool] = None,
538
+ comment_chars: Optional[str] = None,
539
+ skip_rows: int = 0,
540
+ ) -> DataTable:
541
+ """Load a delimited text data file into a :class:`DataTable`.
542
+
543
+ Parameters
544
+ ----------
545
+ path : str or pathlib.Path
546
+ Path to the data file (``.csv``, ``.dat``, ``.txt``, or any
547
+ delimited text file).
548
+ max_preview_rows : int, optional
549
+ If given, only the first ``max_preview_rows`` data rows are
550
+ parsed into ``columns`` (the rest of the file is ignored). This
551
+ is a performance aid for large files when only a preview is
552
+ needed; ``n_rows`` reflects the number of rows actually loaded.
553
+ delimiter : str, optional
554
+ Explicit override for field splitting, bypassing
555
+ :class:`csv.Sniffer`-based detection entirely. ``None`` (the
556
+ default) sniffs/detects the delimiter exactly as before -- byte-
557
+ for-byte identical to calling this function without the keyword
558
+ at all. Pass a single character (``","``, ``"\\t"``, ``";"``, an
559
+ arbitrary custom character, ...) to force that literal delimiter,
560
+ or :data:`WHITESPACE_DELIMITER` (``r"\\s+"`` -- the same token
561
+ :attr:`DataTable.delimiter` uses to *report* whitespace-splitting)
562
+ to force arbitrary-whitespace splitting. An explicit override is
563
+ honored exactly as given: unlike detection, it is not
564
+ second-guessed by the "does it actually split the first row into
565
+ more than one field" fallback that sniffing uses -- if a caller
566
+ forces a delimiter that doesn't apply to the file, that is
567
+ reflected as one wide ragged column, not silently overridden.
568
+ header : bool, optional
569
+ Explicit override for whether the first content row is a header.
570
+ ``None`` (the default) auto-detects exactly as before: a row is a
571
+ header iff at least one field is a genuine non-numeric label.
572
+ ``True``/``False`` forces that decision, bypassing the heuristic
573
+ entirely. A forced ``header=False`` still allows comment-header
574
+ recovery to supply display names from a qualifying comment line
575
+ (see ``comment_chars`` below); a forced ``header=True`` never
576
+ does -- an explicit/inline header always takes precedence, the
577
+ same rule that already applies to auto-detected headers.
578
+ comment_chars : str, optional
579
+ Explicit override for which leading characters mark a
580
+ standalone-line comment. ``None`` (the default) uses the built-in
581
+ convention (``#`` or ``!``, i.e. :func:`_is_comment`) unchanged.
582
+ A non-``None`` value **replaces** the built-in set entirely (it is
583
+ not additive): e.g. ``comment_chars=";"`` recognizes only
584
+ ``;``-prefixed lines as comments, and ``#``/``!`` lines become
585
+ ordinary content; a caller that wants ``;`` *in addition to* the
586
+ defaults should pass ``comment_chars="#!;"``. ``comment_chars=""``
587
+ disables comment-line detection entirely (every non-blank line is
588
+ content). Because the comment-header recovery heuristics
589
+ (:func:`_recover_indexed_comment_header`,
590
+ :func:`_recover_comment_header`) are hardcoded to the ``#``/``!``
591
+ convention, they are only attempted when ``comment_chars is
592
+ None`` -- running them against a different marker set could
593
+ recover a "header" from a line that was actually treated as
594
+ ordinary data (or the reverse), so when ``comment_chars`` is
595
+ overridden, positional ``col1``..``colN`` names are used instead
596
+ whenever there is no inline header.
597
+ skip_rows : int, optional
598
+ Number of leading raw lines to drop before any other processing
599
+ -- comment filtering, delimiter sniffing, and header detection all
600
+ run only on what remains. Default ``0`` (nothing skipped, and
601
+ thus no behavior change). Applied first, so it composes with
602
+ every other parameter.
603
+
604
+ Returns
605
+ -------
606
+ DataTable
607
+
608
+ Raises
609
+ ------
610
+ FileNotFoundError
611
+ If ``path`` does not exist.
612
+ ValueError
613
+ If the file contains no data rows at all (after ``skip_rows`` and
614
+ comment/blank filtering).
615
+
616
+ Notes
617
+ -----
618
+ With every keyword-only parameter left at its default, behavior is
619
+ byte-for-byte identical to calling this function with just ``path``
620
+ (and, optionally, ``max_preview_rows``) as before: the detection
621
+ heuristics, missing-value conventions, and numeric-column typing this
622
+ module has always used are all unchanged for existing callers.
623
+ """
624
+ path = Path(path)
625
+ if not path.exists():
626
+ raise FileNotFoundError(f"Data file not found: {path}")
627
+
628
+ text = _read_text(path)
629
+ raw_lines = text.splitlines()
630
+ if skip_rows:
631
+ raw_lines = raw_lines[skip_rows:]
632
+
633
+ is_comment_line: Callable[[str], bool]
634
+ if comment_chars is None:
635
+ is_comment_line = _is_comment
636
+ else:
637
+ _comment_prefixes = tuple(comment_chars)
638
+
639
+ def is_comment_line(line: str) -> bool:
640
+ return line.lstrip().startswith(_comment_prefixes)
641
+
642
+ # Filter out comment and blank lines up front; header/data detection
643
+ # and delimiter sniffing both operate on this filtered list only.
644
+ content_lines = [ln for ln in raw_lines if ln.strip() and not is_comment_line(ln)]
645
+ comment_lines_skipped = sum(
646
+ 1 for ln in raw_lines if ln.strip() and is_comment_line(ln)
647
+ )
648
+
649
+ if not content_lines:
650
+ raise ValueError(f"No data rows found in {path}")
651
+
652
+ sample_lines = content_lines[:20]
653
+ if delimiter is None:
654
+ resolved_delimiter = _sniff_delimiter(sample_lines)
655
+
656
+ # Validate the sniffed delimiter actually splits the first line into
657
+ # more than one field; otherwise fall back to whitespace splitting.
658
+ if resolved_delimiter is not None:
659
+ first_fields = _split_line(content_lines[0], resolved_delimiter)
660
+ if len(first_fields) <= 1:
661
+ resolved_delimiter = None
662
+ elif delimiter == WHITESPACE_DELIMITER:
663
+ resolved_delimiter = None
664
+ else:
665
+ # Explicit override: honored as given (see docstring), not
666
+ # second-guessed by the single-field fallback above, which only
667
+ # applies to sniffed delimiters.
668
+ resolved_delimiter = delimiter
669
+
670
+ split_rows = [_split_line(line, resolved_delimiter) for line in content_lines]
671
+
672
+ # Header detection: the first row is a header iff at least one field is a
673
+ # genuine non-numeric label -- i.e. it fails float conversion AND is not a
674
+ # missing-value token. A missing token ('*', '?', '-', '.', '', 'nan')
675
+ # also "fails float conversion", but a first data row consisting entirely
676
+ # of missing values (e.g. '* * *') is real DATA (all-NaN), not a header;
677
+ # only a field that is a real word/label marks the row as a header.
678
+ # ``header`` overrides this heuristic entirely when given.
679
+ first_row = split_rows[0]
680
+ if header is None:
681
+ has_header = any(
682
+ _normalize_token(tok) is not None
683
+ and _try_float(_normalize_token(tok)) is None
684
+ for tok in first_row
685
+ )
686
+ else:
687
+ has_header = header
688
+
689
+ if has_header:
690
+ header_fields = [f.strip() for f in first_row]
691
+ data_rows = split_rows[1:]
692
+ else:
693
+ header_fields = None
694
+ data_rows = split_rows
695
+
696
+ if max_preview_rows is not None:
697
+ data_rows = data_rows[:max_preview_rows]
698
+
699
+ if not data_rows:
700
+ raise ValueError(f"No data rows found in {path}")
701
+
702
+ warnings: List[str] = []
703
+
704
+ # Column count is determined by the DATA rows, not the header. A header
705
+ # whose token count differs from the data column count is a *misalignment*
706
+ # (typically a whitespace-split multi-word header, e.g. header
707
+ # 'Temperature (K) Resistance (Ohm)' = 4 tokens over 2 data columns): we
708
+ # still skip the row as a header (it is not data), but we cannot trust its
709
+ # tokens to name columns 1:1, so we synthesize positional names and warn.
710
+ data_col_count = max(len(row) for row in data_rows)
711
+ header_mismatch = header_fields is not None and len(header_fields) != data_col_count
712
+ if header_mismatch:
713
+ warnings.append(
714
+ f"Header row has {len(header_fields or [])} field(s) but the data has "
715
+ f"{data_col_count} column(s); the header does not align with the "
716
+ "columns (e.g. multi-word names split on whitespace). Using "
717
+ "positional column names (col1..colN) instead."
718
+ )
719
+ max_cols = data_col_count
720
+
721
+ padded_rows: List[List[str]] = []
722
+ for row_idx, row in enumerate(data_rows):
723
+ if len(row) < max_cols:
724
+ pad_count = max_cols - len(row)
725
+ warnings.append(
726
+ f"Row {row_idx + 1} has {len(row)} field(s), expected "
727
+ f"{max_cols}; padded {pad_count} missing value(s) with NaN."
728
+ )
729
+ row = row + [""] * pad_count
730
+ elif len(row) > max_cols:
731
+ # Shouldn't normally happen since max_cols is the max data-row
732
+ # width, but guard defensively.
733
+ warnings.append(
734
+ f"Row {row_idx + 1} has {len(row)} field(s), expected "
735
+ f"{max_cols}; extra field(s) truncated."
736
+ )
737
+ row = row[:max_cols]
738
+ padded_rows.append(row)
739
+
740
+ header_source: Optional[str] = None
741
+ if header_fields is None or header_mismatch:
742
+ # No header, or a header that doesn't align 1:1 with the columns.
743
+ column_names = [f"col{i + 1}" for i in range(max_cols)]
744
+ if header_mismatch:
745
+ # The row WAS consumed as a header (has_header is True) even
746
+ # though its tokens couldn't be trusted to name columns 1:1;
747
+ # header_source still reflects that an inline row was found.
748
+ header_source = "row"
749
+
750
+ # Comment-header recovery: real-world .dat files (e.g. GLE-style
751
+ # fit-parameter exports) sometimes carry column names in a COMMENT
752
+ # line rather than an inline header row -- e.g.
753
+ # ! Fit parameter data for GLE export
754
+ # ! Global fitting parameters:
755
+ # ! A_1 (%) = 11.8654 +/- 0.0543966
756
+ # ! x y
757
+ # 1.0 2.0
758
+ # Only attempted when there is no inline header at all (a header
759
+ # row, even a mismatched one, always wins -- see the recognizer's
760
+ # `_recovered_column_names`, which only ever reads `column_names`
761
+ # when `has_header` is True, so this path can never be mistaken
762
+ # for a real header there). Deliberately conservative: most
763
+ # comment lines are NOT headers, so only the LAST comment line
764
+ # immediately before the first data row is even considered, and
765
+ # only accepted if its token count matches the data columns AND
766
+ # it contains a genuine non-numeric label (same rule as normal
767
+ # header detection above) -- see _recover_comment_header.
768
+ #
769
+ # `not header_mismatch` here means genuinely `has_header is False`
770
+ # (not just "mismatched"): we are inside the
771
+ # `header_fields is None or header_mismatch` branch, so if
772
+ # `header_mismatch` is False then `header_fields is None` must be
773
+ # the reason we're here -- i.e. there was no inline header row at
774
+ # all. A mismatched inline header (has_header True, tokens
775
+ # misaligned) is still a real header row and must NOT be
776
+ # second-guessed by a comment line.
777
+ #
778
+ # Also gated on `comment_chars is None`: the recovery heuristics
779
+ # below are hardcoded to the built-in '#'/'!' convention (see
780
+ # docstring), so they only run when that convention is actually
781
+ # what was used to filter comments out above -- otherwise a line
782
+ # treated as data by the (overridden) filter could be mistaken for
783
+ # a recoverable comment header, or vice versa.
784
+ if not header_mismatch and comment_chars is None:
785
+ # Indexed '! c N = name' block (see _recover_indexed_comment_header)
786
+ # is unambiguous -- every column is explicitly named by number --
787
+ # so it takes PRECEDENCE over the positional last-comment-line
788
+ # heuristic below. Only fall back to the positional heuristic
789
+ # when no indexed block is found (or it doesn't achieve full
790
+ # 1..n_cols coverage).
791
+ recovered = _recover_indexed_comment_header(raw_lines, max_cols)
792
+ if recovered is not None:
793
+ column_names = recovered
794
+ header_source = "comment"
795
+ else:
796
+ recovered = _recover_comment_header(
797
+ raw_lines, resolved_delimiter, max_cols
798
+ )
799
+ if recovered is not None:
800
+ column_names = recovered
801
+ header_source = "comment"
802
+ else:
803
+ column_names = list(header_fields)
804
+ # Pad header names if the header row itself was short.
805
+ while len(column_names) < max_cols:
806
+ column_names.append(f"col{len(column_names) + 1}")
807
+ # Fill blank header names with a positional placeholder.
808
+ column_names = [
809
+ name if name else f"col{i + 1}" for i, name in enumerate(column_names)
810
+ ]
811
+ header_source = "row"
812
+
813
+ # Build columns.
814
+ columns: List[np.ndarray] = []
815
+ is_numeric: List[bool] = []
816
+ gle_missing: List[bool] = []
817
+ n_rows = len(padded_rows)
818
+
819
+ for col_idx in range(max_cols):
820
+ raw_values = [row[col_idx] for row in padded_rows]
821
+ gle_missing.append(any(v.strip() in _MISSING_TOKENS for v in raw_values))
822
+ normalized = [_normalize_token(v) for v in raw_values]
823
+ floats = [_try_float(v) for v in normalized]
824
+
825
+ # Numeric iff every non-missing token parsed as a float. A
826
+ # column that is entirely missing is treated as numeric (all NaN)
827
+ # since there's no string content to preserve.
828
+ column_is_numeric = all(
829
+ f is not None or n is None for f, n in zip(floats, normalized)
830
+ )
831
+
832
+ if column_is_numeric:
833
+ arr = np.array(
834
+ [np.nan if f is None else f for f in floats], dtype=np.float64
835
+ )
836
+ columns.append(arr)
837
+ is_numeric.append(True)
838
+ else:
839
+ obj_values = [n for n in normalized] # None for missing, str otherwise
840
+ arr = np.array(obj_values, dtype=object)
841
+ columns.append(arr)
842
+ is_numeric.append(False)
843
+
844
+ return DataTable(
845
+ column_names=column_names,
846
+ columns=columns,
847
+ n_rows=n_rows,
848
+ path=path,
849
+ delimiter=resolved_delimiter if resolved_delimiter is not None else r"\s+",
850
+ has_header=has_header,
851
+ is_numeric=is_numeric,
852
+ warnings=warnings,
853
+ header_source=header_source,
854
+ comment_lines_skipped=comment_lines_skipped,
855
+ gle_missing=gle_missing,
856
+ )
857
+
858
+
859
+ # ---------------------------------------------------------------------------
860
+ # Parser-facing data resolution layer
861
+ # ---------------------------------------------------------------------------
862
+
863
+
864
+ @dataclass
865
+ class ResolvedData:
866
+ """Result of resolving a ``data "..."`` reference from a ``.gle`` script.
867
+
868
+ Exactly one of the following holds:
869
+
870
+ - Success: ``exists`` is ``True``, ``resolved_path`` is set, ``table``
871
+ is a loaded :class:`DataTable`, ``error`` is ``None``.
872
+ - Failure (missing file, unreadable, malformed/empty): ``table`` is
873
+ ``None`` and ``error`` holds a human-readable message. ``exists``
874
+ reflects whether a filesystem entry was found at all (``False`` for
875
+ "file not found", ``True`` for "found but failed to parse" such as
876
+ an unreadable/empty file), and ``resolved_path`` is still populated
877
+ with the path that was attempted (useful for diagnostics / "broken
878
+ series" UI even when nothing could be loaded).
879
+
880
+ This type never results from a raised exception -- construction
881
+ always goes through :func:`resolve_data_reference`, which catches
882
+ everything :func:`load_data_file` can raise.
883
+
884
+ Attributes
885
+ ----------
886
+ resolved_path : pathlib.Path or None
887
+ The absolute path that was attempted, resolved relative to the
888
+ ``.gle`` file's directory (or used as-is if already absolute).
889
+ ``None`` only if ``referenced_name`` itself couldn't be
890
+ interpreted as a path at all (defensive; not expected in
891
+ practice).
892
+ exists : bool
893
+ Whether ``resolved_path`` existed on disk at resolution time.
894
+ table : DataTable or None
895
+ The loaded table on success, else ``None``.
896
+ error : str or None
897
+ Human-readable error message on failure, else ``None``.
898
+ """
899
+
900
+ resolved_path: Optional[Path]
901
+ exists: bool
902
+ table: Optional[DataTable]
903
+ error: Optional[str]
904
+
905
+
906
+ def resolve_data_reference(
907
+ gle_path: Union[str, Path], referenced_name: str
908
+ ) -> ResolvedData:
909
+ """Resolve and load a data file referenced from a ``.gle`` script.
910
+
911
+ Parameters
912
+ ----------
913
+ gle_path : str or pathlib.Path
914
+ Path to the ``.gle`` file doing the referencing. Only its parent
915
+ directory is used (the file itself need not exist).
916
+ referenced_name : str
917
+ The file name/path as written in the GLE ``data`` command, e.g.
918
+ ``"file.dat"`` or ``"subdir/file.dat"``. Interpreted relative to
919
+ ``gle_path``'s directory unless it is already absolute.
920
+
921
+ Returns
922
+ -------
923
+ ResolvedData
924
+ Never raises: missing files, unreadable files, and malformed/
925
+ empty files are all reported via ``.error`` rather than an
926
+ exception, so a parser can represent a broken ``data`` command as
927
+ structured "broken series" info instead of aborting the parse.
928
+ """
929
+ ref = Path(referenced_name)
930
+ if ref.is_absolute():
931
+ resolved_path = ref
932
+ else:
933
+ gle_dir = Path(gle_path).parent
934
+ resolved_path = gle_dir / ref
935
+
936
+ exists = resolved_path.exists()
937
+ if not exists:
938
+ return ResolvedData(
939
+ resolved_path=resolved_path,
940
+ exists=False,
941
+ table=None,
942
+ error=f"Data file not found: {resolved_path}",
943
+ )
944
+
945
+ try:
946
+ table = load_data_file(resolved_path)
947
+ except (OSError, ValueError) as exc:
948
+ return ResolvedData(
949
+ resolved_path=resolved_path,
950
+ exists=True,
951
+ table=None,
952
+ error=str(exc),
953
+ )
954
+
955
+ return ResolvedData(
956
+ resolved_path=resolved_path,
957
+ exists=True,
958
+ table=table,
959
+ error=None,
960
+ )
961
+
962
+
963
+ class ColumnExtractionError(ValueError):
964
+ """Raised by :func:`extract_columns` for out-of-bounds or non-numeric
965
+ column requests.
966
+
967
+ This is a :class:`ValueError` subclass carrying a clear message; per
968
+ the documented convention (see :func:`extract_columns`), column
969
+ extraction problems raise rather than returning a sentinel, since a
970
+ bad column index/reference is a caller (parser) bug or a malformed
971
+ GLE script that should be caught and turned into a broken-series
972
+ diagnostic by the caller, not silently propagated as ``None`` data.
973
+ """
974
+
975
+
976
+ def extract_columns(
977
+ table: DataTable,
978
+ x_col_1based: int,
979
+ y_col_1based: int,
980
+ extra_cols: Optional[List[int]] = None,
981
+ ) -> dict:
982
+ """Extract numeric columns from a :class:`DataTable` by 1-based index.
983
+
984
+ GLE data commands and column references (``d1=c1,c2``) are 1-based,
985
+ matching GLE script syntax, whereas :class:`DataTable.columns` is
986
+ 0-based Python list indexing. This function does the conversion.
987
+
988
+ Parameters
989
+ ----------
990
+ table : DataTable
991
+ The loaded table to extract from.
992
+ x_col_1based : int
993
+ 1-based index of the X column.
994
+ y_col_1based : int
995
+ 1-based index of the Y column.
996
+ extra_cols : list of int, optional
997
+ Additional 1-based column indices to extract (e.g. error-bar or
998
+ extra Y columns for multi-column ``data`` commands). Each is
999
+ keyed ``"c3"``, ``"c4"``, ... in the result, using the column's
1000
+ own 1-based index as the suffix (*not* its position in
1001
+ ``extra_cols``), so callers can match a result key back to the
1002
+ GLE column reference that produced it.
1003
+
1004
+ Returns
1005
+ -------
1006
+ dict
1007
+ Keys ``"x"``, ``"y"``, and ``"c{n}"`` for each entry in
1008
+ ``extra_cols``, values are ``numpy.ndarray`` (``dtype=float64``).
1009
+
1010
+ Raises
1011
+ ------
1012
+ ColumnExtractionError
1013
+ If any requested column index is out of bounds (``< 1`` or
1014
+ ``> table.n_cols``) or refers to a non-numeric column. This is
1015
+ the chosen error convention for this function (as opposed to
1016
+ returning ``None`` + a side-channel error): callers that need
1017
+ to represent a bad reference as a non-fatal "broken series"
1018
+ (e.g. the parser) should catch ``ColumnExtractionError`` around
1019
+ the call, rather than checking a result field on every call site.
1020
+
1021
+ Notes
1022
+ -----
1023
+ Missing values are already ``np.nan`` in numeric columns (per
1024
+ :func:`load_data_file`'s GLE-convention token handling: ``*``, ``?``,
1025
+ ``-``, ``.``, empty, ``nan``), so no further missing-value handling
1026
+ is needed here.
1027
+ """
1028
+
1029
+ def _get(col_1based: int, key: str) -> np.ndarray:
1030
+ # GLE column 0 is the synthesized point index (1, 2, 3, ...), used when
1031
+ # a data command has no x column (single-column file / NOX). Mirror
1032
+ # GLE's ``data_command`` (graph.cpp: ``column == 0`` -> ``j + 1``).
1033
+ if col_1based == 0:
1034
+ return np.arange(1, table.n_rows + 1, dtype=np.float64)
1035
+ if col_1based < 1 or col_1based > table.n_cols:
1036
+ raise ColumnExtractionError(
1037
+ f"Column {col_1based} ({key}) out of bounds: table "
1038
+ f"{table.path} has {table.n_cols} column(s) "
1039
+ f"(1-based indices 1..{table.n_cols})."
1040
+ )
1041
+ idx = col_1based - 1
1042
+ if not table.is_numeric[idx]:
1043
+ name = table.column_names[idx]
1044
+ raise ColumnExtractionError(
1045
+ f"Column {col_1based} ({key}) -- {name!r} in {table.path} -- "
1046
+ "is not numeric and cannot be used as plot data."
1047
+ )
1048
+ return table.columns[idx]
1049
+
1050
+ result = {
1051
+ "x": _get(x_col_1based, "x"),
1052
+ "y": _get(y_col_1based, "y"),
1053
+ }
1054
+ for col in extra_cols or []:
1055
+ result[f"c{col}"] = _get(col, f"c{col}")
1056
+ return result
1057
+
1058
+
1059
+ def classify_data_file(
1060
+ gle_path: Union[str, Path],
1061
+ referenced_name: str,
1062
+ import_list: Optional[List[str]] = None,
1063
+ ) -> str:
1064
+ """Classify a ``data`` reference as an ``"import"`` or a ``"reference"``.
1065
+
1066
+ - ``"import"``: the data was copied into the project by gleplot's
1067
+ own export (a sidecar ``.dat`` file next to the ``.gle`` script)
1068
+ and should round-trip as in-memory/editable series data.
1069
+ - ``"reference"``: the ``.gle`` script points at an external file in
1070
+ place (via ``line_from_file`` / ``errorbar_from_file``-style
1071
+ commands) and the data should stay a file reference rather than
1072
+ being pulled into the project.
1073
+
1074
+ Parameters
1075
+ ----------
1076
+ gle_path : str or pathlib.Path
1077
+ Path to the ``.gle`` file doing the referencing (unused since the
1078
+ filename heuristic was removed; retained for API stability).
1079
+ referenced_name : str
1080
+ The file name/path as written in the GLE ``data`` command.
1081
+ import_list : list of str, optional
1082
+ Authoritative list of file names/paths that gleplot's own writer
1083
+ recorded as "imported" (copied) data files, parsed from the
1084
+ ``! gleplot`` metadata comment block gleplot itself wrote into
1085
+ the ``.gle`` file when exporting. When this is not ``None``,
1086
+ membership in this list (matched against ``referenced_name``,
1087
+ both as given and as a bare filename) *fully decides* the
1088
+ classification. When it is ``None`` (no metadata block present --
1089
+ e.g. a hand-authored or third-party ``.gle`` script), the
1090
+ reference is ALWAYS classified ``"reference"``.
1091
+
1092
+ Returns
1093
+ -------
1094
+ str
1095
+ ``"import"`` or ``"reference"``.
1096
+
1097
+ Notes
1098
+ -----
1099
+ **Only the metadata block's import-data list can vouch a file as
1100
+ gleplot-owned.** Every ``.gle`` gleplot itself saves carries the
1101
+ ``! gleplot`` metadata block naming its import sidecars, so genuine
1102
+ gleplot-authored sidecars are always vouched via ``import_list`` and
1103
+ correctly classified ``"import"``.
1104
+
1105
+ When ``import_list is None`` there is no authoritative signal that
1106
+ gleplot owns any referenced file, so classification is conservative:
1107
+ every reference is treated as an external ``"reference"``. This is a
1108
+ deliberate safety property -- classifying a file as ``"import"``
1109
+ causes gleplot to adopt (and, on the next ``savefig_gle``, REWRITE)
1110
+ that file. A hand-authored ``.gle`` sitting next to an ordinary
1111
+ user data file whose name happens to look like a sidecar (e.g.
1112
+ ``results_2024.dat``) must NEVER have its data silently overwritten.
1113
+ Such files remain read-only references: still plottable and
1114
+ editable-in-style, but never rewritten.
1115
+
1116
+ (Historically a filename heuristic -- ``<prefix>_<digits>.dat`` in
1117
+ the same directory -- classified such files as ``"import"``; that
1118
+ produced a silent user-data-overwrite false positive and has been
1119
+ removed. The metadata block is now the sole source of truth.)
1120
+ """
1121
+ if import_list is None:
1122
+ return "reference"
1123
+ candidates = {referenced_name, Path(referenced_name).name}
1124
+ imported = {str(n) for n in import_list} | {Path(str(n)).name for n in import_list}
1125
+ return "import" if candidates & imported else "reference"
1126
+
1127
+
1128
+ # -- GLE's own reading of a data file ------------------------------------------
1129
+ #
1130
+ # The loader above is tolerant on purpose (sniffed delimiters, comment
1131
+ # headers, ``nan`` as missing). GLE's ``data`` command is not: it splits the
1132
+ # file with ``GLECSVData`` (``glearray.cpp``) under the statement's
1133
+ # ``comment``, ``ignore`` and ``numrows``, then decides the header and the x
1134
+ # column from the cells (``auto_has_header``, ``auto_all_labels_column`` in
1135
+ # ``graph.cpp``). Where the two disagree -- ``inf`` or ``nan`` is text to GLE,
1136
+ # a lone row of names is a header with no data -- only GLE's answer says
1137
+ # which columns a dataset given without ``cN`` reads.
1138
+
1139
+ #: GLE's default ``delimiters`` for a ``data`` command.
1140
+ _GLE_DELIMITERS = " ,;\t"
1141
+
1142
+
1143
+ @dataclass(frozen=True)
1144
+ class GLEDataLayout:
1145
+ """The shape GLE's ``data`` command reads from a file.
1146
+
1147
+ Attributes
1148
+ ----------
1149
+ n_cols : int
1150
+ Cells in the first row (GLE requires every row to have as many).
1151
+ labels_column : bool
1152
+ Every cell of the first column below the header is text, so GLE
1153
+ reads it as x-axis labels and x is the point index. The header is
1154
+ a first row none of whose cells is a number or a missing value.
1155
+ """
1156
+
1157
+ n_cols: int
1158
+ labels_column: bool
1159
+
1160
+
1161
+ def _gle_is_float(text: str) -> bool:
1162
+ """GLE's ``is_float`` (``Tokenizer.cpp``): a decimal number, no ``inf``.
1163
+
1164
+ An exponent ends the check, so ``1e5x`` counts, as it does in GLE.
1165
+ """
1166
+ n = len(text)
1167
+ pos = 0
1168
+ if pos < n and text[pos] in "+-":
1169
+ pos += 1
1170
+ before = after = 0
1171
+ if pos < n and text[pos] == ".":
1172
+ pos += 1
1173
+ while pos < n and text[pos].isdigit() and text[pos].isascii():
1174
+ pos += 1
1175
+ after += 1
1176
+ else:
1177
+ while pos < n and text[pos].isdigit() and text[pos].isascii():
1178
+ pos += 1
1179
+ before += 1
1180
+ if pos < n and text[pos] == ".":
1181
+ pos += 1
1182
+ while pos < n and text[pos].isdigit() and text[pos].isascii():
1183
+ pos += 1
1184
+ after += 1
1185
+ if not (before or after):
1186
+ return False
1187
+ if pos < n and text[pos] in "eE":
1188
+ pos += 1
1189
+ if pos < n and text[pos] in "+-":
1190
+ pos += 1
1191
+ return pos < n and text[pos].isdigit() and text[pos].isascii()
1192
+ return pos == n
1193
+
1194
+
1195
+ def _gle_float_or_missing(cell: str) -> bool:
1196
+ """GLE's ``isFloatMiss``: a number, or a missing value (``*?-.`` or empty)."""
1197
+ return len(cell) == 0 or (len(cell) == 1 and cell in "*?-.") or _gle_is_float(cell)
1198
+
1199
+
1200
+ def _gle_rows(
1201
+ text: str, comment: str, delimiters: str, ignore: int, numrows: int
1202
+ ) -> Iterator[List[str]]:
1203
+ """Rows of cells, as ``GLECSVData::parseBlock`` splits ``text``.
1204
+
1205
+ Each row is yielded once the next one starts, so a caller can stop
1206
+ early.
1207
+
1208
+ Raises
1209
+ ------
1210
+ ValueError
1211
+ For a string GLE leaves unterminated.
1212
+ """
1213
+ data = text
1214
+ size = len(data)
1215
+ pos = 0
1216
+ rows: List[List[str]] = []
1217
+ next_line = True
1218
+ last_delim_space = " " in delimiters or "\t" in delimiters
1219
+
1220
+ def read() -> str:
1221
+ nonlocal pos
1222
+ if pos == size:
1223
+ return ""
1224
+ pos += 1
1225
+ return data[pos - 1]
1226
+
1227
+ def go_back() -> None:
1228
+ nonlocal pos
1229
+ pos = max(pos - 1, 0)
1230
+
1231
+ def is_eol(ch: str) -> bool:
1232
+ return ch in ("\n", "\r") and ch != ""
1233
+
1234
+ def is_space(ch: str) -> bool:
1235
+ return ch in (" ", "\t") and ch != ""
1236
+
1237
+ def is_delim(ch: str) -> bool:
1238
+ return ch != "" and ch in delimiters
1239
+
1240
+ def is_comment(ch: str) -> bool:
1241
+ nonlocal pos
1242
+ start = pos
1243
+ matched = 0
1244
+ while matched < len(comment) and ch == comment[matched]:
1245
+ ch = read()
1246
+ matched += 1
1247
+ if matched == len(comment):
1248
+ go_back()
1249
+ return True
1250
+ pos = start
1251
+ return False
1252
+
1253
+ def create(cell: str) -> None:
1254
+ nonlocal next_line
1255
+ if next_line:
1256
+ rows.append([])
1257
+ next_line = False
1258
+ rows[-1].append(cell)
1259
+
1260
+ def size_ok_at_eol(cell_size: int) -> bool:
1261
+ return cell_size != 0 or not (next_line or last_delim_space)
1262
+
1263
+ def read_newline(prev: str) -> bool:
1264
+ nonlocal next_line
1265
+ next_line = True
1266
+ ch = read()
1267
+ if ch == "":
1268
+ return False
1269
+ if not (is_eol(ch) and ch != prev):
1270
+ go_back()
1271
+ return True
1272
+
1273
+ def skip_till_eol() -> bool:
1274
+ while True:
1275
+ ch = read()
1276
+ if ch == "":
1277
+ return False
1278
+ if is_eol(ch):
1279
+ return read_newline(ch)
1280
+
1281
+ def skip_spaces_and_first_delim(ch: str) -> bool:
1282
+ nonlocal last_delim_space
1283
+ while True:
1284
+ if not is_space(ch):
1285
+ if ch == "":
1286
+ return False
1287
+ if is_eol(ch):
1288
+ return read_newline(ch)
1289
+ if is_delim(ch):
1290
+ last_delim_space = False
1291
+ else:
1292
+ go_back()
1293
+ return True
1294
+ ch = read()
1295
+
1296
+ def read_quoted(quote: str) -> bool:
1297
+ cell = quote
1298
+ while True:
1299
+ ch = read()
1300
+ if ch == "" or is_eol(ch):
1301
+ raise ValueError("unterminated string")
1302
+ cell += ch
1303
+ if ch == quote:
1304
+ ch = read()
1305
+ if ch != quote:
1306
+ create(cell)
1307
+ return skip_spaces_and_first_delim(ch)
1308
+
1309
+ def read_cell() -> bool:
1310
+ nonlocal last_delim_space
1311
+ ch = read()
1312
+ while is_space(ch):
1313
+ ch = read()
1314
+ if ch in ('"', "'"):
1315
+ return read_quoted(ch)
1316
+ start = max(pos - 1, 0)
1317
+ count = cell_size = 0
1318
+ while True:
1319
+ if ch == "":
1320
+ if size_ok_at_eol(cell_size):
1321
+ create(data[start : start + cell_size])
1322
+ return False
1323
+ if is_eol(ch):
1324
+ if size_ok_at_eol(cell_size):
1325
+ create(data[start : start + cell_size])
1326
+ return read_newline(ch)
1327
+ if is_delim(ch):
1328
+ last_delim_space = is_space(ch)
1329
+ if cell_size or not is_space(ch):
1330
+ create(data[start : start + cell_size])
1331
+ return skip_spaces_and_first_delim(ch)
1332
+ if is_comment(ch):
1333
+ if size_ok_at_eol(cell_size):
1334
+ create(data[start : start + cell_size])
1335
+ return skip_till_eol()
1336
+ count += 1
1337
+ if not is_space(ch):
1338
+ cell_size = count
1339
+ ch = read()
1340
+
1341
+ more = True
1342
+ for _ in range(ignore):
1343
+ more = skip_till_eol()
1344
+ done = 0
1345
+ while more:
1346
+ more = read_cell()
1347
+ if next_line and numrows and len(rows) >= numrows:
1348
+ break
1349
+ while done < len(rows) - 1:
1350
+ yield rows[done]
1351
+ done += 1
1352
+ yield from rows[done:]
1353
+
1354
+
1355
+ def gle_data_layout(
1356
+ path: Union[str, Path],
1357
+ *,
1358
+ comment: str = "!",
1359
+ ignore: int = 0,
1360
+ numrows: int = 0,
1361
+ ) -> GLEDataLayout:
1362
+ """How GLE's ``data`` command reads ``path`` under the statement's options.
1363
+
1364
+ Parameters
1365
+ ----------
1366
+ path : str or pathlib.Path
1367
+ The data file.
1368
+ comment, ignore, numrows
1369
+ The ``data`` statement's ``comment``, ``ignore`` and ``numrows``
1370
+ (``numrows 0`` reads every row), with GLE's defaults.
1371
+
1372
+ Returns
1373
+ -------
1374
+ GLEDataLayout
1375
+
1376
+ Raises
1377
+ ------
1378
+ OSError
1379
+ If the file cannot be read.
1380
+ ValueError
1381
+ For a string GLE leaves unterminated in the rows read.
1382
+
1383
+ Notes
1384
+ -----
1385
+ Rows are read only until the answer is known: the first body row with
1386
+ a number in its first column settles it. GLE's check that every row
1387
+ has as many cells is not repeated.
1388
+ """
1389
+ # Bytes as characters: GLE splits bytes, and a UTF-8 BOM is a cell's text.
1390
+ text = Path(path).read_bytes().decode("latin-1")
1391
+ rows = _gle_rows(text, comment, _GLE_DELIMITERS, ignore, numrows)
1392
+ first = next(rows, None)
1393
+ if first is None:
1394
+ return GLEDataLayout(n_cols=0, labels_column=False)
1395
+ header = not any(_gle_float_or_missing(cell) for cell in first)
1396
+ body = rows if header else itertools.chain([first], rows)
1397
+ labels = False
1398
+ for row in body:
1399
+ if _gle_float_or_missing(row[0]):
1400
+ labels = False
1401
+ break
1402
+ labels = True
1403
+ return GLEDataLayout(n_cols=len(first), labels_column=labels)