gleplot 6.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gleplot/__init__.py +772 -0
- gleplot/_warn.py +45 -0
- gleplot/axes.py +10714 -0
- gleplot/cairo_support.py +232 -0
- gleplot/colors.py +721 -0
- gleplot/compiler.py +863 -0
- gleplot/config.py +491 -0
- gleplot/dataio.py +1403 -0
- gleplot/figure.py +5789 -0
- gleplot/guides.py +172 -0
- gleplot/markers.py +424 -0
- gleplot/mathtext.py +679 -0
- gleplot/palettes.py +243 -0
- gleplot/parser/__init__.py +83 -0
- gleplot/parser/expr.py +196 -0
- gleplot/parser/lexer.py +289 -0
- gleplot/parser/metadata.py +374 -0
- gleplot/parser/recognizer.py +12654 -0
- gleplot/parser/report.py +74 -0
- gleplot/parser/syntax.py +756 -0
- gleplot/parser/tables.py +484 -0
- gleplot/parser/units.py +196 -0
- gleplot/py.typed +0 -0
- gleplot/series.py +1609 -0
- gleplot/sources.py +623 -0
- gleplot/writer.py +7215 -0
- gleplot-6.0.0.dist-info/METADATA +138 -0
- gleplot-6.0.0.dist-info/RECORD +31 -0
- gleplot-6.0.0.dist-info/WHEEL +5 -0
- gleplot-6.0.0.dist-info/licenses/LICENSE +25 -0
- gleplot-6.0.0.dist-info/top_level.txt +1 -0
gleplot/dataio.py
ADDED
|
@@ -0,0 +1,1403 @@
|
|
|
1
|
+
"""Pure-Python delimited data file loading and resolution for gleplot.
|
|
2
|
+
|
|
3
|
+
This module has no Qt dependency so it can be imported by the core GLE
|
|
4
|
+
parser (``gleplot.parser``). It provides two layers:
|
|
5
|
+
|
|
6
|
+
1. **Loading** (:func:`load_data_file`, :class:`DataTable`): parses
|
|
7
|
+
comma/tab/semicolon/whitespace-delimited text files (``.csv``,
|
|
8
|
+
``.dat``, ``.txt``, ...) into a column-oriented :class:`DataTable`.
|
|
9
|
+
:func:`load_data_file` also accepts explicit, keyword-only overrides
|
|
10
|
+
(``delimiter``, ``header``, ``comment_chars``, ``skip_rows``) for
|
|
11
|
+
consumers that need to bypass detection for part or all of a file --
|
|
12
|
+
see its docstring. Every override defaults to "detect exactly as
|
|
13
|
+
before", so existing callers are unaffected.
|
|
14
|
+
|
|
15
|
+
2. **Resolution** (:func:`resolve_data_reference`, :func:`extract_columns`,
|
|
16
|
+
:func:`classify_data_file`): a parser-facing layer used to turn a
|
|
17
|
+
``data "file.dat" d1=c1,c2`` command referenced from a ``.gle`` script
|
|
18
|
+
into loaded numpy arrays, without ever raising for missing/unreadable/
|
|
19
|
+
malformed files -- callers get structured error information back
|
|
20
|
+
instead, so a broken data reference can be surfaced in an import report
|
|
21
|
+
rather than crashing the parse.
|
|
22
|
+
|
|
23
|
+
Format handling (loading layer)
|
|
24
|
+
--------------------------------
|
|
25
|
+
- Delimiter sniffing via :class:`csv.Sniffer` on a sample of the file,
|
|
26
|
+
falling back to arbitrary-whitespace splitting when sniffing fails or
|
|
27
|
+
the sniffed delimiter doesn't actually separate the row into multiple
|
|
28
|
+
fields.
|
|
29
|
+
- Comment lines (leading ``#`` or ``!``, ignoring leading whitespace) are
|
|
30
|
+
skipped entirely and never considered for header/data detection, EXCEPT
|
|
31
|
+
for two narrow, conservative cases (tried in this order) when the file
|
|
32
|
+
has no inline header row at all:
|
|
33
|
+
|
|
34
|
+
1. **Indexed block** -- an instrument/export-style comment block with one
|
|
35
|
+
``c <int> = <name>`` line per column (case-insensitive, whitespace
|
|
36
|
+
around ``=`` tolerated), e.g.::
|
|
37
|
+
|
|
38
|
+
! some prose header
|
|
39
|
+
! c 1 = run_id
|
|
40
|
+
! c 2 = field_strength (G)
|
|
41
|
+
! c 7 = err_rate (unit-1)
|
|
42
|
+
!
|
|
43
|
+
! run_id field_strength(G) ... (aligned prose row, ignored)
|
|
44
|
+
1 2.5 ...
|
|
45
|
+
|
|
46
|
+
The WHOLE comment block preceding the first data row is scanned (not
|
|
47
|
+
just the last line); accepted only if the discovered indices are
|
|
48
|
+
EXACTLY ``1..n_cols`` -- full coverage, no gaps/duplicates/out-of-range
|
|
49
|
+
(no partial adoption). Names may contain unicode and internal
|
|
50
|
+
whitespace verbatim (no tokenization: each line is matched whole).
|
|
51
|
+
Unambiguous, so it is tried FIRST and takes precedence over case 2
|
|
52
|
+
below. See :func:`_recover_indexed_comment_header`.
|
|
53
|
+
2. **Last-line heuristic** -- the LAST comment line immediately preceding
|
|
54
|
+
the first data row (blank lines tolerated in between) is checked as a
|
|
55
|
+
possible comment-embedded column-name line -- e.g. a GLE-style
|
|
56
|
+
fit-parameter export ending in ``! x y`` right before the numeric
|
|
57
|
+
data. Accepted as column names only if, once the comment marker is
|
|
58
|
+
stripped and the remainder tokenized with the same delimiter as the
|
|
59
|
+
data, the token count matches the data column count AND at least one
|
|
60
|
+
token is a genuine non-numeric label (same rule as header detection
|
|
61
|
+
below). See :func:`_recover_comment_header`.
|
|
62
|
+
|
|
63
|
+
On acceptance (either case) ``DataTable.column_names`` gets the recovered
|
|
64
|
+
names and ``DataTable.header_source`` is set to ``"comment"``, but
|
|
65
|
+
``DataTable.has_header`` stays ``False`` -- the line(s) are still
|
|
66
|
+
comments, so data-row indexing and any round-trip are unaffected; this
|
|
67
|
+
is display-name recovery only.
|
|
68
|
+
- Header detection: the first non-comment, non-blank row is treated as a
|
|
69
|
+
header if and only if at least one of its fields is a genuine
|
|
70
|
+
non-numeric *label* -- i.e. it fails float conversion AND is not a
|
|
71
|
+
missing-value token. A first data row consisting entirely of missing
|
|
72
|
+
tokens (e.g. ``* * *``) is therefore kept as an all-NaN DATA row, not
|
|
73
|
+
mistaken for a header.
|
|
74
|
+
- Header/column alignment: if a detected header's token count does not
|
|
75
|
+
match the data column count (e.g. a whitespace-split multi-word header
|
|
76
|
+
like ``Temperature (K) Resistance (Ohm)`` yielding 4 tokens over 2
|
|
77
|
+
data columns), the row is still skipped as a header, but its tokens are
|
|
78
|
+
not used to name columns; positional names (``col1``..``colN``) are
|
|
79
|
+
synthesized and a warning is recorded on ``DataTable.warnings``.
|
|
80
|
+
Delimited files (comma/tab/semicolon) split multi-word names correctly,
|
|
81
|
+
so their headers align and are used verbatim.
|
|
82
|
+
- Missing values: empty fields and the GLE-convention tokens ``*``,
|
|
83
|
+
``?``, ``-``, ``.`` (only when they are the *entire* field, not part
|
|
84
|
+
of a longer token) as well as ``nan``/``NaN`` (case-insensitive) become
|
|
85
|
+
``np.nan``.
|
|
86
|
+
- Ragged rows (inconsistent field counts) are right-padded with NaN up to
|
|
87
|
+
the widest row seen; a human-readable warning is recorded for each
|
|
88
|
+
padded row on ``DataTable.warnings``.
|
|
89
|
+
- Columns where every non-missing value parses as a float are numeric
|
|
90
|
+
(``dtype=float64``); otherwise the column is kept as-is with
|
|
91
|
+
``dtype=object`` and flagged not numeric via ``DataTable.is_numeric``.
|
|
92
|
+
- Encoding: ``utf-8-sig`` first (tolerates a UTF-8 BOM and plain UTF-8
|
|
93
|
+
files alike), falling back to ``latin-1`` so no file fails to load
|
|
94
|
+
outright.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
from __future__ import annotations
|
|
98
|
+
|
|
99
|
+
import csv
|
|
100
|
+
import io
|
|
101
|
+
import itertools
|
|
102
|
+
import re
|
|
103
|
+
from dataclasses import dataclass, field
|
|
104
|
+
from pathlib import Path
|
|
105
|
+
from typing import Callable, Iterator, List, Optional, Union
|
|
106
|
+
|
|
107
|
+
import numpy as np
|
|
108
|
+
|
|
109
|
+
__all__ = [
|
|
110
|
+
"DataTable",
|
|
111
|
+
"load_data_file",
|
|
112
|
+
"WHITESPACE_DELIMITER",
|
|
113
|
+
"ResolvedData",
|
|
114
|
+
"resolve_data_reference",
|
|
115
|
+
"extract_columns",
|
|
116
|
+
"ColumnExtractionError",
|
|
117
|
+
"classify_data_file",
|
|
118
|
+
]
|
|
119
|
+
|
|
120
|
+
#: Standalone tokens (case-sensitive, whole-field match) treated as missing
|
|
121
|
+
#: data, per GLE conventions plus common conventions from other tools.
|
|
122
|
+
_MISSING_TOKENS = {"*", "?", "-", "."}
|
|
123
|
+
#: Additional case-insensitive missing tokens.
|
|
124
|
+
_MISSING_TOKENS_CI = {"nan", ""}
|
|
125
|
+
|
|
126
|
+
#: Delimiters tried by the Sniffer / candidate list, in preference order.
|
|
127
|
+
_CANDIDATE_DELIMITERS = [",", "\t", ";"]
|
|
128
|
+
|
|
129
|
+
#: ``load_data_file(delimiter=...)`` override value denoting explicit
|
|
130
|
+
#: whitespace-splitting, as opposed to ``delimiter=None`` (the default),
|
|
131
|
+
#: which means "sniff/detect". Reuses the same token
|
|
132
|
+
#: :attr:`DataTable.delimiter` already uses to *report* whitespace-splitting,
|
|
133
|
+
#: so a caller can round-trip a previously-reported delimiter straight back
|
|
134
|
+
#: in as an override.
|
|
135
|
+
WHITESPACE_DELIMITER = r"\s+"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@dataclass
|
|
139
|
+
class DataTable:
|
|
140
|
+
"""A loaded delimited data file, column-oriented.
|
|
141
|
+
|
|
142
|
+
Attributes
|
|
143
|
+
----------
|
|
144
|
+
column_names : list of str
|
|
145
|
+
One name per column. Synthesized as ``col1``, ``col2``, ... when
|
|
146
|
+
the file has no header row.
|
|
147
|
+
columns : list of numpy.ndarray
|
|
148
|
+
One array per column, same order as ``column_names``. Numeric
|
|
149
|
+
columns have ``dtype=float64`` (missing values are ``np.nan``);
|
|
150
|
+
non-numeric columns have ``dtype=object`` holding the raw string
|
|
151
|
+
tokens (missing values are ``None``).
|
|
152
|
+
is_numeric : list of bool
|
|
153
|
+
Per-column flag: ``True`` if the column is a numeric float array.
|
|
154
|
+
n_rows : int
|
|
155
|
+
Number of data rows (excludes the header row and comment lines).
|
|
156
|
+
path : pathlib.Path
|
|
157
|
+
Path the table was loaded from.
|
|
158
|
+
delimiter : str
|
|
159
|
+
The delimiter used to split fields. ``r"\\s+"`` denotes
|
|
160
|
+
whitespace-splitting rather than a literal single-character
|
|
161
|
+
delimiter.
|
|
162
|
+
has_header : bool
|
|
163
|
+
Whether the first non-comment row was consumed as a header.
|
|
164
|
+
warnings : list of str
|
|
165
|
+
Human-readable warnings, e.g. about ragged rows that were padded.
|
|
166
|
+
header_source : str or None
|
|
167
|
+
Where ``column_names`` came from: ``"row"`` when an inline header
|
|
168
|
+
row was consumed (``has_header`` is ``True``), ``"comment"`` when
|
|
169
|
+
no inline header row exists but names were recovered from a
|
|
170
|
+
trailing comment line (see :func:`load_data_file`'s comment-header
|
|
171
|
+
recovery; ``has_header`` stays ``False`` in this case -- the line
|
|
172
|
+
is still a comment, data-row indexing is unaffected), or ``None``
|
|
173
|
+
when ``column_names`` are the synthesized positional
|
|
174
|
+
``col1``..``colN`` placeholders.
|
|
175
|
+
comment_lines_skipped : int
|
|
176
|
+
Number of non-blank lines recognized as comments (and therefore
|
|
177
|
+
excluded from header/data detection) -- per the built-in ``#``/``!``
|
|
178
|
+
convention, or per ``load_data_file``'s ``comment_chars`` override
|
|
179
|
+
when one was given. Counted after ``skip_rows`` leading lines (if
|
|
180
|
+
any) were dropped. Together with ``delimiter`` and ``has_header``,
|
|
181
|
+
this is enough for a caller to report "delimiter found / header
|
|
182
|
+
found / N comment lines skipped" without recomputing anything.
|
|
183
|
+
gle_missing : list of bool
|
|
184
|
+
Per-column flag: ``True`` if a field in that column is one of GLE's
|
|
185
|
+
missing-value tokens (``*``, ``?``, ``-``, ``.``), which break a
|
|
186
|
+
line where a ``nan`` (also loaded as NaN) is dropped and its
|
|
187
|
+
neighbours joined.
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
column_names: List[str]
|
|
191
|
+
columns: List[np.ndarray]
|
|
192
|
+
n_rows: int
|
|
193
|
+
path: Path
|
|
194
|
+
delimiter: str
|
|
195
|
+
has_header: bool
|
|
196
|
+
is_numeric: List[bool] = field(default_factory=list)
|
|
197
|
+
warnings: List[str] = field(default_factory=list)
|
|
198
|
+
header_source: Optional[str] = None
|
|
199
|
+
comment_lines_skipped: int = 0
|
|
200
|
+
gle_missing: List[bool] = field(default_factory=list)
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def n_cols(self) -> int:
|
|
204
|
+
"""Number of columns."""
|
|
205
|
+
return len(self.column_names)
|
|
206
|
+
|
|
207
|
+
def numeric_column_names(self) -> List[str]:
|
|
208
|
+
"""Names of columns that are numeric (selectable as plot data)."""
|
|
209
|
+
return [
|
|
210
|
+
name for name, numeric in zip(self.column_names, self.is_numeric) if numeric
|
|
211
|
+
]
|
|
212
|
+
|
|
213
|
+
def numeric_column_indices(self) -> List[int]:
|
|
214
|
+
"""0-based indices of columns that are numeric."""
|
|
215
|
+
return [i for i, numeric in enumerate(self.is_numeric) if numeric]
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _is_comment(line: str) -> bool:
|
|
219
|
+
stripped = line.lstrip()
|
|
220
|
+
return stripped.startswith("#") or stripped.startswith("!")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _read_text(path: Union[str, Path]) -> str:
|
|
224
|
+
"""Read file text, tolerating a UTF-8 BOM and falling back to latin-1."""
|
|
225
|
+
raw = Path(path).read_bytes()
|
|
226
|
+
try:
|
|
227
|
+
return raw.decode("utf-8-sig")
|
|
228
|
+
except UnicodeDecodeError:
|
|
229
|
+
return raw.decode("latin-1")
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _sniff_delimiter(sample_lines: List[str]) -> Optional[str]:
|
|
233
|
+
"""Try csv.Sniffer on a sample; return a delimiter char or None."""
|
|
234
|
+
sample = "\n".join(sample_lines)
|
|
235
|
+
if not sample.strip():
|
|
236
|
+
return None
|
|
237
|
+
try:
|
|
238
|
+
dialect = csv.Sniffer().sniff(sample, delimiters="".join(_CANDIDATE_DELIMITERS))
|
|
239
|
+
return dialect.delimiter
|
|
240
|
+
except csv.Error:
|
|
241
|
+
pass
|
|
242
|
+
|
|
243
|
+
# Sniffer failed (e.g. single column, ambiguous, or ragged rows with
|
|
244
|
+
# inconsistent field counts). Fall back to counting occurrences of
|
|
245
|
+
# each candidate delimiter across the sample lines; accept the one
|
|
246
|
+
# that appears on *most* non-empty lines (count > 0), even if the
|
|
247
|
+
# exact count varies row-to-row (ragged data) and even if a minority
|
|
248
|
+
# of rows lack it entirely (e.g. a single-field ragged row like a
|
|
249
|
+
# lone ``2`` in an otherwise comma-delimited file). A separator that
|
|
250
|
+
# is present on the majority of rows is a strong signal of a real
|
|
251
|
+
# field delimiter rather than incidental punctuation; the minority of
|
|
252
|
+
# rows that lack it are handled as ragged rows (right-padded with NaN)
|
|
253
|
+
# downstream. Prefer the delimiter present on the most rows, breaking
|
|
254
|
+
# ties by the most consistent per-row counts, then by candidate order.
|
|
255
|
+
non_empty = [ln for ln in sample_lines if ln.strip()]
|
|
256
|
+
if not non_empty:
|
|
257
|
+
return None
|
|
258
|
+
best_delim = None
|
|
259
|
+
best_key = None
|
|
260
|
+
for delim in _CANDIDATE_DELIMITERS:
|
|
261
|
+
counts = [ln.count(delim) for ln in non_empty]
|
|
262
|
+
present = [c for c in counts if c > 0]
|
|
263
|
+
# Require a strict majority of rows to contain the delimiter, so a
|
|
264
|
+
# stray comma in one row of a whitespace file doesn't hijack it.
|
|
265
|
+
if len(present) * 2 <= len(non_empty):
|
|
266
|
+
continue
|
|
267
|
+
# Consistency measured over the rows that actually have the
|
|
268
|
+
# delimiter; ragged single-field rows (count 0) don't count
|
|
269
|
+
# against it. Fewer distinct counts == more consistent.
|
|
270
|
+
variety = len(set(present))
|
|
271
|
+
# Higher presence first (negate for a min-style comparison), then
|
|
272
|
+
# lower variety. Candidate order breaks any remaining tie.
|
|
273
|
+
key = (-len(present), variety)
|
|
274
|
+
if best_key is None or key < best_key:
|
|
275
|
+
best_delim = delim
|
|
276
|
+
best_key = key
|
|
277
|
+
return best_delim
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _split_line(line: str, delimiter: Optional[str]) -> List[str]:
|
|
281
|
+
"""Split a single line into fields using the resolved delimiter.
|
|
282
|
+
|
|
283
|
+
``delimiter is None`` means whitespace-splitting.
|
|
284
|
+
"""
|
|
285
|
+
if delimiter is None:
|
|
286
|
+
return line.split()
|
|
287
|
+
# csv module handles quoted fields correctly for comma/semicolon/tab.
|
|
288
|
+
reader = csv.reader(io.StringIO(line), delimiter=delimiter)
|
|
289
|
+
try:
|
|
290
|
+
return next(reader)
|
|
291
|
+
except StopIteration:
|
|
292
|
+
return []
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _normalize_token(token: str) -> Optional[str]:
|
|
296
|
+
"""Return the token stripped, or None if it represents a missing value."""
|
|
297
|
+
stripped = token.strip()
|
|
298
|
+
if stripped in _MISSING_TOKENS:
|
|
299
|
+
return None
|
|
300
|
+
if stripped.lower() in _MISSING_TOKENS_CI:
|
|
301
|
+
return None
|
|
302
|
+
return stripped
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _try_float(token: Optional[str]) -> Optional[float]:
|
|
306
|
+
"""Convert a normalized token to float, or None if missing/not numeric."""
|
|
307
|
+
if token is None:
|
|
308
|
+
return None
|
|
309
|
+
try:
|
|
310
|
+
return float(token)
|
|
311
|
+
except ValueError:
|
|
312
|
+
return None
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _strip_comment_marker(line: str) -> str:
|
|
316
|
+
"""Strip a leading comment marker (``#`` or ``!``) and whitespace."""
|
|
317
|
+
stripped = line.lstrip()
|
|
318
|
+
if stripped.startswith("#") or stripped.startswith("!"):
|
|
319
|
+
stripped = stripped[1:]
|
|
320
|
+
return stripped.strip()
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _find_last_comment_before_first_data_row(raw_lines: List[str]) -> Optional[str]:
|
|
324
|
+
"""Return the comment line immediately preceding the first data row.
|
|
325
|
+
|
|
326
|
+
"Immediately preceding" tolerates intervening blank lines (walking
|
|
327
|
+
backward from the first content line, skipping blanks) but stops as
|
|
328
|
+
soon as a non-blank, non-comment line is encountered first -- that
|
|
329
|
+
would mean there is no comment line directly above the first data row
|
|
330
|
+
(just more data / an earlier blank-separated block), so we
|
|
331
|
+
conservatively give up rather than risk grabbing an unrelated comment
|
|
332
|
+
from earlier in the file.
|
|
333
|
+
"""
|
|
334
|
+
# Find the raw-line index of the first content line (same predicate
|
|
335
|
+
# used to build content_lines: non-blank and not a comment).
|
|
336
|
+
first_idx = None
|
|
337
|
+
for i, line in enumerate(raw_lines):
|
|
338
|
+
if line.strip() and not _is_comment(line):
|
|
339
|
+
first_idx = i
|
|
340
|
+
break
|
|
341
|
+
if first_idx is None or first_idx == 0:
|
|
342
|
+
return None
|
|
343
|
+
|
|
344
|
+
for i in range(first_idx - 1, -1, -1):
|
|
345
|
+
line = raw_lines[i]
|
|
346
|
+
if not line.strip():
|
|
347
|
+
continue
|
|
348
|
+
if _is_comment(line):
|
|
349
|
+
return line
|
|
350
|
+
# Hit a non-comment, non-blank line before any comment -- no
|
|
351
|
+
# comment line is "immediately preceding" the first data row.
|
|
352
|
+
return None
|
|
353
|
+
return None
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
#: Matches a comment-body line of the form ``c <int> = <name>`` (case-
|
|
357
|
+
#: insensitive, whitespace-tolerant around the ``=``). Group 1 is the 1-based
|
|
358
|
+
#: column index, group 2 is everything after ``=`` (stripped once by the
|
|
359
|
+
#: caller; internal whitespace in the name is preserved verbatim).
|
|
360
|
+
_INDEXED_COMMENT_HEADER_RE = re.compile(r"^c\s*(\d+)\s*=\s*(.*)$", re.IGNORECASE)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _comment_block_before_first_data_row(raw_lines: List[str]) -> List[str]:
|
|
364
|
+
"""Return every comment/blank line before the first data row, in order.
|
|
365
|
+
|
|
366
|
+
Unlike :func:`_find_last_comment_before_first_data_row` (which returns
|
|
367
|
+
only the single nearest comment line), this collects the WHOLE run of
|
|
368
|
+
comment lines preceding the first data row -- needed to scan a multi-line
|
|
369
|
+
``! c N = name`` block where the column-naming lines are not necessarily
|
|
370
|
+
the very last comment line (a trailing ``!`` separator or an aligned
|
|
371
|
+
prose row -- see ``_recover_indexed_comment_header`` -- often follows
|
|
372
|
+
them). Blank lines are included (as empty strings) so ordering/spacing
|
|
373
|
+
is preserved for callers that care, but they never match the ``c N =``
|
|
374
|
+
pattern so they are harmless filler.
|
|
375
|
+
|
|
376
|
+
Stops (returns only what was collected so far) as soon as a non-blank,
|
|
377
|
+
non-comment line is reached while scanning backward from the first data
|
|
378
|
+
row -- i.e. only an unbroken run of comments/blanks immediately above
|
|
379
|
+
the data counts as "the comment block".
|
|
380
|
+
"""
|
|
381
|
+
first_idx = None
|
|
382
|
+
for i, line in enumerate(raw_lines):
|
|
383
|
+
if line.strip() and not _is_comment(line):
|
|
384
|
+
first_idx = i
|
|
385
|
+
break
|
|
386
|
+
if first_idx is None or first_idx == 0:
|
|
387
|
+
return []
|
|
388
|
+
|
|
389
|
+
block: List[str] = []
|
|
390
|
+
for i in range(first_idx - 1, -1, -1):
|
|
391
|
+
line = raw_lines[i]
|
|
392
|
+
if not line.strip():
|
|
393
|
+
block.append(line)
|
|
394
|
+
continue
|
|
395
|
+
if _is_comment(line):
|
|
396
|
+
block.append(line)
|
|
397
|
+
continue
|
|
398
|
+
break
|
|
399
|
+
block.reverse()
|
|
400
|
+
return block
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _recover_indexed_comment_header(
|
|
404
|
+
raw_lines: List[str],
|
|
405
|
+
data_col_count: int,
|
|
406
|
+
) -> Optional[List[str]]:
|
|
407
|
+
"""Recover column names from a ``! c N = name`` comment block, if present.
|
|
408
|
+
|
|
409
|
+
Real-world shape (e.g. an instrument export with a prose preamble)::
|
|
410
|
+
|
|
411
|
+
! some prose header
|
|
412
|
+
! c 1 = run_id
|
|
413
|
+
! c 2 = field_strength (G)
|
|
414
|
+
...
|
|
415
|
+
! c 7 = err_rate (unit-1)
|
|
416
|
+
!
|
|
417
|
+
! run_id field_strength(G) ...
|
|
418
|
+
1 2.5 ...
|
|
419
|
+
|
|
420
|
+
Scans the WHOLE comment block preceding the first data row (not just the
|
|
421
|
+
last line) for lines matching, case-insensitively and whitespace-
|
|
422
|
+
tolerantly, ``c <int> = <name>`` once the comment marker is stripped.
|
|
423
|
+
Everything after ``=`` becomes the column name, stripped only at the
|
|
424
|
+
ends (internal whitespace, and non-ASCII characters such as ``mu`` or
|
|
425
|
+
superscript ``-1``, are preserved verbatim -- multi-word names are fine
|
|
426
|
+
here since each line is matched whole, not tokenized).
|
|
427
|
+
|
|
428
|
+
This is unambiguous (each column index is explicitly named), so it takes
|
|
429
|
+
PRECEDENCE over the last-comment-line positional heuristic
|
|
430
|
+
(:func:`_recover_comment_header`) -- callers should try this function
|
|
431
|
+
FIRST and only fall back to the positional heuristic if it returns
|
|
432
|
+
``None``.
|
|
433
|
+
|
|
434
|
+
Accepted only when the discovered indices are EXACTLY ``{1, ..., n_cols}``
|
|
435
|
+
-- full coverage, no gaps, no duplicates, no out-of-range indices. This
|
|
436
|
+
is deliberately strict (no partial/subset adoption): an incomplete or
|
|
437
|
+
ambiguous match is much more likely to be a coincidental line (e.g. a
|
|
438
|
+
sentence containing "c 2 = ...") than a genuine column-naming block, and
|
|
439
|
+
a wrong-but-plausible mapping silently mislabels data. Any duplicate
|
|
440
|
+
index, any index outside ``1..n_cols``, or missing coverage rejects the
|
|
441
|
+
WHOLE block (returns ``None``) rather than guessing.
|
|
442
|
+
|
|
443
|
+
A trailing aligned "prose name row" (e.g. a human-readable column
|
|
444
|
+
header whose whitespace-split token count does not match
|
|
445
|
+
``data_col_count``) commonly follows the ``c N =`` lines in this file
|
|
446
|
+
shape; it is irrelevant here since this function only reads ``c N =``
|
|
447
|
+
lines and ignores everything else in the block.
|
|
448
|
+
"""
|
|
449
|
+
block = _comment_block_before_first_data_row(raw_lines)
|
|
450
|
+
if not block:
|
|
451
|
+
return None
|
|
452
|
+
|
|
453
|
+
names_by_index: dict = {}
|
|
454
|
+
for line in block:
|
|
455
|
+
if not _is_comment(line):
|
|
456
|
+
continue
|
|
457
|
+
candidate = _strip_comment_marker(line)
|
|
458
|
+
if not candidate:
|
|
459
|
+
continue
|
|
460
|
+
m = _INDEXED_COMMENT_HEADER_RE.match(candidate.strip())
|
|
461
|
+
if not m:
|
|
462
|
+
continue
|
|
463
|
+
idx = int(m.group(1))
|
|
464
|
+
name = m.group(2).strip()
|
|
465
|
+
if not name:
|
|
466
|
+
continue
|
|
467
|
+
if idx in names_by_index:
|
|
468
|
+
# Duplicate index -- ambiguous, reject the whole block.
|
|
469
|
+
return None
|
|
470
|
+
names_by_index[idx] = name
|
|
471
|
+
|
|
472
|
+
if not names_by_index:
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
found = set(names_by_index)
|
|
476
|
+
expected = set(range(1, data_col_count + 1))
|
|
477
|
+
if found != expected:
|
|
478
|
+
# Incomplete coverage, gap, or an out-of-range index -- reject
|
|
479
|
+
# rather than guess (see docstring: no partial adoption).
|
|
480
|
+
return None
|
|
481
|
+
|
|
482
|
+
return [names_by_index[i] for i in range(1, data_col_count + 1)]
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _recover_comment_header(
|
|
486
|
+
raw_lines: List[str],
|
|
487
|
+
delimiter: Optional[str],
|
|
488
|
+
data_col_count: int,
|
|
489
|
+
) -> Optional[List[str]]:
|
|
490
|
+
"""Recover column names from a trailing comment line, if it qualifies.
|
|
491
|
+
|
|
492
|
+
Only called when the file has no inline header row. Looks at the
|
|
493
|
+
LAST comment line immediately preceding the first data row (tolerating
|
|
494
|
+
intervening blank lines), strips the comment marker, and tokenizes it
|
|
495
|
+
with the same delimiter used for the data. The candidate is accepted
|
|
496
|
+
as column names ONLY IF:
|
|
497
|
+
|
|
498
|
+
- its token count equals the data column count, AND
|
|
499
|
+
- at least one token is a genuine non-numeric label (fails float
|
|
500
|
+
conversion AND is not a missing-value token) -- the same rule
|
|
501
|
+
``load_data_file`` uses to decide a row is a real header rather
|
|
502
|
+
than data, reused here so an all-numeric comment (e.g. a stray
|
|
503
|
+
results line someone commented out) is never mistaken for names.
|
|
504
|
+
|
|
505
|
+
Returns ``None`` when no comment line qualifies, leaving
|
|
506
|
+
``column_names``/``has_header`` untouched by this feature entirely.
|
|
507
|
+
"""
|
|
508
|
+
comment_line = _find_last_comment_before_first_data_row(raw_lines)
|
|
509
|
+
if comment_line is None:
|
|
510
|
+
return None
|
|
511
|
+
|
|
512
|
+
candidate_text = _strip_comment_marker(comment_line)
|
|
513
|
+
if not candidate_text:
|
|
514
|
+
return None
|
|
515
|
+
|
|
516
|
+
tokens = _split_line(candidate_text, delimiter)
|
|
517
|
+
tokens = [t.strip() for t in tokens]
|
|
518
|
+
|
|
519
|
+
if len(tokens) != data_col_count:
|
|
520
|
+
return None
|
|
521
|
+
|
|
522
|
+
has_label = any(
|
|
523
|
+
_normalize_token(tok) is not None and _try_float(_normalize_token(tok)) is None
|
|
524
|
+
for tok in tokens
|
|
525
|
+
)
|
|
526
|
+
if not has_label:
|
|
527
|
+
return None
|
|
528
|
+
|
|
529
|
+
return tokens
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def load_data_file(
|
|
533
|
+
path: Union[str, Path],
|
|
534
|
+
max_preview_rows: Optional[int] = None,
|
|
535
|
+
*,
|
|
536
|
+
delimiter: Optional[str] = None,
|
|
537
|
+
header: Optional[bool] = None,
|
|
538
|
+
comment_chars: Optional[str] = None,
|
|
539
|
+
skip_rows: int = 0,
|
|
540
|
+
) -> DataTable:
|
|
541
|
+
"""Load a delimited text data file into a :class:`DataTable`.
|
|
542
|
+
|
|
543
|
+
Parameters
|
|
544
|
+
----------
|
|
545
|
+
path : str or pathlib.Path
|
|
546
|
+
Path to the data file (``.csv``, ``.dat``, ``.txt``, or any
|
|
547
|
+
delimited text file).
|
|
548
|
+
max_preview_rows : int, optional
|
|
549
|
+
If given, only the first ``max_preview_rows`` data rows are
|
|
550
|
+
parsed into ``columns`` (the rest of the file is ignored). This
|
|
551
|
+
is a performance aid for large files when only a preview is
|
|
552
|
+
needed; ``n_rows`` reflects the number of rows actually loaded.
|
|
553
|
+
delimiter : str, optional
|
|
554
|
+
Explicit override for field splitting, bypassing
|
|
555
|
+
:class:`csv.Sniffer`-based detection entirely. ``None`` (the
|
|
556
|
+
default) sniffs/detects the delimiter exactly as before -- byte-
|
|
557
|
+
for-byte identical to calling this function without the keyword
|
|
558
|
+
at all. Pass a single character (``","``, ``"\\t"``, ``";"``, an
|
|
559
|
+
arbitrary custom character, ...) to force that literal delimiter,
|
|
560
|
+
or :data:`WHITESPACE_DELIMITER` (``r"\\s+"`` -- the same token
|
|
561
|
+
:attr:`DataTable.delimiter` uses to *report* whitespace-splitting)
|
|
562
|
+
to force arbitrary-whitespace splitting. An explicit override is
|
|
563
|
+
honored exactly as given: unlike detection, it is not
|
|
564
|
+
second-guessed by the "does it actually split the first row into
|
|
565
|
+
more than one field" fallback that sniffing uses -- if a caller
|
|
566
|
+
forces a delimiter that doesn't apply to the file, that is
|
|
567
|
+
reflected as one wide ragged column, not silently overridden.
|
|
568
|
+
header : bool, optional
|
|
569
|
+
Explicit override for whether the first content row is a header.
|
|
570
|
+
``None`` (the default) auto-detects exactly as before: a row is a
|
|
571
|
+
header iff at least one field is a genuine non-numeric label.
|
|
572
|
+
``True``/``False`` forces that decision, bypassing the heuristic
|
|
573
|
+
entirely. A forced ``header=False`` still allows comment-header
|
|
574
|
+
recovery to supply display names from a qualifying comment line
|
|
575
|
+
(see ``comment_chars`` below); a forced ``header=True`` never
|
|
576
|
+
does -- an explicit/inline header always takes precedence, the
|
|
577
|
+
same rule that already applies to auto-detected headers.
|
|
578
|
+
comment_chars : str, optional
|
|
579
|
+
Explicit override for which leading characters mark a
|
|
580
|
+
standalone-line comment. ``None`` (the default) uses the built-in
|
|
581
|
+
convention (``#`` or ``!``, i.e. :func:`_is_comment`) unchanged.
|
|
582
|
+
A non-``None`` value **replaces** the built-in set entirely (it is
|
|
583
|
+
not additive): e.g. ``comment_chars=";"`` recognizes only
|
|
584
|
+
``;``-prefixed lines as comments, and ``#``/``!`` lines become
|
|
585
|
+
ordinary content; a caller that wants ``;`` *in addition to* the
|
|
586
|
+
defaults should pass ``comment_chars="#!;"``. ``comment_chars=""``
|
|
587
|
+
disables comment-line detection entirely (every non-blank line is
|
|
588
|
+
content). Because the comment-header recovery heuristics
|
|
589
|
+
(:func:`_recover_indexed_comment_header`,
|
|
590
|
+
:func:`_recover_comment_header`) are hardcoded to the ``#``/``!``
|
|
591
|
+
convention, they are only attempted when ``comment_chars is
|
|
592
|
+
None`` -- running them against a different marker set could
|
|
593
|
+
recover a "header" from a line that was actually treated as
|
|
594
|
+
ordinary data (or the reverse), so when ``comment_chars`` is
|
|
595
|
+
overridden, positional ``col1``..``colN`` names are used instead
|
|
596
|
+
whenever there is no inline header.
|
|
597
|
+
skip_rows : int, optional
|
|
598
|
+
Number of leading raw lines to drop before any other processing
|
|
599
|
+
-- comment filtering, delimiter sniffing, and header detection all
|
|
600
|
+
run only on what remains. Default ``0`` (nothing skipped, and
|
|
601
|
+
thus no behavior change). Applied first, so it composes with
|
|
602
|
+
every other parameter.
|
|
603
|
+
|
|
604
|
+
Returns
|
|
605
|
+
-------
|
|
606
|
+
DataTable
|
|
607
|
+
|
|
608
|
+
Raises
|
|
609
|
+
------
|
|
610
|
+
FileNotFoundError
|
|
611
|
+
If ``path`` does not exist.
|
|
612
|
+
ValueError
|
|
613
|
+
If the file contains no data rows at all (after ``skip_rows`` and
|
|
614
|
+
comment/blank filtering).
|
|
615
|
+
|
|
616
|
+
Notes
|
|
617
|
+
-----
|
|
618
|
+
With every keyword-only parameter left at its default, behavior is
|
|
619
|
+
byte-for-byte identical to calling this function with just ``path``
|
|
620
|
+
(and, optionally, ``max_preview_rows``) as before: the detection
|
|
621
|
+
heuristics, missing-value conventions, and numeric-column typing this
|
|
622
|
+
module has always used are all unchanged for existing callers.
|
|
623
|
+
"""
|
|
624
|
+
path = Path(path)
|
|
625
|
+
if not path.exists():
|
|
626
|
+
raise FileNotFoundError(f"Data file not found: {path}")
|
|
627
|
+
|
|
628
|
+
text = _read_text(path)
|
|
629
|
+
raw_lines = text.splitlines()
|
|
630
|
+
if skip_rows:
|
|
631
|
+
raw_lines = raw_lines[skip_rows:]
|
|
632
|
+
|
|
633
|
+
is_comment_line: Callable[[str], bool]
|
|
634
|
+
if comment_chars is None:
|
|
635
|
+
is_comment_line = _is_comment
|
|
636
|
+
else:
|
|
637
|
+
_comment_prefixes = tuple(comment_chars)
|
|
638
|
+
|
|
639
|
+
def is_comment_line(line: str) -> bool:
|
|
640
|
+
return line.lstrip().startswith(_comment_prefixes)
|
|
641
|
+
|
|
642
|
+
# Filter out comment and blank lines up front; header/data detection
|
|
643
|
+
# and delimiter sniffing both operate on this filtered list only.
|
|
644
|
+
content_lines = [ln for ln in raw_lines if ln.strip() and not is_comment_line(ln)]
|
|
645
|
+
comment_lines_skipped = sum(
|
|
646
|
+
1 for ln in raw_lines if ln.strip() and is_comment_line(ln)
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
if not content_lines:
|
|
650
|
+
raise ValueError(f"No data rows found in {path}")
|
|
651
|
+
|
|
652
|
+
sample_lines = content_lines[:20]
|
|
653
|
+
if delimiter is None:
|
|
654
|
+
resolved_delimiter = _sniff_delimiter(sample_lines)
|
|
655
|
+
|
|
656
|
+
# Validate the sniffed delimiter actually splits the first line into
|
|
657
|
+
# more than one field; otherwise fall back to whitespace splitting.
|
|
658
|
+
if resolved_delimiter is not None:
|
|
659
|
+
first_fields = _split_line(content_lines[0], resolved_delimiter)
|
|
660
|
+
if len(first_fields) <= 1:
|
|
661
|
+
resolved_delimiter = None
|
|
662
|
+
elif delimiter == WHITESPACE_DELIMITER:
|
|
663
|
+
resolved_delimiter = None
|
|
664
|
+
else:
|
|
665
|
+
# Explicit override: honored as given (see docstring), not
|
|
666
|
+
# second-guessed by the single-field fallback above, which only
|
|
667
|
+
# applies to sniffed delimiters.
|
|
668
|
+
resolved_delimiter = delimiter
|
|
669
|
+
|
|
670
|
+
split_rows = [_split_line(line, resolved_delimiter) for line in content_lines]
|
|
671
|
+
|
|
672
|
+
# Header detection: the first row is a header iff at least one field is a
|
|
673
|
+
# genuine non-numeric label -- i.e. it fails float conversion AND is not a
|
|
674
|
+
# missing-value token. A missing token ('*', '?', '-', '.', '', 'nan')
|
|
675
|
+
# also "fails float conversion", but a first data row consisting entirely
|
|
676
|
+
# of missing values (e.g. '* * *') is real DATA (all-NaN), not a header;
|
|
677
|
+
# only a field that is a real word/label marks the row as a header.
|
|
678
|
+
# ``header`` overrides this heuristic entirely when given.
|
|
679
|
+
first_row = split_rows[0]
|
|
680
|
+
if header is None:
|
|
681
|
+
has_header = any(
|
|
682
|
+
_normalize_token(tok) is not None
|
|
683
|
+
and _try_float(_normalize_token(tok)) is None
|
|
684
|
+
for tok in first_row
|
|
685
|
+
)
|
|
686
|
+
else:
|
|
687
|
+
has_header = header
|
|
688
|
+
|
|
689
|
+
if has_header:
|
|
690
|
+
header_fields = [f.strip() for f in first_row]
|
|
691
|
+
data_rows = split_rows[1:]
|
|
692
|
+
else:
|
|
693
|
+
header_fields = None
|
|
694
|
+
data_rows = split_rows
|
|
695
|
+
|
|
696
|
+
if max_preview_rows is not None:
|
|
697
|
+
data_rows = data_rows[:max_preview_rows]
|
|
698
|
+
|
|
699
|
+
if not data_rows:
|
|
700
|
+
raise ValueError(f"No data rows found in {path}")
|
|
701
|
+
|
|
702
|
+
warnings: List[str] = []
|
|
703
|
+
|
|
704
|
+
# Column count is determined by the DATA rows, not the header. A header
|
|
705
|
+
# whose token count differs from the data column count is a *misalignment*
|
|
706
|
+
# (typically a whitespace-split multi-word header, e.g. header
|
|
707
|
+
# 'Temperature (K) Resistance (Ohm)' = 4 tokens over 2 data columns): we
|
|
708
|
+
# still skip the row as a header (it is not data), but we cannot trust its
|
|
709
|
+
# tokens to name columns 1:1, so we synthesize positional names and warn.
|
|
710
|
+
data_col_count = max(len(row) for row in data_rows)
|
|
711
|
+
header_mismatch = header_fields is not None and len(header_fields) != data_col_count
|
|
712
|
+
if header_mismatch:
|
|
713
|
+
warnings.append(
|
|
714
|
+
f"Header row has {len(header_fields or [])} field(s) but the data has "
|
|
715
|
+
f"{data_col_count} column(s); the header does not align with the "
|
|
716
|
+
"columns (e.g. multi-word names split on whitespace). Using "
|
|
717
|
+
"positional column names (col1..colN) instead."
|
|
718
|
+
)
|
|
719
|
+
max_cols = data_col_count
|
|
720
|
+
|
|
721
|
+
padded_rows: List[List[str]] = []
|
|
722
|
+
for row_idx, row in enumerate(data_rows):
|
|
723
|
+
if len(row) < max_cols:
|
|
724
|
+
pad_count = max_cols - len(row)
|
|
725
|
+
warnings.append(
|
|
726
|
+
f"Row {row_idx + 1} has {len(row)} field(s), expected "
|
|
727
|
+
f"{max_cols}; padded {pad_count} missing value(s) with NaN."
|
|
728
|
+
)
|
|
729
|
+
row = row + [""] * pad_count
|
|
730
|
+
elif len(row) > max_cols:
|
|
731
|
+
# Shouldn't normally happen since max_cols is the max data-row
|
|
732
|
+
# width, but guard defensively.
|
|
733
|
+
warnings.append(
|
|
734
|
+
f"Row {row_idx + 1} has {len(row)} field(s), expected "
|
|
735
|
+
f"{max_cols}; extra field(s) truncated."
|
|
736
|
+
)
|
|
737
|
+
row = row[:max_cols]
|
|
738
|
+
padded_rows.append(row)
|
|
739
|
+
|
|
740
|
+
header_source: Optional[str] = None
|
|
741
|
+
if header_fields is None or header_mismatch:
|
|
742
|
+
# No header, or a header that doesn't align 1:1 with the columns.
|
|
743
|
+
column_names = [f"col{i + 1}" for i in range(max_cols)]
|
|
744
|
+
if header_mismatch:
|
|
745
|
+
# The row WAS consumed as a header (has_header is True) even
|
|
746
|
+
# though its tokens couldn't be trusted to name columns 1:1;
|
|
747
|
+
# header_source still reflects that an inline row was found.
|
|
748
|
+
header_source = "row"
|
|
749
|
+
|
|
750
|
+
# Comment-header recovery: real-world .dat files (e.g. GLE-style
|
|
751
|
+
# fit-parameter exports) sometimes carry column names in a COMMENT
|
|
752
|
+
# line rather than an inline header row -- e.g.
|
|
753
|
+
# ! Fit parameter data for GLE export
|
|
754
|
+
# ! Global fitting parameters:
|
|
755
|
+
# ! A_1 (%) = 11.8654 +/- 0.0543966
|
|
756
|
+
# ! x y
|
|
757
|
+
# 1.0 2.0
|
|
758
|
+
# Only attempted when there is no inline header at all (a header
|
|
759
|
+
# row, even a mismatched one, always wins -- see the recognizer's
|
|
760
|
+
# `_recovered_column_names`, which only ever reads `column_names`
|
|
761
|
+
# when `has_header` is True, so this path can never be mistaken
|
|
762
|
+
# for a real header there). Deliberately conservative: most
|
|
763
|
+
# comment lines are NOT headers, so only the LAST comment line
|
|
764
|
+
# immediately before the first data row is even considered, and
|
|
765
|
+
# only accepted if its token count matches the data columns AND
|
|
766
|
+
# it contains a genuine non-numeric label (same rule as normal
|
|
767
|
+
# header detection above) -- see _recover_comment_header.
|
|
768
|
+
#
|
|
769
|
+
# `not header_mismatch` here means genuinely `has_header is False`
|
|
770
|
+
# (not just "mismatched"): we are inside the
|
|
771
|
+
# `header_fields is None or header_mismatch` branch, so if
|
|
772
|
+
# `header_mismatch` is False then `header_fields is None` must be
|
|
773
|
+
# the reason we're here -- i.e. there was no inline header row at
|
|
774
|
+
# all. A mismatched inline header (has_header True, tokens
|
|
775
|
+
# misaligned) is still a real header row and must NOT be
|
|
776
|
+
# second-guessed by a comment line.
|
|
777
|
+
#
|
|
778
|
+
# Also gated on `comment_chars is None`: the recovery heuristics
|
|
779
|
+
# below are hardcoded to the built-in '#'/'!' convention (see
|
|
780
|
+
# docstring), so they only run when that convention is actually
|
|
781
|
+
# what was used to filter comments out above -- otherwise a line
|
|
782
|
+
# treated as data by the (overridden) filter could be mistaken for
|
|
783
|
+
# a recoverable comment header, or vice versa.
|
|
784
|
+
if not header_mismatch and comment_chars is None:
|
|
785
|
+
# Indexed '! c N = name' block (see _recover_indexed_comment_header)
|
|
786
|
+
# is unambiguous -- every column is explicitly named by number --
|
|
787
|
+
# so it takes PRECEDENCE over the positional last-comment-line
|
|
788
|
+
# heuristic below. Only fall back to the positional heuristic
|
|
789
|
+
# when no indexed block is found (or it doesn't achieve full
|
|
790
|
+
# 1..n_cols coverage).
|
|
791
|
+
recovered = _recover_indexed_comment_header(raw_lines, max_cols)
|
|
792
|
+
if recovered is not None:
|
|
793
|
+
column_names = recovered
|
|
794
|
+
header_source = "comment"
|
|
795
|
+
else:
|
|
796
|
+
recovered = _recover_comment_header(
|
|
797
|
+
raw_lines, resolved_delimiter, max_cols
|
|
798
|
+
)
|
|
799
|
+
if recovered is not None:
|
|
800
|
+
column_names = recovered
|
|
801
|
+
header_source = "comment"
|
|
802
|
+
else:
|
|
803
|
+
column_names = list(header_fields)
|
|
804
|
+
# Pad header names if the header row itself was short.
|
|
805
|
+
while len(column_names) < max_cols:
|
|
806
|
+
column_names.append(f"col{len(column_names) + 1}")
|
|
807
|
+
# Fill blank header names with a positional placeholder.
|
|
808
|
+
column_names = [
|
|
809
|
+
name if name else f"col{i + 1}" for i, name in enumerate(column_names)
|
|
810
|
+
]
|
|
811
|
+
header_source = "row"
|
|
812
|
+
|
|
813
|
+
# Build columns.
|
|
814
|
+
columns: List[np.ndarray] = []
|
|
815
|
+
is_numeric: List[bool] = []
|
|
816
|
+
gle_missing: List[bool] = []
|
|
817
|
+
n_rows = len(padded_rows)
|
|
818
|
+
|
|
819
|
+
for col_idx in range(max_cols):
|
|
820
|
+
raw_values = [row[col_idx] for row in padded_rows]
|
|
821
|
+
gle_missing.append(any(v.strip() in _MISSING_TOKENS for v in raw_values))
|
|
822
|
+
normalized = [_normalize_token(v) for v in raw_values]
|
|
823
|
+
floats = [_try_float(v) for v in normalized]
|
|
824
|
+
|
|
825
|
+
# Numeric iff every non-missing token parsed as a float. A
|
|
826
|
+
# column that is entirely missing is treated as numeric (all NaN)
|
|
827
|
+
# since there's no string content to preserve.
|
|
828
|
+
column_is_numeric = all(
|
|
829
|
+
f is not None or n is None for f, n in zip(floats, normalized)
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
if column_is_numeric:
|
|
833
|
+
arr = np.array(
|
|
834
|
+
[np.nan if f is None else f for f in floats], dtype=np.float64
|
|
835
|
+
)
|
|
836
|
+
columns.append(arr)
|
|
837
|
+
is_numeric.append(True)
|
|
838
|
+
else:
|
|
839
|
+
obj_values = [n for n in normalized] # None for missing, str otherwise
|
|
840
|
+
arr = np.array(obj_values, dtype=object)
|
|
841
|
+
columns.append(arr)
|
|
842
|
+
is_numeric.append(False)
|
|
843
|
+
|
|
844
|
+
return DataTable(
|
|
845
|
+
column_names=column_names,
|
|
846
|
+
columns=columns,
|
|
847
|
+
n_rows=n_rows,
|
|
848
|
+
path=path,
|
|
849
|
+
delimiter=resolved_delimiter if resolved_delimiter is not None else r"\s+",
|
|
850
|
+
has_header=has_header,
|
|
851
|
+
is_numeric=is_numeric,
|
|
852
|
+
warnings=warnings,
|
|
853
|
+
header_source=header_source,
|
|
854
|
+
comment_lines_skipped=comment_lines_skipped,
|
|
855
|
+
gle_missing=gle_missing,
|
|
856
|
+
)
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
# ---------------------------------------------------------------------------
|
|
860
|
+
# Parser-facing data resolution layer
|
|
861
|
+
# ---------------------------------------------------------------------------
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
@dataclass
|
|
865
|
+
class ResolvedData:
|
|
866
|
+
"""Result of resolving a ``data "..."`` reference from a ``.gle`` script.
|
|
867
|
+
|
|
868
|
+
Exactly one of the following holds:
|
|
869
|
+
|
|
870
|
+
- Success: ``exists`` is ``True``, ``resolved_path`` is set, ``table``
|
|
871
|
+
is a loaded :class:`DataTable`, ``error`` is ``None``.
|
|
872
|
+
- Failure (missing file, unreadable, malformed/empty): ``table`` is
|
|
873
|
+
``None`` and ``error`` holds a human-readable message. ``exists``
|
|
874
|
+
reflects whether a filesystem entry was found at all (``False`` for
|
|
875
|
+
"file not found", ``True`` for "found but failed to parse" such as
|
|
876
|
+
an unreadable/empty file), and ``resolved_path`` is still populated
|
|
877
|
+
with the path that was attempted (useful for diagnostics / "broken
|
|
878
|
+
series" UI even when nothing could be loaded).
|
|
879
|
+
|
|
880
|
+
This type never results from a raised exception -- construction
|
|
881
|
+
always goes through :func:`resolve_data_reference`, which catches
|
|
882
|
+
everything :func:`load_data_file` can raise.
|
|
883
|
+
|
|
884
|
+
Attributes
|
|
885
|
+
----------
|
|
886
|
+
resolved_path : pathlib.Path or None
|
|
887
|
+
The absolute path that was attempted, resolved relative to the
|
|
888
|
+
``.gle`` file's directory (or used as-is if already absolute).
|
|
889
|
+
``None`` only if ``referenced_name`` itself couldn't be
|
|
890
|
+
interpreted as a path at all (defensive; not expected in
|
|
891
|
+
practice).
|
|
892
|
+
exists : bool
|
|
893
|
+
Whether ``resolved_path`` existed on disk at resolution time.
|
|
894
|
+
table : DataTable or None
|
|
895
|
+
The loaded table on success, else ``None``.
|
|
896
|
+
error : str or None
|
|
897
|
+
Human-readable error message on failure, else ``None``.
|
|
898
|
+
"""
|
|
899
|
+
|
|
900
|
+
resolved_path: Optional[Path]
|
|
901
|
+
exists: bool
|
|
902
|
+
table: Optional[DataTable]
|
|
903
|
+
error: Optional[str]
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def resolve_data_reference(
|
|
907
|
+
gle_path: Union[str, Path], referenced_name: str
|
|
908
|
+
) -> ResolvedData:
|
|
909
|
+
"""Resolve and load a data file referenced from a ``.gle`` script.
|
|
910
|
+
|
|
911
|
+
Parameters
|
|
912
|
+
----------
|
|
913
|
+
gle_path : str or pathlib.Path
|
|
914
|
+
Path to the ``.gle`` file doing the referencing. Only its parent
|
|
915
|
+
directory is used (the file itself need not exist).
|
|
916
|
+
referenced_name : str
|
|
917
|
+
The file name/path as written in the GLE ``data`` command, e.g.
|
|
918
|
+
``"file.dat"`` or ``"subdir/file.dat"``. Interpreted relative to
|
|
919
|
+
``gle_path``'s directory unless it is already absolute.
|
|
920
|
+
|
|
921
|
+
Returns
|
|
922
|
+
-------
|
|
923
|
+
ResolvedData
|
|
924
|
+
Never raises: missing files, unreadable files, and malformed/
|
|
925
|
+
empty files are all reported via ``.error`` rather than an
|
|
926
|
+
exception, so a parser can represent a broken ``data`` command as
|
|
927
|
+
structured "broken series" info instead of aborting the parse.
|
|
928
|
+
"""
|
|
929
|
+
ref = Path(referenced_name)
|
|
930
|
+
if ref.is_absolute():
|
|
931
|
+
resolved_path = ref
|
|
932
|
+
else:
|
|
933
|
+
gle_dir = Path(gle_path).parent
|
|
934
|
+
resolved_path = gle_dir / ref
|
|
935
|
+
|
|
936
|
+
exists = resolved_path.exists()
|
|
937
|
+
if not exists:
|
|
938
|
+
return ResolvedData(
|
|
939
|
+
resolved_path=resolved_path,
|
|
940
|
+
exists=False,
|
|
941
|
+
table=None,
|
|
942
|
+
error=f"Data file not found: {resolved_path}",
|
|
943
|
+
)
|
|
944
|
+
|
|
945
|
+
try:
|
|
946
|
+
table = load_data_file(resolved_path)
|
|
947
|
+
except (OSError, ValueError) as exc:
|
|
948
|
+
return ResolvedData(
|
|
949
|
+
resolved_path=resolved_path,
|
|
950
|
+
exists=True,
|
|
951
|
+
table=None,
|
|
952
|
+
error=str(exc),
|
|
953
|
+
)
|
|
954
|
+
|
|
955
|
+
return ResolvedData(
|
|
956
|
+
resolved_path=resolved_path,
|
|
957
|
+
exists=True,
|
|
958
|
+
table=table,
|
|
959
|
+
error=None,
|
|
960
|
+
)
|
|
961
|
+
|
|
962
|
+
|
|
963
|
+
class ColumnExtractionError(ValueError):
|
|
964
|
+
"""Raised by :func:`extract_columns` for out-of-bounds or non-numeric
|
|
965
|
+
column requests.
|
|
966
|
+
|
|
967
|
+
This is a :class:`ValueError` subclass carrying a clear message; per
|
|
968
|
+
the documented convention (see :func:`extract_columns`), column
|
|
969
|
+
extraction problems raise rather than returning a sentinel, since a
|
|
970
|
+
bad column index/reference is a caller (parser) bug or a malformed
|
|
971
|
+
GLE script that should be caught and turned into a broken-series
|
|
972
|
+
diagnostic by the caller, not silently propagated as ``None`` data.
|
|
973
|
+
"""
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def extract_columns(
|
|
977
|
+
table: DataTable,
|
|
978
|
+
x_col_1based: int,
|
|
979
|
+
y_col_1based: int,
|
|
980
|
+
extra_cols: Optional[List[int]] = None,
|
|
981
|
+
) -> dict:
|
|
982
|
+
"""Extract numeric columns from a :class:`DataTable` by 1-based index.
|
|
983
|
+
|
|
984
|
+
GLE data commands and column references (``d1=c1,c2``) are 1-based,
|
|
985
|
+
matching GLE script syntax, whereas :class:`DataTable.columns` is
|
|
986
|
+
0-based Python list indexing. This function does the conversion.
|
|
987
|
+
|
|
988
|
+
Parameters
|
|
989
|
+
----------
|
|
990
|
+
table : DataTable
|
|
991
|
+
The loaded table to extract from.
|
|
992
|
+
x_col_1based : int
|
|
993
|
+
1-based index of the X column.
|
|
994
|
+
y_col_1based : int
|
|
995
|
+
1-based index of the Y column.
|
|
996
|
+
extra_cols : list of int, optional
|
|
997
|
+
Additional 1-based column indices to extract (e.g. error-bar or
|
|
998
|
+
extra Y columns for multi-column ``data`` commands). Each is
|
|
999
|
+
keyed ``"c3"``, ``"c4"``, ... in the result, using the column's
|
|
1000
|
+
own 1-based index as the suffix (*not* its position in
|
|
1001
|
+
``extra_cols``), so callers can match a result key back to the
|
|
1002
|
+
GLE column reference that produced it.
|
|
1003
|
+
|
|
1004
|
+
Returns
|
|
1005
|
+
-------
|
|
1006
|
+
dict
|
|
1007
|
+
Keys ``"x"``, ``"y"``, and ``"c{n}"`` for each entry in
|
|
1008
|
+
``extra_cols``, values are ``numpy.ndarray`` (``dtype=float64``).
|
|
1009
|
+
|
|
1010
|
+
Raises
|
|
1011
|
+
------
|
|
1012
|
+
ColumnExtractionError
|
|
1013
|
+
If any requested column index is out of bounds (``< 1`` or
|
|
1014
|
+
``> table.n_cols``) or refers to a non-numeric column. This is
|
|
1015
|
+
the chosen error convention for this function (as opposed to
|
|
1016
|
+
returning ``None`` + a side-channel error): callers that need
|
|
1017
|
+
to represent a bad reference as a non-fatal "broken series"
|
|
1018
|
+
(e.g. the parser) should catch ``ColumnExtractionError`` around
|
|
1019
|
+
the call, rather than checking a result field on every call site.
|
|
1020
|
+
|
|
1021
|
+
Notes
|
|
1022
|
+
-----
|
|
1023
|
+
Missing values are already ``np.nan`` in numeric columns (per
|
|
1024
|
+
:func:`load_data_file`'s GLE-convention token handling: ``*``, ``?``,
|
|
1025
|
+
``-``, ``.``, empty, ``nan``), so no further missing-value handling
|
|
1026
|
+
is needed here.
|
|
1027
|
+
"""
|
|
1028
|
+
|
|
1029
|
+
def _get(col_1based: int, key: str) -> np.ndarray:
|
|
1030
|
+
# GLE column 0 is the synthesized point index (1, 2, 3, ...), used when
|
|
1031
|
+
# a data command has no x column (single-column file / NOX). Mirror
|
|
1032
|
+
# GLE's ``data_command`` (graph.cpp: ``column == 0`` -> ``j + 1``).
|
|
1033
|
+
if col_1based == 0:
|
|
1034
|
+
return np.arange(1, table.n_rows + 1, dtype=np.float64)
|
|
1035
|
+
if col_1based < 1 or col_1based > table.n_cols:
|
|
1036
|
+
raise ColumnExtractionError(
|
|
1037
|
+
f"Column {col_1based} ({key}) out of bounds: table "
|
|
1038
|
+
f"{table.path} has {table.n_cols} column(s) "
|
|
1039
|
+
f"(1-based indices 1..{table.n_cols})."
|
|
1040
|
+
)
|
|
1041
|
+
idx = col_1based - 1
|
|
1042
|
+
if not table.is_numeric[idx]:
|
|
1043
|
+
name = table.column_names[idx]
|
|
1044
|
+
raise ColumnExtractionError(
|
|
1045
|
+
f"Column {col_1based} ({key}) -- {name!r} in {table.path} -- "
|
|
1046
|
+
"is not numeric and cannot be used as plot data."
|
|
1047
|
+
)
|
|
1048
|
+
return table.columns[idx]
|
|
1049
|
+
|
|
1050
|
+
result = {
|
|
1051
|
+
"x": _get(x_col_1based, "x"),
|
|
1052
|
+
"y": _get(y_col_1based, "y"),
|
|
1053
|
+
}
|
|
1054
|
+
for col in extra_cols or []:
|
|
1055
|
+
result[f"c{col}"] = _get(col, f"c{col}")
|
|
1056
|
+
return result
|
|
1057
|
+
|
|
1058
|
+
|
|
1059
|
+
def classify_data_file(
|
|
1060
|
+
gle_path: Union[str, Path],
|
|
1061
|
+
referenced_name: str,
|
|
1062
|
+
import_list: Optional[List[str]] = None,
|
|
1063
|
+
) -> str:
|
|
1064
|
+
"""Classify a ``data`` reference as an ``"import"`` or a ``"reference"``.
|
|
1065
|
+
|
|
1066
|
+
- ``"import"``: the data was copied into the project by gleplot's
|
|
1067
|
+
own export (a sidecar ``.dat`` file next to the ``.gle`` script)
|
|
1068
|
+
and should round-trip as in-memory/editable series data.
|
|
1069
|
+
- ``"reference"``: the ``.gle`` script points at an external file in
|
|
1070
|
+
place (via ``line_from_file`` / ``errorbar_from_file``-style
|
|
1071
|
+
commands) and the data should stay a file reference rather than
|
|
1072
|
+
being pulled into the project.
|
|
1073
|
+
|
|
1074
|
+
Parameters
|
|
1075
|
+
----------
|
|
1076
|
+
gle_path : str or pathlib.Path
|
|
1077
|
+
Path to the ``.gle`` file doing the referencing (unused since the
|
|
1078
|
+
filename heuristic was removed; retained for API stability).
|
|
1079
|
+
referenced_name : str
|
|
1080
|
+
The file name/path as written in the GLE ``data`` command.
|
|
1081
|
+
import_list : list of str, optional
|
|
1082
|
+
Authoritative list of file names/paths that gleplot's own writer
|
|
1083
|
+
recorded as "imported" (copied) data files, parsed from the
|
|
1084
|
+
``! gleplot`` metadata comment block gleplot itself wrote into
|
|
1085
|
+
the ``.gle`` file when exporting. When this is not ``None``,
|
|
1086
|
+
membership in this list (matched against ``referenced_name``,
|
|
1087
|
+
both as given and as a bare filename) *fully decides* the
|
|
1088
|
+
classification. When it is ``None`` (no metadata block present --
|
|
1089
|
+
e.g. a hand-authored or third-party ``.gle`` script), the
|
|
1090
|
+
reference is ALWAYS classified ``"reference"``.
|
|
1091
|
+
|
|
1092
|
+
Returns
|
|
1093
|
+
-------
|
|
1094
|
+
str
|
|
1095
|
+
``"import"`` or ``"reference"``.
|
|
1096
|
+
|
|
1097
|
+
Notes
|
|
1098
|
+
-----
|
|
1099
|
+
**Only the metadata block's import-data list can vouch a file as
|
|
1100
|
+
gleplot-owned.** Every ``.gle`` gleplot itself saves carries the
|
|
1101
|
+
``! gleplot`` metadata block naming its import sidecars, so genuine
|
|
1102
|
+
gleplot-authored sidecars are always vouched via ``import_list`` and
|
|
1103
|
+
correctly classified ``"import"``.
|
|
1104
|
+
|
|
1105
|
+
When ``import_list is None`` there is no authoritative signal that
|
|
1106
|
+
gleplot owns any referenced file, so classification is conservative:
|
|
1107
|
+
every reference is treated as an external ``"reference"``. This is a
|
|
1108
|
+
deliberate safety property -- classifying a file as ``"import"``
|
|
1109
|
+
causes gleplot to adopt (and, on the next ``savefig_gle``, REWRITE)
|
|
1110
|
+
that file. A hand-authored ``.gle`` sitting next to an ordinary
|
|
1111
|
+
user data file whose name happens to look like a sidecar (e.g.
|
|
1112
|
+
``results_2024.dat``) must NEVER have its data silently overwritten.
|
|
1113
|
+
Such files remain read-only references: still plottable and
|
|
1114
|
+
editable-in-style, but never rewritten.
|
|
1115
|
+
|
|
1116
|
+
(Historically a filename heuristic -- ``<prefix>_<digits>.dat`` in
|
|
1117
|
+
the same directory -- classified such files as ``"import"``; that
|
|
1118
|
+
produced a silent user-data-overwrite false positive and has been
|
|
1119
|
+
removed. The metadata block is now the sole source of truth.)
|
|
1120
|
+
"""
|
|
1121
|
+
if import_list is None:
|
|
1122
|
+
return "reference"
|
|
1123
|
+
candidates = {referenced_name, Path(referenced_name).name}
|
|
1124
|
+
imported = {str(n) for n in import_list} | {Path(str(n)).name for n in import_list}
|
|
1125
|
+
return "import" if candidates & imported else "reference"
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
# -- GLE's own reading of a data file ------------------------------------------
|
|
1129
|
+
#
|
|
1130
|
+
# The loader above is tolerant on purpose (sniffed delimiters, comment
|
|
1131
|
+
# headers, ``nan`` as missing). GLE's ``data`` command is not: it splits the
|
|
1132
|
+
# file with ``GLECSVData`` (``glearray.cpp``) under the statement's
|
|
1133
|
+
# ``comment``, ``ignore`` and ``numrows``, then decides the header and the x
|
|
1134
|
+
# column from the cells (``auto_has_header``, ``auto_all_labels_column`` in
|
|
1135
|
+
# ``graph.cpp``). Where the two disagree -- ``inf`` or ``nan`` is text to GLE,
|
|
1136
|
+
# a lone row of names is a header with no data -- only GLE's answer says
|
|
1137
|
+
# which columns a dataset given without ``cN`` reads.
|
|
1138
|
+
|
|
1139
|
+
#: GLE's default ``delimiters`` for a ``data`` command.
|
|
1140
|
+
_GLE_DELIMITERS = " ,;\t"
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
@dataclass(frozen=True)
|
|
1144
|
+
class GLEDataLayout:
|
|
1145
|
+
"""The shape GLE's ``data`` command reads from a file.
|
|
1146
|
+
|
|
1147
|
+
Attributes
|
|
1148
|
+
----------
|
|
1149
|
+
n_cols : int
|
|
1150
|
+
Cells in the first row (GLE requires every row to have as many).
|
|
1151
|
+
labels_column : bool
|
|
1152
|
+
Every cell of the first column below the header is text, so GLE
|
|
1153
|
+
reads it as x-axis labels and x is the point index. The header is
|
|
1154
|
+
a first row none of whose cells is a number or a missing value.
|
|
1155
|
+
"""
|
|
1156
|
+
|
|
1157
|
+
n_cols: int
|
|
1158
|
+
labels_column: bool
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
def _gle_is_float(text: str) -> bool:
|
|
1162
|
+
"""GLE's ``is_float`` (``Tokenizer.cpp``): a decimal number, no ``inf``.
|
|
1163
|
+
|
|
1164
|
+
An exponent ends the check, so ``1e5x`` counts, as it does in GLE.
|
|
1165
|
+
"""
|
|
1166
|
+
n = len(text)
|
|
1167
|
+
pos = 0
|
|
1168
|
+
if pos < n and text[pos] in "+-":
|
|
1169
|
+
pos += 1
|
|
1170
|
+
before = after = 0
|
|
1171
|
+
if pos < n and text[pos] == ".":
|
|
1172
|
+
pos += 1
|
|
1173
|
+
while pos < n and text[pos].isdigit() and text[pos].isascii():
|
|
1174
|
+
pos += 1
|
|
1175
|
+
after += 1
|
|
1176
|
+
else:
|
|
1177
|
+
while pos < n and text[pos].isdigit() and text[pos].isascii():
|
|
1178
|
+
pos += 1
|
|
1179
|
+
before += 1
|
|
1180
|
+
if pos < n and text[pos] == ".":
|
|
1181
|
+
pos += 1
|
|
1182
|
+
while pos < n and text[pos].isdigit() and text[pos].isascii():
|
|
1183
|
+
pos += 1
|
|
1184
|
+
after += 1
|
|
1185
|
+
if not (before or after):
|
|
1186
|
+
return False
|
|
1187
|
+
if pos < n and text[pos] in "eE":
|
|
1188
|
+
pos += 1
|
|
1189
|
+
if pos < n and text[pos] in "+-":
|
|
1190
|
+
pos += 1
|
|
1191
|
+
return pos < n and text[pos].isdigit() and text[pos].isascii()
|
|
1192
|
+
return pos == n
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
def _gle_float_or_missing(cell: str) -> bool:
|
|
1196
|
+
"""GLE's ``isFloatMiss``: a number, or a missing value (``*?-.`` or empty)."""
|
|
1197
|
+
return len(cell) == 0 or (len(cell) == 1 and cell in "*?-.") or _gle_is_float(cell)
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def _gle_rows(
|
|
1201
|
+
text: str, comment: str, delimiters: str, ignore: int, numrows: int
|
|
1202
|
+
) -> Iterator[List[str]]:
|
|
1203
|
+
"""Rows of cells, as ``GLECSVData::parseBlock`` splits ``text``.
|
|
1204
|
+
|
|
1205
|
+
Each row is yielded once the next one starts, so a caller can stop
|
|
1206
|
+
early.
|
|
1207
|
+
|
|
1208
|
+
Raises
|
|
1209
|
+
------
|
|
1210
|
+
ValueError
|
|
1211
|
+
For a string GLE leaves unterminated.
|
|
1212
|
+
"""
|
|
1213
|
+
data = text
|
|
1214
|
+
size = len(data)
|
|
1215
|
+
pos = 0
|
|
1216
|
+
rows: List[List[str]] = []
|
|
1217
|
+
next_line = True
|
|
1218
|
+
last_delim_space = " " in delimiters or "\t" in delimiters
|
|
1219
|
+
|
|
1220
|
+
def read() -> str:
|
|
1221
|
+
nonlocal pos
|
|
1222
|
+
if pos == size:
|
|
1223
|
+
return ""
|
|
1224
|
+
pos += 1
|
|
1225
|
+
return data[pos - 1]
|
|
1226
|
+
|
|
1227
|
+
def go_back() -> None:
|
|
1228
|
+
nonlocal pos
|
|
1229
|
+
pos = max(pos - 1, 0)
|
|
1230
|
+
|
|
1231
|
+
def is_eol(ch: str) -> bool:
|
|
1232
|
+
return ch in ("\n", "\r") and ch != ""
|
|
1233
|
+
|
|
1234
|
+
def is_space(ch: str) -> bool:
|
|
1235
|
+
return ch in (" ", "\t") and ch != ""
|
|
1236
|
+
|
|
1237
|
+
def is_delim(ch: str) -> bool:
|
|
1238
|
+
return ch != "" and ch in delimiters
|
|
1239
|
+
|
|
1240
|
+
def is_comment(ch: str) -> bool:
|
|
1241
|
+
nonlocal pos
|
|
1242
|
+
start = pos
|
|
1243
|
+
matched = 0
|
|
1244
|
+
while matched < len(comment) and ch == comment[matched]:
|
|
1245
|
+
ch = read()
|
|
1246
|
+
matched += 1
|
|
1247
|
+
if matched == len(comment):
|
|
1248
|
+
go_back()
|
|
1249
|
+
return True
|
|
1250
|
+
pos = start
|
|
1251
|
+
return False
|
|
1252
|
+
|
|
1253
|
+
def create(cell: str) -> None:
|
|
1254
|
+
nonlocal next_line
|
|
1255
|
+
if next_line:
|
|
1256
|
+
rows.append([])
|
|
1257
|
+
next_line = False
|
|
1258
|
+
rows[-1].append(cell)
|
|
1259
|
+
|
|
1260
|
+
def size_ok_at_eol(cell_size: int) -> bool:
|
|
1261
|
+
return cell_size != 0 or not (next_line or last_delim_space)
|
|
1262
|
+
|
|
1263
|
+
def read_newline(prev: str) -> bool:
|
|
1264
|
+
nonlocal next_line
|
|
1265
|
+
next_line = True
|
|
1266
|
+
ch = read()
|
|
1267
|
+
if ch == "":
|
|
1268
|
+
return False
|
|
1269
|
+
if not (is_eol(ch) and ch != prev):
|
|
1270
|
+
go_back()
|
|
1271
|
+
return True
|
|
1272
|
+
|
|
1273
|
+
def skip_till_eol() -> bool:
|
|
1274
|
+
while True:
|
|
1275
|
+
ch = read()
|
|
1276
|
+
if ch == "":
|
|
1277
|
+
return False
|
|
1278
|
+
if is_eol(ch):
|
|
1279
|
+
return read_newline(ch)
|
|
1280
|
+
|
|
1281
|
+
def skip_spaces_and_first_delim(ch: str) -> bool:
|
|
1282
|
+
nonlocal last_delim_space
|
|
1283
|
+
while True:
|
|
1284
|
+
if not is_space(ch):
|
|
1285
|
+
if ch == "":
|
|
1286
|
+
return False
|
|
1287
|
+
if is_eol(ch):
|
|
1288
|
+
return read_newline(ch)
|
|
1289
|
+
if is_delim(ch):
|
|
1290
|
+
last_delim_space = False
|
|
1291
|
+
else:
|
|
1292
|
+
go_back()
|
|
1293
|
+
return True
|
|
1294
|
+
ch = read()
|
|
1295
|
+
|
|
1296
|
+
def read_quoted(quote: str) -> bool:
|
|
1297
|
+
cell = quote
|
|
1298
|
+
while True:
|
|
1299
|
+
ch = read()
|
|
1300
|
+
if ch == "" or is_eol(ch):
|
|
1301
|
+
raise ValueError("unterminated string")
|
|
1302
|
+
cell += ch
|
|
1303
|
+
if ch == quote:
|
|
1304
|
+
ch = read()
|
|
1305
|
+
if ch != quote:
|
|
1306
|
+
create(cell)
|
|
1307
|
+
return skip_spaces_and_first_delim(ch)
|
|
1308
|
+
|
|
1309
|
+
def read_cell() -> bool:
|
|
1310
|
+
nonlocal last_delim_space
|
|
1311
|
+
ch = read()
|
|
1312
|
+
while is_space(ch):
|
|
1313
|
+
ch = read()
|
|
1314
|
+
if ch in ('"', "'"):
|
|
1315
|
+
return read_quoted(ch)
|
|
1316
|
+
start = max(pos - 1, 0)
|
|
1317
|
+
count = cell_size = 0
|
|
1318
|
+
while True:
|
|
1319
|
+
if ch == "":
|
|
1320
|
+
if size_ok_at_eol(cell_size):
|
|
1321
|
+
create(data[start : start + cell_size])
|
|
1322
|
+
return False
|
|
1323
|
+
if is_eol(ch):
|
|
1324
|
+
if size_ok_at_eol(cell_size):
|
|
1325
|
+
create(data[start : start + cell_size])
|
|
1326
|
+
return read_newline(ch)
|
|
1327
|
+
if is_delim(ch):
|
|
1328
|
+
last_delim_space = is_space(ch)
|
|
1329
|
+
if cell_size or not is_space(ch):
|
|
1330
|
+
create(data[start : start + cell_size])
|
|
1331
|
+
return skip_spaces_and_first_delim(ch)
|
|
1332
|
+
if is_comment(ch):
|
|
1333
|
+
if size_ok_at_eol(cell_size):
|
|
1334
|
+
create(data[start : start + cell_size])
|
|
1335
|
+
return skip_till_eol()
|
|
1336
|
+
count += 1
|
|
1337
|
+
if not is_space(ch):
|
|
1338
|
+
cell_size = count
|
|
1339
|
+
ch = read()
|
|
1340
|
+
|
|
1341
|
+
more = True
|
|
1342
|
+
for _ in range(ignore):
|
|
1343
|
+
more = skip_till_eol()
|
|
1344
|
+
done = 0
|
|
1345
|
+
while more:
|
|
1346
|
+
more = read_cell()
|
|
1347
|
+
if next_line and numrows and len(rows) >= numrows:
|
|
1348
|
+
break
|
|
1349
|
+
while done < len(rows) - 1:
|
|
1350
|
+
yield rows[done]
|
|
1351
|
+
done += 1
|
|
1352
|
+
yield from rows[done:]
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
def gle_data_layout(
|
|
1356
|
+
path: Union[str, Path],
|
|
1357
|
+
*,
|
|
1358
|
+
comment: str = "!",
|
|
1359
|
+
ignore: int = 0,
|
|
1360
|
+
numrows: int = 0,
|
|
1361
|
+
) -> GLEDataLayout:
|
|
1362
|
+
"""How GLE's ``data`` command reads ``path`` under the statement's options.
|
|
1363
|
+
|
|
1364
|
+
Parameters
|
|
1365
|
+
----------
|
|
1366
|
+
path : str or pathlib.Path
|
|
1367
|
+
The data file.
|
|
1368
|
+
comment, ignore, numrows
|
|
1369
|
+
The ``data`` statement's ``comment``, ``ignore`` and ``numrows``
|
|
1370
|
+
(``numrows 0`` reads every row), with GLE's defaults.
|
|
1371
|
+
|
|
1372
|
+
Returns
|
|
1373
|
+
-------
|
|
1374
|
+
GLEDataLayout
|
|
1375
|
+
|
|
1376
|
+
Raises
|
|
1377
|
+
------
|
|
1378
|
+
OSError
|
|
1379
|
+
If the file cannot be read.
|
|
1380
|
+
ValueError
|
|
1381
|
+
For a string GLE leaves unterminated in the rows read.
|
|
1382
|
+
|
|
1383
|
+
Notes
|
|
1384
|
+
-----
|
|
1385
|
+
Rows are read only until the answer is known: the first body row with
|
|
1386
|
+
a number in its first column settles it. GLE's check that every row
|
|
1387
|
+
has as many cells is not repeated.
|
|
1388
|
+
"""
|
|
1389
|
+
# Bytes as characters: GLE splits bytes, and a UTF-8 BOM is a cell's text.
|
|
1390
|
+
text = Path(path).read_bytes().decode("latin-1")
|
|
1391
|
+
rows = _gle_rows(text, comment, _GLE_DELIMITERS, ignore, numrows)
|
|
1392
|
+
first = next(rows, None)
|
|
1393
|
+
if first is None:
|
|
1394
|
+
return GLEDataLayout(n_cols=0, labels_column=False)
|
|
1395
|
+
header = not any(_gle_float_or_missing(cell) for cell in first)
|
|
1396
|
+
body = rows if header else itertools.chain([first], rows)
|
|
1397
|
+
labels = False
|
|
1398
|
+
for row in body:
|
|
1399
|
+
if _gle_float_or_missing(row[0]):
|
|
1400
|
+
labels = False
|
|
1401
|
+
break
|
|
1402
|
+
labels = True
|
|
1403
|
+
return GLEDataLayout(n_cols=len(first), labels_column=labels)
|