compass-lib 0.0.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,701 @@
1
+ # -*- coding: utf-8 -*-
2
+ """Parser for Compass .DAT survey data files.
3
+
4
+ This module implements the parser for reading Compass survey data files,
5
+ which contain cave survey measurements organized into surveys.
6
+
7
+ Architecture: The parser produces dictionaries (like loading JSON) which are
8
+ then fed to Pydantic models via a single `model_validate()` call. This keeps
9
+ parsing logic separate from model construction.
10
+
11
+ All data in a .DAT file uses fixed internal units (feet, degrees) and a fixed
12
+ column order regardless of the FORMAT string. The FORMAT string is purely
13
+ display metadata for the Compass editor and is stored as a raw string.
14
+ """
15
+
16
+ import logging
17
+ import re
18
+ from datetime import date
19
+ from pathlib import Path
20
+ from typing import Any
21
+
22
+ from compass_lib.constants import ASCII_ENCODING
23
+ from compass_lib.constants import MISSING_ANGLE_THRESHOLD
24
+ from compass_lib.constants import MISSING_VALUE_THRESHOLD
25
+ from compass_lib.enums import Severity
26
+ from compass_lib.errors import CompassParseError
27
+ from compass_lib.errors import SourceLocation
28
+ from compass_lib.survey.models import CompassDatFile
29
+ from compass_lib.validation import days_in_month
30
+
31
+ logger = logging.getLogger(__name__)
32
+
33
+
34
+ class CompassSurveyParser:
35
+ """Parser for Compass .DAT survey data files.
36
+
37
+ This parser reads Compass survey data files and produces dictionaries
38
+ (like loading JSON from disk). The dictionaries can then be fed to
39
+ Pydantic models via a single `model_validate()` call.
40
+
41
+ Errors are collected rather than thrown, allowing partial parsing
42
+ of malformed files.
43
+
44
+ Attributes:
45
+ errors: List of parsing errors and warnings encountered
46
+ """
47
+
48
+ # Regex patterns
49
+ EOL = re.compile(r"\r\n|\r|\n")
50
+ COLUMN_HEADER = re.compile(
51
+ r"^\s*FROM\s+TO[^\r\n]+(\r\n|\r|\n){2}",
52
+ re.MULTILINE | re.IGNORECASE,
53
+ )
54
+ NON_WHITESPACE = re.compile(r"\S+")
55
+ HEADER_FIELDS = re.compile(
56
+ r"SURVEY (NAME|DATE|TEAM):|COMMENT:|DECLINATION:|FORMAT:|CORRECTIONS2?:|FROM",
57
+ re.IGNORECASE,
58
+ )
59
+
60
+ def __init__(self) -> None:
61
+ """Initialize a new parser with empty error list."""
62
+ self.errors: list[CompassParseError] = []
63
+ self._source: str = "<string>"
64
+
65
+ def _add_error(
66
+ self,
67
+ message: str,
68
+ text: str = "",
69
+ line: int = 0,
70
+ column: int = 0,
71
+ ) -> None:
72
+ """Add an error to the error list."""
73
+ self.errors.append(
74
+ CompassParseError(
75
+ severity=Severity.ERROR,
76
+ message=message,
77
+ location=SourceLocation(
78
+ source=self._source,
79
+ line=line,
80
+ column=column,
81
+ text=text,
82
+ ),
83
+ )
84
+ )
85
+
86
+ def _add_warning(
87
+ self,
88
+ message: str,
89
+ text: str = "",
90
+ line: int = 0,
91
+ column: int = 0,
92
+ ) -> None:
93
+ """Add a warning to the error list."""
94
+ self.errors.append(
95
+ CompassParseError(
96
+ severity=Severity.WARNING,
97
+ message=message,
98
+ location=SourceLocation(
99
+ source=self._source,
100
+ line=line,
101
+ column=column,
102
+ text=text,
103
+ ),
104
+ )
105
+ )
106
+
107
+ # -------------------------------------------------------------------------
108
+ # Dictionary-returning methods (primary API)
109
+ # -------------------------------------------------------------------------
110
+
111
+ def parse_file_to_dict(self, path: Path) -> dict[str, Any]:
112
+ """Parse a Compass survey data file to dictionary.
113
+
114
+ This is the primary parsing method. It returns a dictionary that
115
+ can be directly fed to `CompassDatFile.model_validate()`.
116
+
117
+ Args:
118
+ path: Path to the .DAT file
119
+
120
+ Returns:
121
+ Dictionary with "surveys" key containing list of survey dicts
122
+ """
123
+ self._source = str(path)
124
+ with path.open(mode="r", encoding=ASCII_ENCODING, errors="replace") as f:
125
+ data = f.read()
126
+ return self.parse_string_to_dict(data, str(path))
127
+
128
+ def parse_string_to_dict(
129
+ self,
130
+ data: str,
131
+ source: str = "<string>",
132
+ ) -> dict[str, Any]:
133
+ """Parse survey data from a string to dictionary.
134
+
135
+ Args:
136
+ data: Survey data as string
137
+ source: Source identifier for error messages
138
+
139
+ Returns:
140
+ Dictionary with "surveys" key containing list of survey dicts
141
+ """
142
+ self._source = source
143
+ surveys: list[dict[str, Any]] = []
144
+
145
+ # Split on form feed character
146
+ sections = data.split("\f")
147
+ for section in sections:
148
+ _section = section.strip()
149
+ if _section:
150
+ survey = self._parse_survey_to_dict(_section)
151
+ if survey:
152
+ surveys.append(survey)
153
+
154
+ return {"surveys": surveys}
155
+
156
+ # -------------------------------------------------------------------------
157
+ # Legacy model-returning methods (thin wrappers for backwards compat)
158
+ # -------------------------------------------------------------------------
159
+
160
+ def parse_file(self, path: Path) -> list["CompassSurvey"]: # noqa: F821
161
+ """Parse a Compass survey data file.
162
+
163
+ DEPRECATED: Use parse_file_to_dict() for new code.
164
+
165
+ Args:
166
+ path: Path to the .DAT file
167
+
168
+ Returns:
169
+ List of parsed surveys
170
+ """
171
+
172
+ data = self.parse_file_to_dict(path)
173
+ dat_file = CompassDatFile.model_validate(data)
174
+ return dat_file.surveys
175
+
176
+ def parse_string(
177
+ self,
178
+ data: str,
179
+ source: str = "<string>",
180
+ ) -> list["CompassSurvey"]: # noqa: F821
181
+ """Parse survey data from a string.
182
+
183
+ DEPRECATED: Use parse_string_to_dict() for new code.
184
+
185
+ Args:
186
+ data: Survey data as string
187
+ source: Source identifier for error messages
188
+
189
+ Returns:
190
+ List of parsed surveys
191
+ """
192
+
193
+ parsed = self.parse_string_to_dict(data, source)
194
+ dat_file = CompassDatFile.model_validate(parsed)
195
+ return dat_file.surveys
196
+
197
+ def _split_header_and_data(self, text: str) -> tuple[str, str]:
198
+ """Split survey text into header and shot data sections.
199
+
200
+ Args:
201
+ text: Complete survey text
202
+
203
+ Returns:
204
+ Tuple of (header_text, data_text)
205
+ """
206
+ match = self.COLUMN_HEADER.search(text)
207
+ if match:
208
+ header_end = match.end()
209
+ return text[:header_end].strip(), text[header_end:].strip()
210
+ return text.strip(), ""
211
+
212
+ def _parse_survey_to_dict(self, text: str) -> dict[str, Any] | None:
213
+ """Parse a single survey from text to dictionary.
214
+
215
+ Args:
216
+ text: survey text (header + shots)
217
+
218
+ Returns:
219
+ Dictionary with "header" and "shots" keys, or None if parsing fails
220
+ """
221
+ header_text, data_text = self._split_header_and_data(text)
222
+
223
+ header = self._parse_survey_header_to_dict(header_text)
224
+ if header is None:
225
+ return None
226
+
227
+ shots: list[dict[str, Any]] = []
228
+ if data_text:
229
+ for line in self.EOL.split(data_text):
230
+ _line = line.strip()
231
+ if _line:
232
+ if shot := self._parse_shot_to_dict(_line, header):
233
+ shots.append(shot)
234
+
235
+ return {"header": header, "shots": shots}
236
+
237
+ def _parse_survey_header_to_dict(self, text: str) -> dict[str, Any] | None:
238
+ """Parse survey header from text to dictionary.
239
+
240
+ Args:
241
+ text: Header text
242
+
243
+ Returns:
244
+ Dictionary with header fields, or None if parsing fails
245
+ """
246
+ lines = self.EOL.split(text, maxsplit=1)
247
+ if len(lines) < 2:
248
+ return None
249
+
250
+ # Build header dictionary
251
+ header: dict[str, Any] = {
252
+ "cave_name": lines[0].strip(),
253
+ "survey_name": None,
254
+ "date": None,
255
+ "comment": None,
256
+ "team": None,
257
+ "declination": 0.0,
258
+ "format_string": None,
259
+ "has_backsights": True,
260
+ "length_correction": 0.0,
261
+ "frontsight_azimuth_correction": 0.0,
262
+ "frontsight_inclination_correction": 0.0,
263
+ "backsight_azimuth_correction": 0.0,
264
+ "backsight_inclination_correction": 0.0,
265
+ }
266
+
267
+ # Extract fields using regex
268
+ rest = lines[1]
269
+ fields = self._extract_fields(rest)
270
+
271
+ for field_name, field_value in fields:
272
+ field_upper = field_name.upper()
273
+ value = field_value.strip()
274
+
275
+ if field_upper == "SURVEY NAME:":
276
+ match = self.NON_WHITESPACE.search(value)
277
+ if match:
278
+ header["survey_name"] = match.group()
279
+
280
+ elif field_upper == "SURVEY DATE:":
281
+ parsed_date = self._parse_date(value)
282
+ if parsed_date:
283
+ header["date"] = parsed_date.isoformat()
284
+
285
+ elif field_upper == "COMMENT:":
286
+ header["comment"] = value if value else None
287
+
288
+ elif field_upper == "SURVEY TEAM:":
289
+ header["team"] = value if value else None
290
+
291
+ elif field_upper == "DECLINATION:":
292
+ dec = self._parse_measurement(value)
293
+ if dec is not None:
294
+ header["declination"] = dec
295
+
296
+ elif field_upper == "FORMAT:":
297
+ self._parse_shot_format_to_dict(header, value)
298
+
299
+ elif field_upper == "CORRECTIONS:":
300
+ parts = self.NON_WHITESPACE.findall(value)
301
+ if len(parts) >= 1:
302
+ val = self._parse_measurement(parts[0])
303
+ if val is not None:
304
+ header["length_correction"] = val
305
+ if len(parts) >= 2:
306
+ val = self._parse_measurement(parts[1])
307
+ if val is not None:
308
+ header["frontsight_azimuth_correction"] = val
309
+ if len(parts) >= 3:
310
+ val = self._parse_measurement(parts[2])
311
+ if val is not None:
312
+ header["frontsight_inclination_correction"] = val
313
+
314
+ elif field_upper == "CORRECTIONS2:":
315
+ parts = self.NON_WHITESPACE.findall(value)
316
+ if len(parts) >= 1:
317
+ val = self._parse_measurement(parts[0])
318
+ if val is not None:
319
+ header["backsight_azimuth_correction"] = val
320
+ if len(parts) >= 2:
321
+ val = self._parse_measurement(parts[1])
322
+ if val is not None:
323
+ header["backsight_inclination_correction"] = val
324
+
325
+ return header
326
+
327
+ def _extract_fields(self, text: str) -> list[tuple[str, str]]:
328
+ """Extract field name-value pairs from header text.
329
+
330
+ Args:
331
+ text: Header text after cave name
332
+
333
+ Returns:
334
+ List of (field_name, field_value) tuples
335
+ """
336
+ fields: list[tuple[str, str]] = []
337
+ matches = list(self.HEADER_FIELDS.finditer(text))
338
+
339
+ for i, match in enumerate(matches):
340
+ field_name = match.group()
341
+ start = match.end()
342
+ end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
343
+ field_value = text[start:end]
344
+ fields.append((field_name, field_value))
345
+
346
+ return fields
347
+
348
+ def _parse_date(self, text: str) -> date | None: # noqa: PLR0911
349
+ """Parse date from text (month day year format).
350
+
351
+ Args:
352
+ text: Date text
353
+
354
+ Returns:
355
+ Parsed date or None if invalid
356
+ """
357
+ parts = self.NON_WHITESPACE.findall(text)
358
+ if len(parts) < 3:
359
+ self._add_error(f"incomplete date: {text}", text)
360
+ return None
361
+
362
+ try:
363
+ month = int(parts[0])
364
+ day = int(parts[1])
365
+ year = int(parts[2])
366
+ except ValueError:
367
+ self._add_error(f"invalid date: {text}", text)
368
+ return None
369
+
370
+ # Handle 2-digit years (years < 100 are treated as offset from 1900)
371
+ if year < 100:
372
+ year += 1900
373
+
374
+ # Validate ranges with leap year aware day validation
375
+ if not 1 <= month <= 12:
376
+ self._add_error(f"month must be between 1 and 12: {month}", parts[0])
377
+ return None
378
+ if year < 0:
379
+ self._add_error(f"year must be >= 0: {year}", parts[2])
380
+ return None
381
+
382
+ max_day = days_in_month(month, year)
383
+ if not 1 <= day <= max_day:
384
+ self._add_error(
385
+ f"day must be between 1 and {max_day} for month {month}: {day}",
386
+ parts[1],
387
+ )
388
+ return None
389
+
390
+ try:
391
+ return date(year, month, day)
392
+ except ValueError as e:
393
+ self._add_error(f"invalid date: {e}", text)
394
+ return None
395
+
396
+ def _parse_measurement(self, text: str) -> float | None:
397
+ """Parse a numeric measurement from text.
398
+
399
+ Values >= 990 indicate missing data.
400
+
401
+ Args:
402
+ text: Measurement text
403
+
404
+ Returns:
405
+ Parsed value or None if missing/invalid
406
+ """
407
+ try:
408
+ value = float(text)
409
+ if value >= MISSING_VALUE_THRESHOLD:
410
+ return None
411
+ except ValueError:
412
+ return None
413
+
414
+ return value
415
+
416
+ def _parse_angle_measurement(self, text: str) -> float | None:
417
+ """Parse an angle measurement from text.
418
+
419
+ Values < -900 or >= 990 indicate missing data.
420
+
421
+ Args:
422
+ text: Measurement text
423
+
424
+ Returns:
425
+ Parsed value or None if missing/invalid
426
+ """
427
+ try:
428
+ value = float(text)
429
+ if value < MISSING_ANGLE_THRESHOLD:
430
+ return None
431
+ if value >= MISSING_VALUE_THRESHOLD:
432
+ return None
433
+
434
+ except ValueError:
435
+ return None
436
+
437
+ return value
438
+
439
+ def _parse_azimuth(self, text: str) -> float | None:
440
+ """Parse azimuth measurement with validation.
441
+
442
+ Args:
443
+ text: Azimuth text
444
+
445
+ Returns:
446
+ Parsed value or None if invalid
447
+ """
448
+ value = self._parse_angle_measurement(text)
449
+ if value is None:
450
+ return None
451
+ if value < 0 or value >= 360:
452
+ self._add_error(
453
+ f"azimuth must be >= 0 and < 360, got {value}",
454
+ text,
455
+ )
456
+ # Still return the value for continued parsing
457
+ return value
458
+
459
+ def _parse_inclination(self, text: str) -> float | None:
460
+ """Parse inclination measurement with validation.
461
+
462
+ Args:
463
+ text: Inclination text
464
+
465
+ Returns:
466
+ Parsed value or None if invalid
467
+ """
468
+ value = self._parse_angle_measurement(text)
469
+ if value is None:
470
+ return None
471
+ if value < -90 or value > 90:
472
+ self._add_error(
473
+ f"inclination must be between -90 and 90, got {value}",
474
+ text,
475
+ )
476
+ # Still return the value for continued parsing
477
+ return value
478
+
479
+ def _parse_distance(self, text: str) -> float | None:
480
+ """Parse distance measurement with validation (must be >= 0).
481
+
482
+ Args:
483
+ text: Distance text
484
+
485
+ Returns:
486
+ Parsed value or None if invalid
487
+ """
488
+ value = self._parse_measurement(text)
489
+ if value is None:
490
+ return None
491
+ if value < 0:
492
+ self._add_error(f"distance must be >= 0, got {value}", text)
493
+ return value
494
+
495
+ def _parse_lrud(self, text: str) -> float | None:
496
+ """Parse LRUD measurement with validation.
497
+
498
+ Values < -1 or > 990 indicate missing data.
499
+ Values between -1 and 0 generate an error.
500
+
501
+ Args:
502
+ text: LRUD text
503
+
504
+ Returns:
505
+ Parsed value or None if missing
506
+ """
507
+ try:
508
+ value = float(text)
509
+ except ValueError:
510
+ return None
511
+
512
+ # Missing data indicators
513
+ if value < -1 or value > MISSING_VALUE_THRESHOLD:
514
+ return None
515
+
516
+ # Values between -1 and 0 are errors but still parse
517
+ if value < 0:
518
+ self._add_error(f"LRUD must be >= 0, got {value}", text)
519
+
520
+ return value
521
+
522
+ def _parse_shot_format_to_dict(
523
+ self,
524
+ header: dict[str, Any],
525
+ format_str: str,
526
+ ) -> None:
527
+ """Parse the FORMAT string and update header dictionary.
528
+
529
+ The FORMAT string is purely display metadata for the Compass editor.
530
+ It does NOT affect how shot data is parsed -- all data uses fixed
531
+ internal units (feet, degrees) and fixed column order.
532
+
533
+ The only structural information extracted is ``has_backsights``,
534
+ which determines whether backsight columns are present in the data.
535
+
536
+ The raw FORMAT string is stored as ``format_string`` for round-trip
537
+ fidelity.
538
+
539
+ Args:
540
+ header: Header dictionary to update
541
+ format_str: Format string (11-15 characters)
542
+ """
543
+ format_str = format_str.strip()
544
+ if len(format_str) < 11:
545
+ self._add_error(
546
+ f"format must be at least 11 characters, got {len(format_str)}",
547
+ format_str,
548
+ )
549
+ return
550
+
551
+ # Store the raw format string
552
+ header["format_string"] = format_str
553
+
554
+ # Extract has_backsights from the backsight flag position.
555
+ # The backsight flag position depends on the format string length:
556
+ # 11-char: no backsight flag -> default False
557
+ # 12-char: backsight flag at position 11
558
+ # 13-char: backsight flag at position 11
559
+ # 15-char: backsight flag at position 13 (5-item shot order)
560
+ # 14-char: unusual, treat backsight flag at position 11
561
+ format_len = len(format_str)
562
+ if format_len >= 15:
563
+ backsight_pos = 13
564
+ elif format_len >= 12:
565
+ backsight_pos = 11
566
+ else:
567
+ # 11-char format has no backsight flag
568
+ header["has_backsights"] = False
569
+ return
570
+
571
+ header["has_backsights"] = format_str[backsight_pos].upper() == "B"
572
+
573
+ def _parse_shot_to_dict(
574
+ self,
575
+ line: str,
576
+ header: dict[str, Any],
577
+ ) -> dict[str, Any] | None:
578
+ """Parse a single shot line to dictionary.
579
+
580
+ Compass .DAT files have FIXED column order regardless of FORMAT string.
581
+ Columns are always:
582
+ FROM TO LENGTH AZIMUTH INCLINATION LEFT UP DOWN RIGHT [BS_AZ BS_INC] [FLAGS] [COMMENT]
583
+
584
+ All values are in fixed internal units: feet for lengths, degrees for angles.
585
+
586
+ Args:
587
+ line: Shot line text
588
+ header: Survey header dictionary (needs ``has_backsights``)
589
+
590
+ Returns:
591
+ Shot dictionary or None if line is invalid
592
+ """ # noqa: E501
593
+ parts = self.NON_WHITESPACE.findall(line)
594
+ if len(parts) < 2:
595
+ return None
596
+
597
+ idx = 0
598
+
599
+ # Station names (always positions 0, 1)
600
+ from_station = parts[idx]
601
+ idx += 1
602
+ to_station = parts[idx]
603
+ idx += 1
604
+
605
+ # Parse LENGTH (always position 2)
606
+ length = self._parse_distance(parts[idx]) if idx < len(parts) else None
607
+ idx += 1
608
+
609
+ # Parse AZIMUTH (always position 3)
610
+ fs_azimuth = self._parse_azimuth(parts[idx]) if idx < len(parts) else None
611
+ idx += 1
612
+
613
+ # Parse INCLINATION (always position 4) -- always in degrees
614
+ fs_inclination = (
615
+ self._parse_inclination(parts[idx]) if idx < len(parts) else None
616
+ )
617
+ idx += 1
618
+
619
+ # Check if length is valid - if not and we've exhausted parts, skip
620
+ if length is None and idx >= len(parts):
621
+ return None
622
+
623
+ # LRUD values - FIXED order: LEFT UP DOWN RIGHT (positions 5-8)
624
+ left = self._parse_lrud(parts[idx]) if idx < len(parts) else None
625
+ idx += 1
626
+ up = self._parse_lrud(parts[idx]) if idx < len(parts) else None
627
+ idx += 1
628
+ down = self._parse_lrud(parts[idx]) if idx < len(parts) else None
629
+ idx += 1
630
+ right = self._parse_lrud(parts[idx]) if idx < len(parts) else None
631
+ idx += 1
632
+
633
+ # Backsight values (optional, positions 9-10)
634
+ bs_azimuth: float | None = None
635
+ bs_inclination: float | None = None
636
+ has_backsights = header.get("has_backsights", False)
637
+
638
+ if has_backsights:
639
+ if idx < len(parts):
640
+ bs_azimuth = self._parse_azimuth(parts[idx])
641
+ idx += 1
642
+ if idx < len(parts):
643
+ bs_inclination = self._parse_inclination(parts[idx])
644
+ idx += 1
645
+
646
+ # Flags and comment
647
+ excluded_from_length = False
648
+ excluded_from_plotting = False
649
+ excluded_from_all_processing = False
650
+ do_not_adjust = False
651
+ comment = None
652
+
653
+ # Look for remaining parts (flags and comments)
654
+ remaining = " ".join(parts[idx:]) if idx < len(parts) else ""
655
+
656
+ # Check for flags pattern #|..#
657
+ flag_match = re.search(r"#\|([^#]*?)#", remaining)
658
+ if flag_match:
659
+ flags_str = flag_match.group(1)
660
+ for flag in flags_str:
661
+ flag_upper = flag.upper()
662
+ if flag_upper == "L":
663
+ excluded_from_length = True
664
+ elif flag_upper == "P":
665
+ excluded_from_plotting = True
666
+ elif flag_upper == "X":
667
+ excluded_from_all_processing = True
668
+ elif flag_upper == "C":
669
+ do_not_adjust = True
670
+ elif flag_upper == " ":
671
+ pass # Spaces are allowed
672
+ else:
673
+ self._add_warning(f"unrecognized flag: {flag}", flag)
674
+
675
+ # Comment is after the flags
676
+ comment_start = flag_match.end()
677
+ if comment_start < len(remaining):
678
+ comment = remaining[comment_start:].strip() or None
679
+ # No flags, remaining is comment
680
+ elif remaining.strip():
681
+ comment = remaining.strip()
682
+
683
+ # Return dictionary with alias names for Pydantic
684
+ return {
685
+ "from_station": from_station,
686
+ "to_station": to_station,
687
+ "distance": length,
688
+ "frontsight_azimuth": fs_azimuth,
689
+ "frontsight_inclination": fs_inclination,
690
+ "backsight_azimuth": bs_azimuth,
691
+ "backsight_inclination": bs_inclination,
692
+ "left": left,
693
+ "right": right,
694
+ "up": up,
695
+ "down": down,
696
+ "comment": comment,
697
+ "exclude_distance": excluded_from_length,
698
+ "excluded_from_plotting": excluded_from_plotting,
699
+ "excluded_from_all_processing": excluded_from_all_processing,
700
+ "do_not_adjust": do_not_adjust,
701
+ }