aims-convert 2.0__tar.gz → 2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aims_convert-2.0 → aims_convert-2.2}/PKG-INFO +3 -1
- {aims_convert-2.0 → aims_convert-2.2}/aims/cli.py +9 -3
- {aims_convert-2.0 → aims_convert-2.2}/aims/data_structures.py +14 -12
- {aims_convert-2.0 → aims_convert-2.2}/aims/gui.py +7 -5
- {aims_convert-2.0 → aims_convert-2.2}/aims/logbook_report.py +11 -27
- {aims_convert-2.0 → aims_convert-2.2}/aims/output.py +74 -48
- aims_convert-2.2/aims/parse.py +16 -0
- aims_convert-2.2/aims/roster.py +239 -0
- aims_convert-2.2/aims/version.py +4 -0
- {aims_convert-2.0 → aims_convert-2.2}/aims_convert.egg-info/PKG-INFO +3 -1
- {aims_convert-2.0 → aims_convert-2.2}/aims_convert.egg-info/SOURCES.txt +3 -0
- {aims_convert-2.0 → aims_convert-2.2}/aims_convert.egg-info/requires.txt +3 -0
- {aims_convert-2.0 → aims_convert-2.2}/setup.py +4 -1
- aims_convert-2.2/tests/test_compound.py +34 -0
- {aims_convert-2.0 → aims_convert-2.2}/tests/test_logbook.py +15 -17
- aims_convert-2.2/tests/test_output.py +231 -0
- {aims_convert-2.0 → aims_convert-2.2}/tests/test_roster.py +74 -67
- aims_convert-2.0/aims/parse.py +0 -21
- aims_convert-2.0/aims/roster.py +0 -131
- {aims_convert-2.0 → aims_convert-2.2}/README.md +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/aims/__init__.py +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/aims/py.typed +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/aims_convert.egg-info/dependency_links.txt +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/aims_convert.egg-info/entry_points.txt +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/aims_convert.egg-info/top_level.txt +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/pyproject.toml +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/setup.cfg +0 -0
- {aims_convert-2.0 → aims_convert-2.2}/tests/test_parse.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: aims-convert
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2
|
|
4
4
|
Summary: Extract useful information from AIMS
|
|
5
5
|
Home-page: https://github.com/JonHurst/aims-convert
|
|
6
6
|
Author: Jon Hurst
|
|
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
Requires-Dist: nightflight
|
|
16
16
|
Requires-Dist: bs4
|
|
17
17
|
Requires-Dist: html5lib
|
|
18
|
+
Provides-Extra: unit-tests
|
|
19
|
+
Requires-Dist: freezegun; extra == "unit-tests"
|
|
18
20
|
|
|
19
21
|
# AIMS Roster Data Extraction #
|
|
20
22
|
|
|
@@ -5,6 +5,7 @@ import argparse
|
|
|
5
5
|
|
|
6
6
|
from aims.parse import parse
|
|
7
7
|
import aims.output as output
|
|
8
|
+
from aims.version import VERSION
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
def _args():
|
|
@@ -12,14 +13,17 @@ def _args():
|
|
|
12
13
|
description=(
|
|
13
14
|
'Process an AIMS detailed roster into various useful formats.'))
|
|
14
15
|
parser.add_argument('format',
|
|
15
|
-
choices=['roster', 'efj', 'csv', 'ical'])
|
|
16
|
+
choices=['roster', 'efj', 'csv', 'ical', 'version'])
|
|
16
17
|
parser.add_argument('--ade', action="store_true")
|
|
17
18
|
return parser.parse_args()
|
|
18
19
|
|
|
19
20
|
|
|
20
21
|
def main() -> int:
|
|
21
22
|
args = _args()
|
|
22
|
-
|
|
23
|
+
if args.format == "version":
|
|
24
|
+
print(f"Version: {VERSION}")
|
|
25
|
+
return 0
|
|
26
|
+
duties, ade = parse(sys.stdin.read())
|
|
23
27
|
if args.format == "roster":
|
|
24
28
|
print(output.roster(duties))
|
|
25
29
|
elif args.format == "efj":
|
|
@@ -27,7 +31,9 @@ def main() -> int:
|
|
|
27
31
|
elif args.format == "csv":
|
|
28
32
|
print(output.csv(duties))
|
|
29
33
|
elif args.format == "ical":
|
|
30
|
-
|
|
34
|
+
if not args.ade:
|
|
35
|
+
ade = ()
|
|
36
|
+
print(output.ical(duties, ade))
|
|
31
37
|
return 0
|
|
32
38
|
|
|
33
39
|
|
|
@@ -2,6 +2,16 @@ from typing import NamedTuple, Optional
|
|
|
2
2
|
import datetime as dt
|
|
3
3
|
|
|
4
4
|
|
|
5
|
+
class AllDayEvent(NamedTuple):
|
|
6
|
+
date: dt.date
|
|
7
|
+
code: str
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class CrewMember(NamedTuple):
|
|
11
|
+
name: str
|
|
12
|
+
role: str
|
|
13
|
+
|
|
14
|
+
|
|
5
15
|
class Sector(NamedTuple):
|
|
6
16
|
name: str
|
|
7
17
|
reg: Optional[str]
|
|
@@ -12,26 +22,18 @@ class Sector(NamedTuple):
|
|
|
12
22
|
on: dt.datetime
|
|
13
23
|
quasi: bool
|
|
14
24
|
position: bool
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class CrewMember(NamedTuple):
|
|
18
|
-
name: str
|
|
19
|
-
role: str
|
|
25
|
+
crew: tuple[CrewMember, ...]
|
|
20
26
|
|
|
21
27
|
|
|
22
28
|
class Duty(NamedTuple):
|
|
23
|
-
code: Optional[str] # only all day event has code
|
|
24
29
|
start: dt.datetime
|
|
25
|
-
finish:
|
|
30
|
+
finish: dt.datetime
|
|
26
31
|
sectors: tuple[Sector, ...]
|
|
27
|
-
crew: tuple[CrewMember, ...]
|
|
28
32
|
|
|
29
33
|
|
|
30
34
|
class RosterException(Exception):
|
|
31
|
-
|
|
32
|
-
def __str__(self):
|
|
33
|
-
return self.__doc__
|
|
35
|
+
"Base class"
|
|
34
36
|
|
|
35
37
|
|
|
36
38
|
class InputFileException(RosterException):
|
|
37
|
-
"
|
|
39
|
+
"Error in input file"
|
|
@@ -9,8 +9,8 @@ import ctypes
|
|
|
9
9
|
import aims.parse
|
|
10
10
|
from aims.data_structures import RosterException, InputFileException
|
|
11
11
|
from aims.output import csv, ical, efj
|
|
12
|
+
from aims.version import VERSION
|
|
12
13
|
|
|
13
|
-
VERSION = "2.0"
|
|
14
14
|
|
|
15
15
|
SETTINGS_FILE = os.path.expanduser("~/.aimsgui")
|
|
16
16
|
|
|
@@ -328,7 +328,7 @@ class MainWindow(ttk.Frame):
|
|
|
328
328
|
self.txt.insert(tk.END, "Getting registration and type info...")
|
|
329
329
|
self.txt.update()
|
|
330
330
|
|
|
331
|
-
duties = aims.parse.parse(html)
|
|
331
|
+
duties, _ = aims.parse.parse(html)
|
|
332
332
|
# note: normalise newlines for Text widget - will restore on output
|
|
333
333
|
txt = csv(duties).replace("\r\n", "\n")
|
|
334
334
|
self.txt.delete('1.0', tk.END)
|
|
@@ -338,9 +338,11 @@ class MainWindow(ttk.Frame):
|
|
|
338
338
|
html = self.__roster_html()
|
|
339
339
|
if not html:
|
|
340
340
|
return
|
|
341
|
-
duties = aims.parse.parse(html)
|
|
341
|
+
duties, ade = aims.parse.parse(html)
|
|
342
342
|
# note: normalise newlines for Text widget - will restore on output
|
|
343
|
-
|
|
343
|
+
if not self.ms.with_ade.get():
|
|
344
|
+
ade = ()
|
|
345
|
+
txt = ical(duties, ade).replace("\r\n", "\n")
|
|
344
346
|
self.txt.delete('1.0', tk.END)
|
|
345
347
|
self.txt.insert(tk.END, txt, 'ical')
|
|
346
348
|
|
|
@@ -351,7 +353,7 @@ class MainWindow(ttk.Frame):
|
|
|
351
353
|
self.txt.delete('1.0', tk.END)
|
|
352
354
|
self.txt.insert(tk.END, "Working…", 'efj')
|
|
353
355
|
self.txt.update()
|
|
354
|
-
duties = aims.parse.parse(html)
|
|
356
|
+
duties, _ = aims.parse.parse(html)
|
|
355
357
|
txt = efj(duties)
|
|
356
358
|
self.txt.delete('1.0', tk.END)
|
|
357
359
|
self.txt.insert(tk.END, txt, 'efj')
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
#!/usr/bin/python3
|
|
2
1
|
import datetime as dt
|
|
3
|
-
from bs4 import BeautifulSoup # type: ignore
|
|
4
|
-
import sys
|
|
5
2
|
import re
|
|
6
3
|
from typing import Optional
|
|
7
4
|
|
|
8
|
-
from
|
|
5
|
+
from bs4 import BeautifulSoup # type: ignore
|
|
6
|
+
|
|
7
|
+
from aims.data_structures import Duty, Sector, CrewMember, AllDayEvent
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
DATE, FLTNUM, FROM, OFF, TO, ON, TYPE, REG, BLOCK, CP = range(1, 11)
|
|
@@ -23,29 +22,24 @@ def _sector(row: tuple[str, ...]) -> Optional[Sector]:
|
|
|
23
22
|
dt.datetime.strptime(row[ON], "%H:%M").time())
|
|
24
23
|
if on < off:
|
|
25
24
|
on += dt.timedelta(1)
|
|
25
|
+
crew = (CrewMember(row[CP], "CP"), )
|
|
26
26
|
return Sector(row[FLTNUM], row[REG], row[TYPE],
|
|
27
27
|
row[FROM], row[TO],
|
|
28
28
|
off, on,
|
|
29
|
-
False, False)
|
|
29
|
+
False, False, crew)
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def _duty(
|
|
33
|
-
sectors: tuple[Sector, ...],
|
|
34
|
-
crew: dict[dt.datetime, str]
|
|
35
|
-
) -> Duty:
|
|
32
|
+
def _duty(sectors: tuple[Sector, ...]) -> Duty:
|
|
36
33
|
lowest_off = min([X.off for X in sectors])
|
|
37
34
|
highest_on = max([X.on for X in sectors])
|
|
38
|
-
|
|
39
|
-
return Duty(None,
|
|
40
|
-
lowest_off - dt.timedelta(hours=1),
|
|
35
|
+
return Duty(lowest_off - dt.timedelta(hours=1),
|
|
41
36
|
highest_on + dt.timedelta(minutes=30),
|
|
42
|
-
sectors
|
|
43
|
-
tuple(CrewMember(X, "CP") for X in sorted(cpt_names) if X))
|
|
37
|
+
sectors)
|
|
44
38
|
|
|
45
39
|
|
|
46
|
-
def duties(
|
|
40
|
+
def duties(html) -> tuple[tuple[Duty, ...], tuple[AllDayEvent, ...]]:
|
|
41
|
+
soup = BeautifulSoup(html, "html5lib")
|
|
47
42
|
sectors: list[Sector] = []
|
|
48
|
-
crew: dict[dt.datetime, str] = {}
|
|
49
43
|
re_date = re.compile(r"^\d{2}/\d{2}/\d{2}$")
|
|
50
44
|
for row in soup.find_all("tr"):
|
|
51
45
|
strings = tuple(Y[0].replace("\xa0", " ") if Y else "" for Y in
|
|
@@ -54,7 +48,6 @@ def duties(soup) -> tuple[Duty, ...]:
|
|
|
54
48
|
sector = _sector(strings)
|
|
55
49
|
if sector:
|
|
56
50
|
sectors.append(sector)
|
|
57
|
-
crew[sector.off] = strings[CP]
|
|
58
51
|
groups = [[sectors[0]]]
|
|
59
52
|
last_on = sectors[0].on
|
|
60
53
|
for sector in sectors[1:]:
|
|
@@ -63,13 +56,4 @@ def duties(soup) -> tuple[Duty, ...]:
|
|
|
63
56
|
else:
|
|
64
57
|
groups[-1].append(sector)
|
|
65
58
|
last_on = sector.on
|
|
66
|
-
return tuple(_duty(tuple(X)
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
if __name__ == "__main__":
|
|
70
|
-
soup = BeautifulSoup(sys.stdin.read(), "html5lib")
|
|
71
|
-
for d in duties(soup):
|
|
72
|
-
print(d.start, d.finish, d.crew)
|
|
73
|
-
for s in d.sectors:
|
|
74
|
-
print(" ", s)
|
|
75
|
-
print("----")
|
|
59
|
+
return (tuple(_duty(tuple(X)) for X in groups), ())
|
|
@@ -5,12 +5,15 @@ import io
|
|
|
5
5
|
import csv as libcsv
|
|
6
6
|
import datetime as dt
|
|
7
7
|
import re
|
|
8
|
-
|
|
8
|
+
import itertools
|
|
9
9
|
|
|
10
|
-
from aims.data_structures import Duty, Sector, CrewMember
|
|
10
|
+
from aims.data_structures import Duty, Sector, CrewMember, AllDayEvent
|
|
11
11
|
import nightflight.night as nightcalc # type: ignore
|
|
12
12
|
from nightflight.airport_nvecs import airfields as nvecs # type: ignore
|
|
13
13
|
|
|
14
|
+
UTC = Z("UTC")
|
|
15
|
+
LT = Z("Europe/London")
|
|
16
|
+
|
|
14
17
|
|
|
15
18
|
def clean_name(name: str) -> str:
|
|
16
19
|
parts = [X.strip().capitalize() for X in name.split()]
|
|
@@ -52,46 +55,68 @@ def _night(sector: Sector) -> tuple[int, bool]:
|
|
|
52
55
|
return (round(night_duration), night_landing)
|
|
53
56
|
|
|
54
57
|
|
|
55
|
-
def
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
58
|
+
def _build_roster_string(start, end, block, string):
|
|
59
|
+
start, end = [X.replace(tzinfo=UTC).astimezone(LT)
|
|
60
|
+
for X in (start, end)]
|
|
61
|
+
duration = int((end - start).total_seconds()) // 60
|
|
62
|
+
duration_str = f"{duration // 60}:{duration % 60:02d}"
|
|
63
|
+
block_str = f"{block // 60}:{block % 60:02d}"
|
|
64
|
+
return (f"{start:%d/%m/%Y %H:%M}-{end:%H:%M} "
|
|
65
|
+
f"{string} {block_str}/{duration_str}")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _roster_quasi(
|
|
69
|
+
qsectors: tuple[Sector, ...],
|
|
70
|
+
cursor: dt.datetime
|
|
71
|
+
) -> tuple[tuple[str, ...], dt.datetime]:
|
|
72
|
+
retval: list[str] = []
|
|
73
|
+
name = "Brief"
|
|
74
|
+
for s in qsectors:
|
|
75
|
+
if cursor < s.off:
|
|
76
|
+
retval.append(_build_roster_string(cursor, s.off, 0, name))
|
|
77
|
+
name = "Debrief"
|
|
78
|
+
retval.append(_build_roster_string(s.off, s.on, 0, s.name))
|
|
79
|
+
cursor = s.on
|
|
80
|
+
return (tuple(retval), cursor)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _roster_real(
|
|
84
|
+
sectors: tuple[Sector, ...],
|
|
85
|
+
cursor: dt.datetime
|
|
86
|
+
) -> tuple[str, dt.datetime]:
|
|
87
|
+
assert sectors[0].from_
|
|
88
|
+
airports: list[str] = [sectors[0].from_]
|
|
59
89
|
block = 0
|
|
60
|
-
|
|
90
|
+
start = cursor
|
|
61
91
|
for sector in sectors:
|
|
62
|
-
if not from_ and sector.from_:
|
|
63
|
-
from_ = sector.from_
|
|
64
92
|
if sector.position:
|
|
65
93
|
airports.append("[psn]")
|
|
66
|
-
|
|
67
|
-
if sector.from_:
|
|
68
|
-
airports.append(f"[{sector.name}]")
|
|
69
|
-
else:
|
|
70
|
-
airports.append(sector.name)
|
|
71
|
-
if sector.to:
|
|
72
|
-
airports.append(sector.to)
|
|
73
|
-
if not sector.quasi and not sector.position:
|
|
94
|
+
else:
|
|
74
95
|
block += int((sector.on - sector.off).total_seconds()) // 60
|
|
75
|
-
|
|
76
|
-
airports
|
|
77
|
-
|
|
96
|
+
assert sector.to
|
|
97
|
+
airports.append(sector.to)
|
|
98
|
+
cursor = sectors[-1].on + dt.timedelta(minutes=30)
|
|
99
|
+
return (
|
|
100
|
+
_build_roster_string(start, cursor, block, '-'.join(airports)),
|
|
101
|
+
cursor)
|
|
78
102
|
|
|
79
103
|
|
|
80
104
|
def roster(duties: tuple[Duty, ...]) -> str:
|
|
81
|
-
|
|
82
|
-
LT = Z("Europe/London")
|
|
83
|
-
output = []
|
|
105
|
+
output: list[str] = []
|
|
84
106
|
for duty in duties:
|
|
85
|
-
if not duty.finish:
|
|
107
|
+
if not duty.finish: # an all day duty
|
|
86
108
|
continue
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
109
|
+
cursor = duty.start
|
|
110
|
+
for k, g in itertools.groupby(duty.sectors, key=lambda X: X.quasi):
|
|
111
|
+
if k: # group of quasi sectors
|
|
112
|
+
new_output, cursor = _roster_quasi(tuple(g), cursor)
|
|
113
|
+
output += new_output
|
|
114
|
+
else: # group of real sectors
|
|
115
|
+
new, cursor = _roster_real(tuple(g), cursor)
|
|
116
|
+
output.append(new)
|
|
117
|
+
if cursor < duty.finish:
|
|
118
|
+
output.append(
|
|
119
|
+
_build_roster_string(cursor, duty.finish, 0, "Debrief"))
|
|
95
120
|
return "\n".join(output)
|
|
96
121
|
|
|
97
122
|
|
|
@@ -118,13 +143,16 @@ def efj(duties: tuple[Duty, ...]) -> str:
|
|
|
118
143
|
if (len(duty.sectors) == 1 and duty.sectors[0].quasi):
|
|
119
144
|
comment = f" #{duty.sectors[0].name}"
|
|
120
145
|
output.append(f"{duty.start:%H%M}/{duty.finish:%H%M}{comment}")
|
|
121
|
-
if duty.crew:
|
|
122
|
-
crew = [f"{X.role}:{clean_name(X.name)}" for X in duty.crew]
|
|
123
|
-
output.append(f"{{ {', '.join(crew)} }}")
|
|
124
146
|
last_airframe = None
|
|
147
|
+
last_crew = None
|
|
125
148
|
for sector in duty.sectors:
|
|
126
149
|
if sector.position or sector.quasi:
|
|
127
150
|
continue
|
|
151
|
+
if sector.crew:
|
|
152
|
+
crew = [f"{X.role}:{clean_name(X.name)}" for X in sector.crew]
|
|
153
|
+
if crew != last_crew:
|
|
154
|
+
output.append(f"{{ {', '.join(crew)} }}")
|
|
155
|
+
last_crew = crew
|
|
128
156
|
reg = sector.reg or "?-????"
|
|
129
157
|
type_ = sector.type_ or "???"
|
|
130
158
|
if last_airframe != (reg, type_):
|
|
@@ -145,10 +173,10 @@ def csv(duties: tuple[Duty, ...]) -> str:
|
|
|
145
173
|
extrasaction='ignore')
|
|
146
174
|
writer.writeheader()
|
|
147
175
|
for duty in duties:
|
|
148
|
-
crew = [CrewMember(clean_name(X[0]), X[1]) for X in duty.crew]
|
|
149
176
|
for sector in duty.sectors:
|
|
150
177
|
if sector.position or sector.quasi:
|
|
151
178
|
continue
|
|
179
|
+
crew = [CrewMember(clean_name(X[0]), X[1]) for X in sector.crew]
|
|
152
180
|
out_dict = {
|
|
153
181
|
'Off Blocks': sector.off,
|
|
154
182
|
'On Blocks': sector.on,
|
|
@@ -243,20 +271,18 @@ def _build_dict(duty: Duty) -> dict[str, str]:
|
|
|
243
271
|
return event
|
|
244
272
|
|
|
245
273
|
|
|
246
|
-
def ical(duties: tuple[Duty, ...],
|
|
274
|
+
def ical(duties: tuple[Duty, ...], ade: tuple[AllDayEvent, ...]) -> str:
|
|
247
275
|
events = []
|
|
248
276
|
for duty in duties:
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
date
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
day=date,
|
|
259
|
-
ev=duty.code,
|
|
277
|
+
d = _build_dict(duty)
|
|
278
|
+
events.append(vevent.format(**d))
|
|
279
|
+
for e in ade:
|
|
280
|
+
uid = "{}{}@HURSTS.ORG.UK".format(
|
|
281
|
+
e.date.isoformat(), e.code)
|
|
282
|
+
modified = ical_datetime.format(dt.datetime.utcnow())
|
|
283
|
+
events.append(advevent.format(
|
|
284
|
+
day=e.date,
|
|
285
|
+
ev=e.code,
|
|
260
286
|
modified=modified,
|
|
261
287
|
uid=uid))
|
|
262
288
|
return vcalendar.format("\r\n".join(events))
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from aims.data_structures import Duty, AllDayEvent, InputFileException
|
|
2
|
+
import aims.roster
|
|
3
|
+
import aims.logbook_report
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def parse(html: str) -> tuple[tuple[Duty, ...], tuple[AllDayEvent, ...]]:
|
|
7
|
+
# check it's an html5 file
|
|
8
|
+
html5_header = "<!DOCTYPE html><html>"
|
|
9
|
+
if html[:len(html5_header)] != html5_header:
|
|
10
|
+
raise InputFileException("HTML5 header not found.")
|
|
11
|
+
if html.find("Personal Crew Schedule Report") != -1:
|
|
12
|
+
return aims.roster.duties(html)
|
|
13
|
+
elif html.find("Pilot Logbook") != -1:
|
|
14
|
+
return aims.logbook_report.duties(html)
|
|
15
|
+
else:
|
|
16
|
+
raise InputFileException("Report type marker not found")
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""Processes the data extracted from a 'vertical' HTML AIMS roster.
|
|
2
|
+
|
|
3
|
+
The module's public interface is the duties() function, which accepts "soup" as
|
|
4
|
+
produced by processing the roster with BeautifulSoup.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import datetime as dt
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
from bs4 import BeautifulSoup # type: ignore
|
|
11
|
+
|
|
12
|
+
from aims.data_structures import (
|
|
13
|
+
Duty, Sector, CrewMember, AllDayEvent, InputFileException)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
Row = tuple[tuple[str, ...], ...]
|
|
17
|
+
|
|
18
|
+
# column indices
|
|
19
|
+
DATE, CODES, DETAILS, DSTART, TIMES, DEND, BHR, DHR, IND, CREW = range(1, 11)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _convert_datestring(in_: str) -> dt.date:
|
|
23
|
+
return dt.datetime.strptime(in_.split()[0], "%d/%m/%Y").date()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _convert_timestring(in_: str, date: dt.date) -> dt.datetime:
|
|
27
|
+
if in_[-2:] == "⁺¹":
|
|
28
|
+
in_ = in_[:-2]
|
|
29
|
+
date = date + dt.timedelta(1)
|
|
30
|
+
in_ = in_.replace("A", "").replace("E", "")
|
|
31
|
+
time = dt.datetime.strptime(in_, "%H:%M").time()
|
|
32
|
+
return dt.datetime.combine(date, time)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _crew(strings: tuple[str, ...]) -> tuple[CrewMember, ...]:
|
|
36
|
+
"""Convert crew cell to a tuple of CrewMember objects.
|
|
37
|
+
|
|
38
|
+
Each string may either represent a crew member or be a continuation of a
|
|
39
|
+
crew member's name if the details didn't all fit on one line. In the first
|
|
40
|
+
case the line will start with "CP -", "FO -" etc. The line is split on a
|
|
41
|
+
space if it is split.
|
|
42
|
+
|
|
43
|
+
For postioning crew, the string has the form "CP - PAX - 1234 - NAME MY".
|
|
44
|
+
Positioning crew should not be included in the crew list.
|
|
45
|
+
|
|
46
|
+
:param strings: The tuple of strings from the crew field of a duty record.
|
|
47
|
+
:return: A tuple of CrewMember objects.
|
|
48
|
+
|
|
49
|
+
"""
|
|
50
|
+
re_first = re.compile(r"[A-Z]{2} - ")
|
|
51
|
+
joined_strings: list[str] = []
|
|
52
|
+
for s in strings:
|
|
53
|
+
if re_first.match(s):
|
|
54
|
+
joined_strings.append(s)
|
|
55
|
+
else:
|
|
56
|
+
joined_strings[-1] += f" {s}"
|
|
57
|
+
crew: list[CrewMember] = []
|
|
58
|
+
for j in joined_strings:
|
|
59
|
+
fields = j.split(" - ")
|
|
60
|
+
if len(fields) < 3:
|
|
61
|
+
raise InputFileException("Bad crew block")
|
|
62
|
+
if fields[1] != "PAX":
|
|
63
|
+
crew.append(CrewMember(fields[-1], fields[0]))
|
|
64
|
+
return tuple(crew)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _sectors(data: Row, date: dt.date) -> tuple[Sector, ...]:
|
|
68
|
+
retval = []
|
|
69
|
+
for c, code in enumerate(data[CODES]):
|
|
70
|
+
code_split = code.split()
|
|
71
|
+
name = code_split[0]
|
|
72
|
+
type_ = None
|
|
73
|
+
if len(code_split) == 2 and code_split[1][0] == "[":
|
|
74
|
+
type_ = code_split[1][1:-1]
|
|
75
|
+
airports = [X.strip() for X in data[DETAILS][c].split(" - ")]
|
|
76
|
+
times = data[TIMES][c].split("/")[0].split(" - ")
|
|
77
|
+
crew = _crew(data[CREW])
|
|
78
|
+
if len(airports) == 2: # Not an all day event or unused standby
|
|
79
|
+
position = False
|
|
80
|
+
if airports[0][0] == "*": # Either ground or air positioning
|
|
81
|
+
airports[0] = airports[0][1:]
|
|
82
|
+
position = True
|
|
83
|
+
quasi = False
|
|
84
|
+
if not type_: # If no type in code, assume quasi sector
|
|
85
|
+
quasi = True
|
|
86
|
+
crew = ()
|
|
87
|
+
retval.append(
|
|
88
|
+
Sector(name, None, type_, airports[0], airports[1],
|
|
89
|
+
_convert_timestring(times[0], date),
|
|
90
|
+
_convert_timestring(times[1], date),
|
|
91
|
+
quasi, position, tuple(crew)))
|
|
92
|
+
else:
|
|
93
|
+
retval.append(
|
|
94
|
+
Sector(name, None, None, None, None,
|
|
95
|
+
_convert_timestring(times[0], date),
|
|
96
|
+
_convert_timestring(times[1], date),
|
|
97
|
+
True, False, tuple(crew)))
|
|
98
|
+
return tuple(retval)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _duty(row: Row) -> Duty:
|
|
102
|
+
"""Creates a Duty object from a structured representation of an HTML row.
|
|
103
|
+
|
|
104
|
+
The input object is a 12 cell tuple with each cell containing the contents
|
|
105
|
+
of a "td" from the original HTML row. The contents are in the form of a
|
|
106
|
+
variable length tuple of strings since the original contents of the cells
|
|
107
|
+
can be multi-line. The first and last items are empty tuples since the
|
|
108
|
+
original table has empty cells at the beginning and end of each row.
|
|
109
|
+
|
|
110
|
+
The items in the row, from index 1 to index 10 are:
|
|
111
|
+
|
|
112
|
+
1: Date of the form "01/01/2000 Mon", which may be split over two lines at
|
|
113
|
+
the space.
|
|
114
|
+
|
|
115
|
+
2: Either:
|
|
116
|
+
|
|
117
|
+
+ An empty tuple for an unpublished duty; or
|
|
118
|
+
|
|
119
|
+
+ For all day duties, the code for the all day duty; or
|
|
120
|
+
|
|
121
|
+
+ For normal sectors, the flight number plus aircraft type in square
|
|
122
|
+
brackets e.g. "1234 [320]", one for each sector in the duty; or
|
|
123
|
+
|
|
124
|
+
+ For quasi sectors, the code of the quasi sector (e.g. "ESBY", "ADTY",
|
|
125
|
+
"TAXI123"), one for each quasi sector in the duty. These can be mixed
|
|
126
|
+
in with normal sectors.
|
|
127
|
+
|
|
128
|
+
3: Either:
|
|
129
|
+
|
|
130
|
+
+ For all day duties, a textual description of the duty; or
|
|
131
|
+
|
|
132
|
+
+ For sectors, the airports invloved in the form "BRS - FNC", one for
|
|
133
|
+
each sector or quasi sector in the duty. For quasi sectors the two
|
|
134
|
+
airports may be the same (e.g. "LGW - LGW" for a sim) or, to indicate
|
|
135
|
+
postioning, may start with a "*" (e.g "*LGW - BRS" for a taxi ride).
|
|
136
|
+
|
|
137
|
+
4: Report time. This is not necessarily duty start time -- there may be
|
|
138
|
+
standby duties before report, and it is empty for standbys without a
|
|
139
|
+
call out.
|
|
140
|
+
|
|
141
|
+
5: Sector times of the form "11:00 - 13:00", one for each sector in the
|
|
142
|
+
duty, including for quasi sectors. For duties that have already taken
|
|
143
|
+
place, the fact that the times are actual times may be indicated with an
|
|
144
|
+
"A", giving the form "A11:00 - A13:00", although this doesn't seem to be
|
|
145
|
+
especially consistently done. The string for the last sector may also
|
|
146
|
+
incorporate the delay in the form "A11:00 - A13:00/01:00". For an all
|
|
147
|
+
day duty there are no times to record, and hence the cell will be an
|
|
148
|
+
empty tuple.
|
|
149
|
+
|
|
150
|
+
6: Duty end time. Empty for standby without callout.
|
|
151
|
+
|
|
152
|
+
7: Block hours
|
|
153
|
+
|
|
154
|
+
8: Duty hours
|
|
155
|
+
|
|
156
|
+
9: Markers for memos etc.
|
|
157
|
+
|
|
158
|
+
10: Crew list, one string per crew member. The crew is associated with the
|
|
159
|
+
entire duty rather than per sector.
|
|
160
|
+
|
|
161
|
+
:param row: A 12 cell tuple, with each cell a tuple of strings.
|
|
162
|
+
:return: The Duty object represented by the row.
|
|
163
|
+
|
|
164
|
+
"""
|
|
165
|
+
try:
|
|
166
|
+
assert row[CODES] and row[TIMES]
|
|
167
|
+
date = _convert_datestring(row[DATE][0])
|
|
168
|
+
if row[DSTART]:
|
|
169
|
+
assert row[DEND]
|
|
170
|
+
start = _convert_timestring(row[DSTART][0], date)
|
|
171
|
+
end = _convert_timestring(row[DEND][0], date)
|
|
172
|
+
else:
|
|
173
|
+
# If duty start/finish does not exist, take the times from
|
|
174
|
+
# the only sector.
|
|
175
|
+
assert len(row[TIMES]) == 1
|
|
176
|
+
times = row[TIMES][0].split(" - ")
|
|
177
|
+
start = _convert_timestring(times[0], date)
|
|
178
|
+
end = _convert_timestring(times[1], date)
|
|
179
|
+
sectors = _sectors(row, date)
|
|
180
|
+
# on duty can be before report if quasi sectors at start
|
|
181
|
+
for sector in sectors:
|
|
182
|
+
if not sector.quasi:
|
|
183
|
+
break
|
|
184
|
+
start = min(start, sector.off)
|
|
185
|
+
end = max(end, sector.on)
|
|
186
|
+
return Duty(start, end, sectors)
|
|
187
|
+
except (IndexError, ValueError):
|
|
188
|
+
raise InputFileException(f"Bad Duty Record: {str(row)}")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _ade(row: Row) -> AllDayEvent:
|
|
192
|
+
try:
|
|
193
|
+
return AllDayEvent(_convert_datestring(row[DATE][0]), row[CODES][0])
|
|
194
|
+
except (IndexError, ValueError):
|
|
195
|
+
raise InputFileException(f"Bad All Day Duty Record: {str(row)}")
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def duties(html: str) -> tuple[tuple[Duty, ...], tuple[AllDayEvent, ...]]:
|
|
199
|
+
"""Extract the data from an AIMS vertical roster.
|
|
200
|
+
|
|
201
|
+
The entire document is a single table (how retro!). The interesting part
|
|
202
|
+
starts with the row two rows below the row with a cell containing the
|
|
203
|
+
phrase "Schedule Details" and ends with the row above the first row with a
|
|
204
|
+
blank date field. Each row between these represents a single duty or an all
|
|
205
|
+
dat event.
|
|
206
|
+
|
|
207
|
+
The date to the left of each row is the date that the duty started on. If
|
|
208
|
+
duties continue past midnight, relevant times are marked with a superscript
|
|
209
|
+
'+1' rather than being pushed to the next row of the table. This makes the
|
|
210
|
+
vertical roster much saner to parse than the alternatives since there are
|
|
211
|
+
no concerns about missing data at the start and end of the roster period.
|
|
212
|
+
|
|
213
|
+
:param html: The html of a 'vertical' HTML AIMS roster.
|
|
214
|
+
:return: A tuple of Duty objects and a tuple of AllDayEvent objects
|
|
215
|
+
|
|
216
|
+
"""
|
|
217
|
+
soup = BeautifulSoup(html, "html5lib")
|
|
218
|
+
rows = iter(soup.find_all("tr"))
|
|
219
|
+
try:
|
|
220
|
+
while "Schedule Details" not in next(rows).stripped_strings:
|
|
221
|
+
pass
|
|
222
|
+
next(rows)
|
|
223
|
+
duty_list: list[Duty] = []
|
|
224
|
+
ade_list: list[AllDayEvent] = []
|
|
225
|
+
while True:
|
|
226
|
+
row: Row = tuple(
|
|
227
|
+
tuple(Y.replace("\xa0", " ") for Y in X.stripped_strings)
|
|
228
|
+
for X in next(rows)("td"))
|
|
229
|
+
if not row[DATE]: # line without date ends table
|
|
230
|
+
break
|
|
231
|
+
if not row[CODES]: # unpublished duty
|
|
232
|
+
continue
|
|
233
|
+
if not row[TIMES]: # an all day event
|
|
234
|
+
ade_list.append(_ade(row))
|
|
235
|
+
elif duty := _duty(row): # a normal duty
|
|
236
|
+
duty_list.append(duty)
|
|
237
|
+
return (tuple(duty_list), tuple(ade_list))
|
|
238
|
+
except (StopIteration, IndexError):
|
|
239
|
+
raise InputFileException("Duty table ended unexpectedly")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: aims-convert
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2
|
|
4
4
|
Summary: Extract useful information from AIMS
|
|
5
5
|
Home-page: https://github.com/JonHurst/aims-convert
|
|
6
6
|
Author: Jon Hurst
|
|
@@ -15,6 +15,8 @@ Description-Content-Type: text/markdown
|
|
|
15
15
|
Requires-Dist: nightflight
|
|
16
16
|
Requires-Dist: bs4
|
|
17
17
|
Requires-Dist: html5lib
|
|
18
|
+
Provides-Extra: unit-tests
|
|
19
|
+
Requires-Dist: freezegun; extra == "unit-tests"
|
|
18
20
|
|
|
19
21
|
# AIMS Roster Data Extraction #
|
|
20
22
|
|