pyopensky 2.0__tar.gz → 2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyopensky-2.0 → pyopensky-2.2}/PKG-INFO +10 -7
- {pyopensky-2.0 → pyopensky-2.2}/pyproject.toml +5 -2
- {pyopensky-2.0 → pyopensky-2.2}/readme.md +8 -6
- pyopensky-2.2/src/pyopensky/__init__.py +3 -0
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/api.py +1 -0
- pyopensky-2.2/src/pyopensky/config.py +219 -0
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/impala.py +19 -14
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/rest.py +8 -4
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/schema.py +19 -10
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/trino.py +17 -11
- pyopensky-2.0/src/pyopensky/__init__.py +0 -0
- pyopensky-2.0/src/pyopensky/config.py +0 -94
- {pyopensky-2.0 → pyopensky-2.2}/license.txt +0 -0
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/py.typed +0 -0
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/s3.py +0 -0
- {pyopensky-2.0 → pyopensky-2.2}/src/pyopensky/time.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pyopensky
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.2
|
|
4
4
|
Summary: A Python interface for OpenSky database
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Xavier Olive
|
|
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.8
|
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.9
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
16
|
Requires-Dist: appdirs (>=1.4.4)
|
|
16
17
|
Requires-Dist: minio (>=7.1)
|
|
17
18
|
Requires-Dist: pandas (>=1.5)
|
|
@@ -25,7 +26,9 @@ Description-Content-Type: text/markdown
|
|
|
25
26
|
|
|
26
27
|
# pyopensky
|
|
27
28
|
|
|
28
|
-
The pyopensky Python library provides
|
|
29
|
+
The `pyopensky` Python library provides functions to download data from the OpenSky Network live API and historical databases. It aims at making ADS-B and Mode S data from OpenSky easily accessible in the Python programming environment.
|
|
30
|
+
|
|
31
|
+
Full documentation on https://open-aviation.github.io/pyopensky
|
|
29
32
|
|
|
30
33
|
## Installation
|
|
31
34
|
|
|
@@ -54,7 +57,7 @@ poetry install
|
|
|
54
57
|
| Impala | https://opensky-network.org/data/impala |
|
|
55
58
|
| Trino | contact@opensky-network.org (or specify it in the Impala form) |
|
|
56
59
|
|
|
57
|
-
The first time you use the library, a configuration file named `
|
|
60
|
+
The first time you use the library, a configuration file named `settings.conf` is created, including the following content:
|
|
58
61
|
|
|
59
62
|
```text
|
|
60
63
|
[default]
|
|
@@ -62,7 +65,7 @@ username =
|
|
|
62
65
|
password =
|
|
63
66
|
```
|
|
64
67
|
|
|
65
|
-
You will identify the folder where the `
|
|
68
|
+
You will identify the folder where the `settings.conf` is located:
|
|
66
69
|
|
|
67
70
|
```python
|
|
68
71
|
from pyopensky.config import opensky_config_dir
|
|
@@ -72,7 +75,7 @@ print(opensky_config_dir)
|
|
|
72
75
|
|
|
73
76
|
## Usage
|
|
74
77
|
|
|
75
|
-
- from the REST API:
|
|
78
|
+
- from the [REST API](https://open-aviation.github.io/pyopensky/rest.html):
|
|
76
79
|
|
|
77
80
|
```python
|
|
78
81
|
from pyopensky.rest import REST
|
|
@@ -87,7 +90,7 @@ print(opensky_config_dir)
|
|
|
87
90
|
rest.departure(airport, begin, end)
|
|
88
91
|
```
|
|
89
92
|
|
|
90
|
-
- from the Impala shell (requires authentication):
|
|
93
|
+
- from the [Impala](https://open-aviation.github.io/pyopensky/impala.html) shell (requires authentication):
|
|
91
94
|
|
|
92
95
|
```python
|
|
93
96
|
from pyopensky.impala import Impala
|
|
@@ -99,7 +102,7 @@ print(opensky_config_dir)
|
|
|
99
102
|
impala.rawdata(start, stop, *, callsign, icao24, bounds)
|
|
100
103
|
```
|
|
101
104
|
|
|
102
|
-
- from the Trino
|
|
105
|
+
- from the [Trino](https://open-aviation.github.io/pyopensky/trino.html) database (requires authentication):
|
|
103
106
|
|
|
104
107
|
```python
|
|
105
108
|
from pyopensky.trino import Trino
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "pyopensky"
|
|
3
|
-
version = "2.
|
|
3
|
+
version = "2.2"
|
|
4
4
|
description = "A Python interface for OpenSky database"
|
|
5
5
|
authors = ["Xavier Olive <git@xoolive.org>", "Junzi Sun <j.sun-1@tudelft.nl>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -24,13 +24,16 @@ paramiko = ">=2.7"
|
|
|
24
24
|
|
|
25
25
|
[tool.poetry.group.dev.dependencies]
|
|
26
26
|
pre-commit = ">=3.1.1"
|
|
27
|
-
black = ">=23.1.0"
|
|
28
27
|
mypy = ">=1.0.1"
|
|
29
28
|
ruff = ">=0.0.253"
|
|
30
29
|
pytest = ">=7.2.1"
|
|
31
30
|
ipykernel = ">=6.21.2"
|
|
32
31
|
types-requests = ">=2.28.11.15"
|
|
33
32
|
types-paramiko = ">=3.3.0.0"
|
|
33
|
+
Sphinx = ">=5.1"
|
|
34
|
+
sphinx-design = ">=0.4.1"
|
|
35
|
+
sphinx-rtd-theme = ">=0.5.2"
|
|
36
|
+
sphinx-autodoc-typehints = ">=1.17,!=1.21.4"
|
|
34
37
|
|
|
35
38
|
[build-system]
|
|
36
39
|
requires = ["poetry-core"]
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# pyopensky
|
|
2
2
|
|
|
3
|
-
The pyopensky Python library provides
|
|
3
|
+
The `pyopensky` Python library provides functions to download data from the OpenSky Network live API and historical databases. It aims at making ADS-B and Mode S data from OpenSky easily accessible in the Python programming environment.
|
|
4
|
+
|
|
5
|
+
Full documentation on https://open-aviation.github.io/pyopensky
|
|
4
6
|
|
|
5
7
|
## Installation
|
|
6
8
|
|
|
@@ -29,7 +31,7 @@ poetry install
|
|
|
29
31
|
| Impala | https://opensky-network.org/data/impala |
|
|
30
32
|
| Trino | contact@opensky-network.org (or specify it in the Impala form) |
|
|
31
33
|
|
|
32
|
-
The first time you use the library, a configuration file named `
|
|
34
|
+
The first time you use the library, a configuration file named `settings.conf` is created, including the following content:
|
|
33
35
|
|
|
34
36
|
```text
|
|
35
37
|
[default]
|
|
@@ -37,7 +39,7 @@ username =
|
|
|
37
39
|
password =
|
|
38
40
|
```
|
|
39
41
|
|
|
40
|
-
You will identify the folder where the `
|
|
42
|
+
You will identify the folder where the `settings.conf` is located:
|
|
41
43
|
|
|
42
44
|
```python
|
|
43
45
|
from pyopensky.config import opensky_config_dir
|
|
@@ -47,7 +49,7 @@ print(opensky_config_dir)
|
|
|
47
49
|
|
|
48
50
|
## Usage
|
|
49
51
|
|
|
50
|
-
- from the REST API:
|
|
52
|
+
- from the [REST API](https://open-aviation.github.io/pyopensky/rest.html):
|
|
51
53
|
|
|
52
54
|
```python
|
|
53
55
|
from pyopensky.rest import REST
|
|
@@ -62,7 +64,7 @@ print(opensky_config_dir)
|
|
|
62
64
|
rest.departure(airport, begin, end)
|
|
63
65
|
```
|
|
64
66
|
|
|
65
|
-
- from the Impala shell (requires authentication):
|
|
67
|
+
- from the [Impala](https://open-aviation.github.io/pyopensky/impala.html) shell (requires authentication):
|
|
66
68
|
|
|
67
69
|
```python
|
|
68
70
|
from pyopensky.impala import Impala
|
|
@@ -74,7 +76,7 @@ print(opensky_config_dir)
|
|
|
74
76
|
impala.rawdata(start, stop, *, callsign, icao24, bounds)
|
|
75
77
|
```
|
|
76
78
|
|
|
77
|
-
- from the Trino
|
|
79
|
+
- from the [Trino](https://open-aviation.github.io/pyopensky/trino.html) database (requires authentication):
|
|
78
80
|
|
|
79
81
|
```python
|
|
80
82
|
from pyopensky.trino import Trino
|
|
@@ -50,6 +50,7 @@ class OpenSkyDBAPI(Protocol):
|
|
|
50
50
|
departure_airport: None | str = None,
|
|
51
51
|
arrival_airport: None | str = None,
|
|
52
52
|
airport: None | str = None,
|
|
53
|
+
time_buffer: None | str | pd.Timedelta = None,
|
|
53
54
|
cached: bool = True,
|
|
54
55
|
compress: bool = False,
|
|
55
56
|
limit: None | int = None,
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import configparser
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TypedDict
|
|
8
|
+
|
|
9
|
+
from appdirs import user_cache_dir, user_config_dir
|
|
10
|
+
|
|
11
|
+
import pandas as pd
|
|
12
|
+
|
|
13
|
+
_log = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
DEFAULT_CONFIG = """
|
|
17
|
+
[default]
|
|
18
|
+
username =
|
|
19
|
+
password =
|
|
20
|
+
|
|
21
|
+
# If you use different credentials for Impala and Trino, uncomment the following
|
|
22
|
+
# [impala]
|
|
23
|
+
# username =
|
|
24
|
+
# password =
|
|
25
|
+
#
|
|
26
|
+
# [trino]
|
|
27
|
+
# username =
|
|
28
|
+
# password =
|
|
29
|
+
|
|
30
|
+
[cache]
|
|
31
|
+
## You may set a different cache folder here if you have a preferred space
|
|
32
|
+
## for storing possibly large files.
|
|
33
|
+
## Default settings:
|
|
34
|
+
## - $HOME/.cache/opensky (Linux)
|
|
35
|
+
## - $HOME/Library/Caches/opensky (MacOS)
|
|
36
|
+
## - C:\\Users\\<username>\\AppData\\Local\\opensky\\Cache (Windows)
|
|
37
|
+
##
|
|
38
|
+
# path =
|
|
39
|
+
|
|
40
|
+
## The purge parameter refers to the cache folder containing data from the
|
|
41
|
+
## Opensky databases. Cache files tend to take space so you may want to let
|
|
42
|
+
## the library delete older files.
|
|
43
|
+
## Files are removed when the library is imported.
|
|
44
|
+
purge = 90 days
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
opensky_config_dir = Path(user_config_dir("pyopensky"))
|
|
48
|
+
opensky_config = configparser.ConfigParser()
|
|
49
|
+
|
|
50
|
+
if (opensky_config_file := opensky_config_dir / "settings.conf").exists():
|
|
51
|
+
opensky_config.read(opensky_config_file.as_posix())
|
|
52
|
+
elif (previous_config_file := opensky_config_dir / "secret.conf").exists():
|
|
53
|
+
previous_config_file.rename(opensky_config_file)
|
|
54
|
+
opensky_config.read(opensky_config_file.as_posix())
|
|
55
|
+
else:
|
|
56
|
+
if not opensky_config_dir.exists():
|
|
57
|
+
opensky_config_dir.mkdir(parents=True)
|
|
58
|
+
opensky_config_file.write_text(DEFAULT_CONFIG)
|
|
59
|
+
|
|
60
|
+
traffic_config_dir = Path(user_config_dir("traffic"))
|
|
61
|
+
traffic_config = configparser.ConfigParser()
|
|
62
|
+
|
|
63
|
+
if (traffic_config_file := traffic_config_dir / "traffic.conf").exists():
|
|
64
|
+
traffic_config.read(traffic_config_file.as_posix())
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def purge_cache(cache_path: Path) -> None:
|
|
68
|
+
cache_no_expire = False
|
|
69
|
+
cache_no_expire |= bool(os.environ.get("TRAFFIC_CACHE_NO_EXPIRE"))
|
|
70
|
+
cache_no_expire |= bool(os.environ.get("OPENSKY_CACHE_NO_EXPIRE"))
|
|
71
|
+
|
|
72
|
+
cache_purge = "90 days"
|
|
73
|
+
if purge := traffic_config.get("cache", "purge", fallback=None):
|
|
74
|
+
if purge.strip() != "":
|
|
75
|
+
cache_purge = purge
|
|
76
|
+
if purge := opensky_config.get("cache", "purge", fallback=None):
|
|
77
|
+
if purge.strip() != "":
|
|
78
|
+
cache_purge = purge
|
|
79
|
+
|
|
80
|
+
if not cache_no_expire: # coverage: ignore
|
|
81
|
+
expiration = pd.Timestamp("now") - pd.Timedelta(cache_purge)
|
|
82
|
+
|
|
83
|
+
for cache_file in cache_path.glob("*"):
|
|
84
|
+
ctime = cache_file.stat().st_ctime
|
|
85
|
+
if ctime < expiration.timestamp():
|
|
86
|
+
_log.warn(f"Removing {cache_file} created on {ctime}")
|
|
87
|
+
cache_file.unlink()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class Resolution(TypedDict, total=False):
|
|
91
|
+
opensky_category: str
|
|
92
|
+
opensky_name: str
|
|
93
|
+
traffic_category: str
|
|
94
|
+
traffic_name: str
|
|
95
|
+
environment_variable: str
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
NAME_RESOLUTION: dict[str, Resolution] = {
|
|
99
|
+
# Cache configuration
|
|
100
|
+
"cache_dir": dict(
|
|
101
|
+
opensky_category="cache",
|
|
102
|
+
opensky_name="path",
|
|
103
|
+
traffic_category="cache",
|
|
104
|
+
traffic_name="path",
|
|
105
|
+
),
|
|
106
|
+
# Network configuration
|
|
107
|
+
"http_proxy": dict(
|
|
108
|
+
environment_variable="http_proxy",
|
|
109
|
+
opensky_category="default",
|
|
110
|
+
opensky_name="http_proxy",
|
|
111
|
+
traffic_category="network",
|
|
112
|
+
traffic_name="http_proxy",
|
|
113
|
+
),
|
|
114
|
+
"ssh_proxycommand": dict(
|
|
115
|
+
opensky_category="impala",
|
|
116
|
+
opensky_name="ssh.proxycommand",
|
|
117
|
+
traffic_category="opensky",
|
|
118
|
+
traffic_name="ssh.proxycommand",
|
|
119
|
+
),
|
|
120
|
+
# Credentials configuration
|
|
121
|
+
"access_key": dict(
|
|
122
|
+
opensky_category="s3",
|
|
123
|
+
opensky_name="access_key",
|
|
124
|
+
traffic_category="opensky",
|
|
125
|
+
traffic_name="access_key",
|
|
126
|
+
),
|
|
127
|
+
"secret_key": dict(
|
|
128
|
+
opensky_category="s3",
|
|
129
|
+
opensky_name="secret_key",
|
|
130
|
+
traffic_category="opensky",
|
|
131
|
+
traffic_name="secret_key",
|
|
132
|
+
),
|
|
133
|
+
"username": dict(
|
|
134
|
+
environment_variable="OPENSKY_USERNAME",
|
|
135
|
+
opensky_category="default",
|
|
136
|
+
opensky_name="username",
|
|
137
|
+
traffic_category="opensky",
|
|
138
|
+
traffic_name="username",
|
|
139
|
+
),
|
|
140
|
+
"password": dict(
|
|
141
|
+
environment_variable="OPENSKY_PASSWORD",
|
|
142
|
+
opensky_category="default",
|
|
143
|
+
opensky_name="password",
|
|
144
|
+
traffic_category="opensky",
|
|
145
|
+
traffic_name="password",
|
|
146
|
+
),
|
|
147
|
+
"trino_username": dict(opensky_category="trino", opensky_name="username"),
|
|
148
|
+
"trino_password": dict(opensky_category="trino", opensky_name="password"),
|
|
149
|
+
"impala_username": dict(opensky_category="impala", opensky_name="username"),
|
|
150
|
+
"impala_password": dict(opensky_category="impala", opensky_name="password"),
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
__all__ = list(NAME_RESOLUTION.keys())
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def get_config(
|
|
157
|
+
opensky_category: str,
|
|
158
|
+
opensky_name: str,
|
|
159
|
+
traffic_category: None | str = None,
|
|
160
|
+
traffic_name: None | str = None,
|
|
161
|
+
environment_variable: None | str = None,
|
|
162
|
+
) -> None | str:
|
|
163
|
+
if opensky_value := opensky_config.get(
|
|
164
|
+
opensky_category, opensky_name, fallback=None
|
|
165
|
+
):
|
|
166
|
+
return opensky_value
|
|
167
|
+
|
|
168
|
+
if traffic_category is not None and traffic_name is not None:
|
|
169
|
+
if traffic_value := traffic_config.get(
|
|
170
|
+
traffic_category, traffic_name, fallback=None
|
|
171
|
+
):
|
|
172
|
+
return traffic_value
|
|
173
|
+
|
|
174
|
+
if environment_variable is not None:
|
|
175
|
+
return os.environ.get(environment_variable)
|
|
176
|
+
|
|
177
|
+
return None
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
cache_dir = get_config(**NAME_RESOLUTION["cache_dir"])
|
|
181
|
+
if cache_dir is None:
|
|
182
|
+
cache_dir = user_cache_dir("opensky")
|
|
183
|
+
cache_path = Path(cache_dir)
|
|
184
|
+
if not cache_path.exists():
|
|
185
|
+
cache_path.mkdir(parents=True)
|
|
186
|
+
|
|
187
|
+
purge_cache(cache_path)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def __getattr__(name: str) -> None | str:
|
|
191
|
+
# Pick in order:
|
|
192
|
+
# 1. pyopensky -> settings.conf
|
|
193
|
+
# 2. traffic -> traffic.conf
|
|
194
|
+
# 3. environment variables
|
|
195
|
+
|
|
196
|
+
if name == "trino_username":
|
|
197
|
+
if value := get_config(**NAME_RESOLUTION[name]):
|
|
198
|
+
return value
|
|
199
|
+
return get_config(**NAME_RESOLUTION["username"])
|
|
200
|
+
|
|
201
|
+
if name == "trino_password":
|
|
202
|
+
if value := get_config(**NAME_RESOLUTION[name]):
|
|
203
|
+
return value
|
|
204
|
+
return get_config(**NAME_RESOLUTION["password"])
|
|
205
|
+
|
|
206
|
+
if name == "impala_username":
|
|
207
|
+
if value := get_config(**NAME_RESOLUTION[name]):
|
|
208
|
+
return value
|
|
209
|
+
return get_config(**NAME_RESOLUTION["username"])
|
|
210
|
+
|
|
211
|
+
if name == "impala_password":
|
|
212
|
+
if value := get_config(**NAME_RESOLUTION[name]):
|
|
213
|
+
return value
|
|
214
|
+
return get_config(**NAME_RESOLUTION["password"])
|
|
215
|
+
|
|
216
|
+
if name in __all__:
|
|
217
|
+
return get_config(**NAME_RESOLUTION[name])
|
|
218
|
+
|
|
219
|
+
raise AttributeError
|
|
@@ -20,7 +20,12 @@ import pandas as pd
|
|
|
20
20
|
from pandas.errors import ParserError
|
|
21
21
|
|
|
22
22
|
from .api import HasBounds, OpenSkyDBAPI, ProgressbarType
|
|
23
|
-
from .config import
|
|
23
|
+
from .config import (
|
|
24
|
+
cache_path,
|
|
25
|
+
impala_password,
|
|
26
|
+
impala_username,
|
|
27
|
+
ssh_proxycommand,
|
|
28
|
+
)
|
|
24
29
|
from .time import split_times, timelike, to_datetime
|
|
25
30
|
|
|
26
31
|
_log = logging.getLogger(__name__)
|
|
@@ -137,21 +142,21 @@ class Impala(OpenSkyDBAPI):
|
|
|
137
142
|
# actually ChannelStderrFile
|
|
138
143
|
|
|
139
144
|
def __init__(self, **kwargs: Any) -> None:
|
|
140
|
-
if
|
|
145
|
+
if impala_username is None or impala_password is None:
|
|
141
146
|
_log.warn("No credentials provided")
|
|
142
147
|
|
|
143
|
-
self.username =
|
|
144
|
-
self.password =
|
|
148
|
+
self.username = impala_username
|
|
149
|
+
self.password = impala_password
|
|
145
150
|
self.proxy_command = ssh_proxycommand
|
|
146
151
|
self.connected = False
|
|
147
152
|
self.cache_dir = cache_path
|
|
148
153
|
if not self.cache_dir.exists():
|
|
149
154
|
self.cache_dir.mkdir(parents=True)
|
|
150
155
|
|
|
151
|
-
if
|
|
156
|
+
if impala_username == "" or impala_password == "":
|
|
152
157
|
self.auth = None
|
|
153
158
|
else:
|
|
154
|
-
self.auth = (
|
|
159
|
+
self.auth = (impala_username, impala_password)
|
|
155
160
|
|
|
156
161
|
def clear_cache(self) -> None: # coverage: ignore
|
|
157
162
|
"""Clear cache files for OpenSky.
|
|
@@ -651,12 +656,12 @@ class Impala(OpenSkyDBAPI):
|
|
|
651
656
|
departure_airport: None | str = None,
|
|
652
657
|
arrival_airport: None | str = None,
|
|
653
658
|
airport: None | str = None,
|
|
659
|
+
time_buffer: None | str | pd.Timedelta = None,
|
|
654
660
|
cached: bool = True,
|
|
655
661
|
compress: bool = False,
|
|
656
662
|
limit: None | int = None,
|
|
657
663
|
other_tables: str = "",
|
|
658
664
|
other_params: str = "",
|
|
659
|
-
time_buffer: None | str | pd.Timedelta = None,
|
|
660
665
|
progressbar: bool | ProgressbarType[Any] = True,
|
|
661
666
|
date_delta: timedelta = timedelta(hours=1),
|
|
662
667
|
count: bool = False,
|
|
@@ -706,6 +711,10 @@ class Impala(OpenSkyDBAPI):
|
|
|
706
711
|
:param airport: a string for the ICAO identifier of the airport. Selects
|
|
707
712
|
flights departing from or arriving at the airport between the two
|
|
708
713
|
timestamps;
|
|
714
|
+
:param time_buffer: (default: None) time buffer used to extend time
|
|
715
|
+
bounds for flights in the OpenSky flight tables: requests will get
|
|
716
|
+
flights between ``start - time_buffer`` and ``stop + time_buffer``.
|
|
717
|
+
If no airport is specified, the parameter is ignored.
|
|
709
718
|
|
|
710
719
|
.. warning::
|
|
711
720
|
|
|
@@ -723,10 +732,6 @@ class Impala(OpenSkyDBAPI):
|
|
|
723
732
|
received each record;
|
|
724
733
|
:param nautical_units: (default: True) convert data stored in Impala to
|
|
725
734
|
standard nautical units (ft, ft/min, knots).
|
|
726
|
-
:param time_buffer: (default: None) time buffer used to extend time
|
|
727
|
-
bounds for flights in the OpenSky flight tables: requests will get
|
|
728
|
-
flights between ``start - time_buffer`` and ``stop + time_buffer``.
|
|
729
|
-
If no airport is specified, the parameter is ignored.
|
|
730
735
|
:param cached: (default: True) switch to False to force a new request to
|
|
731
736
|
the database regardless of the cached files. This option also
|
|
732
737
|
deletes previous cache files;
|
|
@@ -799,7 +804,7 @@ class Impala(OpenSkyDBAPI):
|
|
|
799
804
|
else:
|
|
800
805
|
other_params += f"and RTRIM(callsign) REGEXP('{callsign}') "
|
|
801
806
|
|
|
802
|
-
elif callsign.find("%")
|
|
807
|
+
elif callsign.find("%") >= 0 or callsign.find("_") >= 0:
|
|
803
808
|
other_params += "and callsign ilike '{}' ".format(callsign)
|
|
804
809
|
else:
|
|
805
810
|
other_params += "and callsign='{:<8s}' ".format(callsign)
|
|
@@ -1005,7 +1010,7 @@ class Impala(OpenSkyDBAPI):
|
|
|
1005
1010
|
other_params += "and not typeogn "
|
|
1006
1011
|
|
|
1007
1012
|
if isinstance(sensor_name, str):
|
|
1008
|
-
if sensor_name.find("%") >
|
|
1013
|
+
if sensor_name.find("%") > -1 or sensor_name.find("_") > -1:
|
|
1009
1014
|
other_params += "and sensorname ilike '{}' ".format(sensor_name)
|
|
1010
1015
|
else:
|
|
1011
1016
|
other_params += "and sensorname='{:<8s}' ".format(sensor_name)
|
|
@@ -1236,7 +1241,7 @@ class Impala(OpenSkyDBAPI):
|
|
|
1236
1241
|
"supported at the moment."
|
|
1237
1242
|
)
|
|
1238
1243
|
if isinstance(callsign, str):
|
|
1239
|
-
if callsign.find("%") >
|
|
1244
|
+
if callsign.find("%") > -1 or callsign.find("_") > -1:
|
|
1240
1245
|
callsigns = "and callsign ilike '{}' ".format(callsign)
|
|
1241
1246
|
else:
|
|
1242
1247
|
callsigns = "and callsign='{:<8s}' ".format(callsign)
|
|
@@ -64,6 +64,7 @@ class REST:
|
|
|
64
64
|
| str
|
|
65
65
|
| HasBounds
|
|
66
66
|
| tuple[float, float, float, float] = None,
|
|
67
|
+
retry: int = 5,
|
|
67
68
|
) -> pd.DataFrame:
|
|
68
69
|
"""Returns the current state vectors from OpenSky REST API.
|
|
69
70
|
|
|
@@ -132,9 +133,12 @@ class REST:
|
|
|
132
133
|
columns.append("_")
|
|
133
134
|
r = pd.DataFrame.from_records(json["states"], columns=columns)
|
|
134
135
|
except Exception:
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
136
|
+
if retry > 0:
|
|
137
|
+
retry = retry - 1
|
|
138
|
+
_log.warning("Retrying in 10 seconds... {retry} more time")
|
|
139
|
+
time.sleep(10)
|
|
140
|
+
return self.states(own, bounds, retry=retry)
|
|
141
|
+
raise
|
|
138
142
|
|
|
139
143
|
return r.assign(
|
|
140
144
|
timestamp=lambda df: pd.to_datetime(
|
|
@@ -210,7 +214,7 @@ class REST:
|
|
|
210
214
|
c.raise_for_status()
|
|
211
215
|
json = c.json()
|
|
212
216
|
|
|
213
|
-
return tuple(json["route"])
|
|
217
|
+
return tuple(json["route"])
|
|
214
218
|
|
|
215
219
|
def aircraft(
|
|
216
220
|
self,
|
|
@@ -31,6 +31,8 @@ class UTCTimestampInteger(TypeDecorator[pd.Timestamp]):
|
|
|
31
31
|
value = pd.to_datetime(value, utc=True)
|
|
32
32
|
if isinstance(value, pd.Timestamp):
|
|
33
33
|
return int(value.timestamp())
|
|
34
|
+
if isinstance(value, pd.Timedelta):
|
|
35
|
+
return int(value.total_seconds())
|
|
34
36
|
return super().process_bind_param(value, dialect)
|
|
35
37
|
|
|
36
38
|
def process_result_value(
|
|
@@ -83,6 +85,7 @@ class CallsignString(TypeDecorator[str]):
|
|
|
83
85
|
operator.ne, # !=
|
|
84
86
|
sql_operators.eq,
|
|
85
87
|
sql_operators.ne,
|
|
88
|
+
sql_operators.in_op,
|
|
86
89
|
sql_operators.contains, # .in()
|
|
87
90
|
):
|
|
88
91
|
return self
|
|
@@ -408,16 +411,22 @@ class PositionData4(Base):
|
|
|
408
411
|
maxtime: Mapped[pd.Timestamp] = mapped_column(UTCTimestampFloat)
|
|
409
412
|
msgcount: Mapped[int]
|
|
410
413
|
icao24: Mapped[str]
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
414
|
+
nicsuppla: Mapped[bool]
|
|
415
|
+
hcr: Mapped[float]
|
|
416
|
+
nic: Mapped[int]
|
|
417
|
+
survstatus: Mapped[int]
|
|
418
|
+
nicsupplb: Mapped[bool]
|
|
419
|
+
odd: Mapped[bool]
|
|
420
|
+
baroalt: Mapped[bool]
|
|
421
|
+
lat: Mapped[float]
|
|
422
|
+
lon: Mapped[float]
|
|
423
|
+
alt: Mapped[float]
|
|
424
|
+
nicsupplc: Mapped[bool]
|
|
425
|
+
groundspeed: Mapped[float]
|
|
426
|
+
gsresolution: Mapped[float]
|
|
427
|
+
heading: Mapped[float]
|
|
428
|
+
timeflag: Mapped[bool]
|
|
429
|
+
surface: Mapped[bool]
|
|
421
430
|
|
|
422
431
|
# Whatever, we pick this one as primary key BUT
|
|
423
432
|
# - this is not true
|
|
@@ -22,6 +22,7 @@ from sqlalchemy import (
|
|
|
22
22
|
)
|
|
23
23
|
from sqlalchemy.orm import aliased
|
|
24
24
|
from sqlalchemy.orm.attributes import InstrumentedAttribute
|
|
25
|
+
from sqlalchemy.sql import ColumnExpressionArgument
|
|
25
26
|
from sqlalchemy.sql.expression import text
|
|
26
27
|
from tqdm import tqdm
|
|
27
28
|
from trino.auth import JWTAuthentication, OAuth2Authentication
|
|
@@ -30,7 +31,7 @@ from trino.sqlalchemy import URL
|
|
|
30
31
|
import pandas as pd
|
|
31
32
|
|
|
32
33
|
from .api import HasBounds, OpenSkyDBAPI
|
|
33
|
-
from .config import cache_path,
|
|
34
|
+
from .config import cache_path, trino_password, trino_username
|
|
34
35
|
from .schema import (
|
|
35
36
|
FlightsData4,
|
|
36
37
|
FlightsData5,
|
|
@@ -60,7 +61,7 @@ class Trino(OpenSkyDBAPI):
|
|
|
60
61
|
_token: None | Token = None
|
|
61
62
|
|
|
62
63
|
def token(self, **kwargs: Any) -> None | str:
|
|
63
|
-
if
|
|
64
|
+
if trino_username is None or trino_password is None:
|
|
64
65
|
_log.warn(
|
|
65
66
|
"No credentials provided, "
|
|
66
67
|
"falling back to browser authentication"
|
|
@@ -81,8 +82,8 @@ class Trino(OpenSkyDBAPI):
|
|
|
81
82
|
data={
|
|
82
83
|
"client_id": "trino-client",
|
|
83
84
|
"grant_type": "password",
|
|
84
|
-
"username":
|
|
85
|
-
"password":
|
|
85
|
+
"username": trino_username,
|
|
86
|
+
"password": trino_password,
|
|
86
87
|
},
|
|
87
88
|
**kwargs,
|
|
88
89
|
)
|
|
@@ -108,7 +109,7 @@ class Trino(OpenSkyDBAPI):
|
|
|
108
109
|
URL(
|
|
109
110
|
"trino.opensky-network.org",
|
|
110
111
|
port=443,
|
|
111
|
-
user=
|
|
112
|
+
user=trino_username,
|
|
112
113
|
catalog="minio",
|
|
113
114
|
schema="osky",
|
|
114
115
|
),
|
|
@@ -224,7 +225,7 @@ class Trino(OpenSkyDBAPI):
|
|
|
224
225
|
self,
|
|
225
226
|
start: timelike,
|
|
226
227
|
stop: None | timelike = None,
|
|
227
|
-
*args:
|
|
228
|
+
*args: ColumnExpressionArgument[bool],
|
|
228
229
|
departure_airport: None | str | list[str] = None,
|
|
229
230
|
arrival_airport: None | str | list[str] = None,
|
|
230
231
|
airport: None | str | list[str] = None,
|
|
@@ -345,6 +346,9 @@ class Trino(OpenSkyDBAPI):
|
|
|
345
346
|
Table.day < stop_ts.ceil("1d"),
|
|
346
347
|
)
|
|
347
348
|
|
|
349
|
+
for condition in args:
|
|
350
|
+
stmt = stmt.where(condition)
|
|
351
|
+
|
|
348
352
|
if limit is not None:
|
|
349
353
|
stmt = stmt.limit(limit)
|
|
350
354
|
|
|
@@ -364,7 +368,7 @@ class Trino(OpenSkyDBAPI):
|
|
|
364
368
|
self,
|
|
365
369
|
start: timelike,
|
|
366
370
|
stop: None | timelike = None,
|
|
367
|
-
*args:
|
|
371
|
+
*args: ColumnExpressionArgument[bool],
|
|
368
372
|
# date_delta: timedelta = timedelta(hours=1),
|
|
369
373
|
callsign: None | str | list[str] = None,
|
|
370
374
|
icao24: None | str | list[str] = None,
|
|
@@ -581,7 +585,7 @@ class Trino(OpenSkyDBAPI):
|
|
|
581
585
|
self,
|
|
582
586
|
start: timelike,
|
|
583
587
|
stop: None | timelike = None,
|
|
584
|
-
*args:
|
|
588
|
+
*args: ColumnExpressionArgument[bool],
|
|
585
589
|
icao24: None | str | list[str] = None,
|
|
586
590
|
serials: None | int | Iterable[int] = None,
|
|
587
591
|
bounds: None | HasBounds | tuple[float, float, float, float] = None,
|
|
@@ -593,6 +597,7 @@ class Trino(OpenSkyDBAPI):
|
|
|
593
597
|
compress: bool = False,
|
|
594
598
|
limit: None | int = None,
|
|
595
599
|
Table: Type[RawTable] = RollcallRepliesData4,
|
|
600
|
+
extra_columns: tuple[InstrumentedAttribute[Any], ...] = (),
|
|
596
601
|
**kwargs: Any,
|
|
597
602
|
) -> None | pd.DataFrame:
|
|
598
603
|
"""Get raw message from the OpenSky Trino database.
|
|
@@ -687,11 +692,9 @@ class Trino(OpenSkyDBAPI):
|
|
|
687
692
|
select(Table)
|
|
688
693
|
.with_only_columns(
|
|
689
694
|
Table.mintime,
|
|
690
|
-
Table.maxtime,
|
|
691
695
|
Table.rawmsg,
|
|
692
|
-
Table.msgcount,
|
|
693
696
|
Table.icao24,
|
|
694
|
-
|
|
697
|
+
*extra_columns,
|
|
695
698
|
)
|
|
696
699
|
.where(Table.rawmsg.is_not(None))
|
|
697
700
|
)
|
|
@@ -807,6 +810,9 @@ class Trino(OpenSkyDBAPI):
|
|
|
807
810
|
)
|
|
808
811
|
)
|
|
809
812
|
|
|
813
|
+
for condition in args:
|
|
814
|
+
stmt = stmt.where(condition)
|
|
815
|
+
|
|
810
816
|
stmt = self.stmt_where_str(stmt, icao24, Table.icao24)
|
|
811
817
|
|
|
812
818
|
stmt = stmt.where(
|
|
File without changes
|
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import configparser
|
|
4
|
-
import logging
|
|
5
|
-
import os
|
|
6
|
-
from pathlib import Path
|
|
7
|
-
|
|
8
|
-
from appdirs import user_cache_dir, user_config_dir
|
|
9
|
-
|
|
10
|
-
import pandas as pd
|
|
11
|
-
|
|
12
|
-
_log = logging.getLogger(__name__)
|
|
13
|
-
|
|
14
|
-
__all__ = [ # noqa: F822
|
|
15
|
-
"access_key",
|
|
16
|
-
"secret_key",
|
|
17
|
-
"username",
|
|
18
|
-
"password",
|
|
19
|
-
"http_proxy",
|
|
20
|
-
"cache_path",
|
|
21
|
-
"ssh_proxycommand",
|
|
22
|
-
"ssh.proxycommand",
|
|
23
|
-
]
|
|
24
|
-
|
|
25
|
-
DEFAULT_CONFIG = """
|
|
26
|
-
[default]
|
|
27
|
-
username =
|
|
28
|
-
password =
|
|
29
|
-
"""
|
|
30
|
-
|
|
31
|
-
traffic_config_dir = Path(user_config_dir("traffic"))
|
|
32
|
-
opensky_config_dir = Path(user_config_dir("pyopensky"))
|
|
33
|
-
|
|
34
|
-
traffic_config = configparser.ConfigParser()
|
|
35
|
-
if (traffic_config_file := (traffic_config_dir / "traffic.conf")).exists():
|
|
36
|
-
traffic_config.read(traffic_config_file.as_posix())
|
|
37
|
-
|
|
38
|
-
opensky_config = configparser.ConfigParser()
|
|
39
|
-
if (opensky_config_file := opensky_config_dir / "secret.conf").exists():
|
|
40
|
-
opensky_config.read(opensky_config_file.as_posix())
|
|
41
|
-
else:
|
|
42
|
-
opensky_config_dir.mkdir(parents=True)
|
|
43
|
-
opensky_config_file.write_text(DEFAULT_CONFIG)
|
|
44
|
-
|
|
45
|
-
cache_dir = user_cache_dir("opensky")
|
|
46
|
-
cache_path = Path(cache_dir)
|
|
47
|
-
if not cache_path.exists():
|
|
48
|
-
cache_path.mkdir(parents=True)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
cache_purge = traffic_config.get("cache", "purge", fallback="90 days")
|
|
52
|
-
cache_no_expire = bool(os.environ.get("TRAFFIC_CACHE_NO_EXPIRE"))
|
|
53
|
-
|
|
54
|
-
if cache_purge != "" and not cache_no_expire: # coverage: ignore
|
|
55
|
-
expiration = pd.Timestamp("now") - pd.Timedelta(cache_purge)
|
|
56
|
-
|
|
57
|
-
for cache_file in cache_path.glob("*"):
|
|
58
|
-
ctime = cache_file.stat().st_ctime
|
|
59
|
-
if ctime < expiration.timestamp():
|
|
60
|
-
_log.warn(f"Removing {cache_file} created on {ctime}")
|
|
61
|
-
cache_file.unlink()
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def __getattr__(name: str) -> None | str:
|
|
65
|
-
# Pick in order:
|
|
66
|
-
# 1. traffic.conf (traffic)
|
|
67
|
-
# 2. secret.conf (pyopensky)
|
|
68
|
-
# 3. environment variables
|
|
69
|
-
|
|
70
|
-
traffic_section = "opensky"
|
|
71
|
-
|
|
72
|
-
if name == "http_proxy":
|
|
73
|
-
traffic_section = "network"
|
|
74
|
-
|
|
75
|
-
if name == "ssh_proxycommand":
|
|
76
|
-
name = "ssh.proxycommand"
|
|
77
|
-
traffic_section = "network"
|
|
78
|
-
|
|
79
|
-
if option := traffic_config.get(traffic_section, name, fallback=None):
|
|
80
|
-
return option
|
|
81
|
-
|
|
82
|
-
if option := opensky_config.get("default", name, fallback=None):
|
|
83
|
-
return option
|
|
84
|
-
|
|
85
|
-
if name in ["username", "password"]:
|
|
86
|
-
return os.environ.get(f"OPENSKY_{name.upper()}", None)
|
|
87
|
-
|
|
88
|
-
if name in ["http_proxy"]:
|
|
89
|
-
return os.environ.get(name)
|
|
90
|
-
|
|
91
|
-
if name in __all__:
|
|
92
|
-
return None
|
|
93
|
-
|
|
94
|
-
raise AttributeError
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|