oonidata 0.3.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oonidata/__about__.py +1 -0
- oonidata/__init__.py +0 -0
- oonidata/apiclient.py +31 -0
- oonidata/cli/__init__.py +1 -0
- oonidata/cli/command.py +117 -0
- oonidata/compat.py +30 -0
- oonidata/dataclient.py +836 -0
- oonidata/datautils.py +454 -0
- oonidata/legacy/normalize_yamlooni.py +606 -0
- oonidata/main.py +9 -0
- oonidata/models/__init__.py +0 -0
- oonidata/models/analysis.py +132 -0
- oonidata/models/base.py +37 -0
- oonidata/models/dataformats.py +367 -0
- oonidata/models/experiment_result.py +304 -0
- oonidata/models/nettests/__init__.py +50 -0
- oonidata/models/nettests/base_measurement.py +47 -0
- oonidata/models/nettests/browser_web.py +24 -0
- oonidata/models/nettests/dnscheck.py +41 -0
- oonidata/models/nettests/facebook_messenger.py +45 -0
- oonidata/models/nettests/http_header_field_manipulation.py +39 -0
- oonidata/models/nettests/http_invalid_request_line.py +23 -0
- oonidata/models/nettests/signal.py +37 -0
- oonidata/models/nettests/stun_reachability.py +28 -0
- oonidata/models/nettests/telegram.py +39 -0
- oonidata/models/nettests/tor.py +45 -0
- oonidata/models/nettests/urlgetter.py +36 -0
- oonidata/models/nettests/web_connectivity.py +109 -0
- oonidata/models/nettests/whatsapp.py +43 -0
- oonidata/models/observations.py +385 -0
- oonidata-0.3.0.dist-info/METADATA +17 -0
- oonidata-0.3.0.dist-info/RECORD +33 -0
- oonidata-0.3.0.dist-info/WHEEL +5 -0
oonidata/__about__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
VERSION = "0.3.0"
|
oonidata/__init__.py
ADDED
|
File without changes
|
oonidata/apiclient.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Optional
|
|
3
|
+
import orjson
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def get_raw_measurement(report_id: str, input: Optional[str] = None) -> bytes:
|
|
8
|
+
params = params = {"report_id": report_id, "full": True}
|
|
9
|
+
if input:
|
|
10
|
+
params["input"] = input
|
|
11
|
+
r = requests.get("https://api.ooni.io/api/v1/measurement_meta", params=params)
|
|
12
|
+
j = r.json()
|
|
13
|
+
return j["raw_measurement"].encode("utf-8")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_measurement_dict(report_id: str, input: Optional[str] = None) -> dict:
|
|
17
|
+
params = params = {"report_id": report_id, "full": True}
|
|
18
|
+
if input:
|
|
19
|
+
params["input"] = input
|
|
20
|
+
r = requests.get("https://api.ooni.io/api/v1/measurement_meta", params=params)
|
|
21
|
+
j = r.json()
|
|
22
|
+
msmt_dict = orjson.loads(j["raw_measurement"])
|
|
23
|
+
msmt_dict["measurement_uid"] = j["measurement_uid"]
|
|
24
|
+
return msmt_dict
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def get_measurement_dict_by_uid(measurement_uid: str) -> dict:
|
|
28
|
+
r = requests.get(f"https://api.ooni.io/api/v1/measurement/{measurement_uid}")
|
|
29
|
+
msmt_dict = r.json()
|
|
30
|
+
msmt_dict["measurement_uid"] = measurement_uid
|
|
31
|
+
return msmt_dict
|
oonidata/cli/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .command import cli
|
oonidata/cli/command.py
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
from datetime import date, timedelta, datetime
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
from click_loglevel import LogLevel
|
|
10
|
+
|
|
11
|
+
from oonidata.__about__ import VERSION
|
|
12
|
+
from oonidata.dataclient import (
|
|
13
|
+
sync_measurements,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
log = logging.getLogger("oonidata")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _parse_date(ctx, param, date_str: str) -> date:
|
|
20
|
+
return datetime.strptime(date_str, "%Y-%m-%d").date()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _parse_csv(ctx, param, s: Optional[str]) -> List[str]:
|
|
24
|
+
if s:
|
|
25
|
+
return s.split(",")
|
|
26
|
+
return []
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
probe_cc_option = click.option(
|
|
30
|
+
"--probe-cc",
|
|
31
|
+
callback=_parse_csv,
|
|
32
|
+
help="two letter country code, can be comma separated for a list (eg. IT,US). If omitted will select process all countries.",
|
|
33
|
+
)
|
|
34
|
+
test_name_option = click.option(
|
|
35
|
+
"--test-name",
|
|
36
|
+
type=str,
|
|
37
|
+
callback=_parse_csv,
|
|
38
|
+
help="test_name you care to process, can be comma separated for a list (eg. web_connectivity,whatsapp). If omitted will select process all test names.",
|
|
39
|
+
)
|
|
40
|
+
start_day_option = click.option(
|
|
41
|
+
"--start-day",
|
|
42
|
+
default=date.today() - timedelta(days=14),
|
|
43
|
+
callback=_parse_date,
|
|
44
|
+
help="""the timestamp of the day for which we should start processing data (inclusive).
|
|
45
|
+
|
|
46
|
+
Note: this is the upload date, which doesn't necessarily match the measurement date.
|
|
47
|
+
""",
|
|
48
|
+
)
|
|
49
|
+
end_day_option = click.option(
|
|
50
|
+
"--end-day",
|
|
51
|
+
default=date.today() + timedelta(days=1),
|
|
52
|
+
callback=_parse_date,
|
|
53
|
+
help="""the timestamp of the day for which we should start processing data (inclusive).
|
|
54
|
+
|
|
55
|
+
Note: this is the upload date, which doesn't necessarily match the measurement date.
|
|
56
|
+
""",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@click.group()
|
|
61
|
+
@click.option("--error-log-file", type=Path)
|
|
62
|
+
@click.option(
|
|
63
|
+
"-l",
|
|
64
|
+
"--log-level",
|
|
65
|
+
type=LogLevel(),
|
|
66
|
+
default="INFO",
|
|
67
|
+
help="Set logging level",
|
|
68
|
+
show_default=True,
|
|
69
|
+
)
|
|
70
|
+
@click.version_option(VERSION)
|
|
71
|
+
def cli(error_log_file: Path, log_level: int):
|
|
72
|
+
log.addHandler(logging.StreamHandler(sys.stderr))
|
|
73
|
+
log.setLevel(log_level)
|
|
74
|
+
if error_log_file:
|
|
75
|
+
logging.basicConfig(
|
|
76
|
+
filename=error_log_file, encoding="utf-8", level=logging.ERROR
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@cli.command()
|
|
81
|
+
@click.option("--output-dir", type=Path, required=True)
|
|
82
|
+
@probe_cc_option
|
|
83
|
+
@test_name_option
|
|
84
|
+
@start_day_option
|
|
85
|
+
@end_day_option
|
|
86
|
+
@click.option("--max-string-size", type=int)
|
|
87
|
+
def sync(
|
|
88
|
+
output_dir: Path,
|
|
89
|
+
probe_cc: List[str],
|
|
90
|
+
test_name: List[str],
|
|
91
|
+
start_day: date,
|
|
92
|
+
end_day: date,
|
|
93
|
+
max_string_size: Optional[int] = None,
|
|
94
|
+
):
|
|
95
|
+
"""
|
|
96
|
+
Sync OONI measurements to a directory
|
|
97
|
+
"""
|
|
98
|
+
click.echo(
|
|
99
|
+
f"Downloading measurements for {start_day} - {end_day} into {output_dir}"
|
|
100
|
+
)
|
|
101
|
+
if probe_cc:
|
|
102
|
+
click.echo(f"probe_cc: {','.join(probe_cc)}")
|
|
103
|
+
if test_name:
|
|
104
|
+
click.echo(f"test_name: {','.join(test_name)}")
|
|
105
|
+
|
|
106
|
+
sync_measurements(
|
|
107
|
+
output_dir=output_dir,
|
|
108
|
+
probe_cc=probe_cc,
|
|
109
|
+
test_name=test_name,
|
|
110
|
+
start_day=start_day,
|
|
111
|
+
end_day=end_day,
|
|
112
|
+
max_string_size=max_string_size,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__":
|
|
117
|
+
cli()
|
oonidata/compat.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import dataclasses
|
|
2
|
+
from typing import TypeVar, Any
|
|
3
|
+
|
|
4
|
+
T = TypeVar("T", bound=Any)
|
|
5
|
+
|
|
6
|
+
# Taken from: https://github.com/ericvsmith/dataclasses/blob/master/dataclass_tools.py
|
|
7
|
+
def add_slots(cls: T) -> T:
|
|
8
|
+
# Need to create a new class, since we can't set __slots__
|
|
9
|
+
# after a class has been created.
|
|
10
|
+
|
|
11
|
+
# Make sure __slots__ isn't already set.
|
|
12
|
+
if "__slots__" in cls.__dict__:
|
|
13
|
+
raise TypeError(f"{cls.__name__} already specifies __slots__")
|
|
14
|
+
|
|
15
|
+
# Create a new dict for our new class.
|
|
16
|
+
cls_dict = dict(cls.__dict__)
|
|
17
|
+
field_names = tuple(f.name for f in dataclasses.fields(cls))
|
|
18
|
+
cls_dict["__slots__"] = field_names
|
|
19
|
+
for field_name in field_names:
|
|
20
|
+
# Remove our attributes, if present. They'll still be
|
|
21
|
+
# available in _MARKER.
|
|
22
|
+
cls_dict.pop(field_name, None)
|
|
23
|
+
# Remove __dict__ itself.
|
|
24
|
+
cls_dict.pop("__dict__", None)
|
|
25
|
+
# And finally create the class.
|
|
26
|
+
qualname = getattr(cls, "__qualname__", None)
|
|
27
|
+
cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
|
|
28
|
+
if qualname is not None:
|
|
29
|
+
cls.__qualname__ = qualname
|
|
30
|
+
return cls
|