oonidata 0.3.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
oonidata/__about__.py ADDED
@@ -0,0 +1 @@
1
+ VERSION = "0.3.0"
oonidata/__init__.py ADDED
File without changes
oonidata/apiclient.py ADDED
@@ -0,0 +1,31 @@
1
+ from pathlib import Path
2
+ from typing import Optional
3
+ import orjson
4
+ import requests
5
+
6
+
7
+ def get_raw_measurement(report_id: str, input: Optional[str] = None) -> bytes:
8
+ params = params = {"report_id": report_id, "full": True}
9
+ if input:
10
+ params["input"] = input
11
+ r = requests.get("https://api.ooni.io/api/v1/measurement_meta", params=params)
12
+ j = r.json()
13
+ return j["raw_measurement"].encode("utf-8")
14
+
15
+
16
+ def get_measurement_dict(report_id: str, input: Optional[str] = None) -> dict:
17
+ params = params = {"report_id": report_id, "full": True}
18
+ if input:
19
+ params["input"] = input
20
+ r = requests.get("https://api.ooni.io/api/v1/measurement_meta", params=params)
21
+ j = r.json()
22
+ msmt_dict = orjson.loads(j["raw_measurement"])
23
+ msmt_dict["measurement_uid"] = j["measurement_uid"]
24
+ return msmt_dict
25
+
26
+
27
+ def get_measurement_dict_by_uid(measurement_uid: str) -> dict:
28
+ r = requests.get(f"https://api.ooni.io/api/v1/measurement/{measurement_uid}")
29
+ msmt_dict = r.json()
30
+ msmt_dict["measurement_uid"] = measurement_uid
31
+ return msmt_dict
@@ -0,0 +1 @@
1
+ from .command import cli
@@ -0,0 +1,117 @@
1
+ import logging
2
+ import sys
3
+
4
+ from pathlib import Path
5
+ from typing import List, Optional
6
+ from datetime import date, timedelta, datetime
7
+
8
+ import click
9
+ from click_loglevel import LogLevel
10
+
11
+ from oonidata.__about__ import VERSION
12
+ from oonidata.dataclient import (
13
+ sync_measurements,
14
+ )
15
+
16
+ log = logging.getLogger("oonidata")
17
+
18
+
19
+ def _parse_date(ctx, param, date_str: str) -> date:
20
+ return datetime.strptime(date_str, "%Y-%m-%d").date()
21
+
22
+
23
+ def _parse_csv(ctx, param, s: Optional[str]) -> List[str]:
24
+ if s:
25
+ return s.split(",")
26
+ return []
27
+
28
+
29
+ probe_cc_option = click.option(
30
+ "--probe-cc",
31
+ callback=_parse_csv,
32
+ help="two letter country code, can be comma separated for a list (eg. IT,US). If omitted will select process all countries.",
33
+ )
34
+ test_name_option = click.option(
35
+ "--test-name",
36
+ type=str,
37
+ callback=_parse_csv,
38
+ help="test_name you care to process, can be comma separated for a list (eg. web_connectivity,whatsapp). If omitted will select process all test names.",
39
+ )
40
+ start_day_option = click.option(
41
+ "--start-day",
42
+ default=date.today() - timedelta(days=14),
43
+ callback=_parse_date,
44
+ help="""the timestamp of the day for which we should start processing data (inclusive).
45
+
46
+ Note: this is the upload date, which doesn't necessarily match the measurement date.
47
+ """,
48
+ )
49
+ end_day_option = click.option(
50
+ "--end-day",
51
+ default=date.today() + timedelta(days=1),
52
+ callback=_parse_date,
53
+ help="""the timestamp of the day for which we should start processing data (inclusive).
54
+
55
+ Note: this is the upload date, which doesn't necessarily match the measurement date.
56
+ """,
57
+ )
58
+
59
+
60
+ @click.group()
61
+ @click.option("--error-log-file", type=Path)
62
+ @click.option(
63
+ "-l",
64
+ "--log-level",
65
+ type=LogLevel(),
66
+ default="INFO",
67
+ help="Set logging level",
68
+ show_default=True,
69
+ )
70
+ @click.version_option(VERSION)
71
+ def cli(error_log_file: Path, log_level: int):
72
+ log.addHandler(logging.StreamHandler(sys.stderr))
73
+ log.setLevel(log_level)
74
+ if error_log_file:
75
+ logging.basicConfig(
76
+ filename=error_log_file, encoding="utf-8", level=logging.ERROR
77
+ )
78
+
79
+
80
+ @cli.command()
81
+ @click.option("--output-dir", type=Path, required=True)
82
+ @probe_cc_option
83
+ @test_name_option
84
+ @start_day_option
85
+ @end_day_option
86
+ @click.option("--max-string-size", type=int)
87
+ def sync(
88
+ output_dir: Path,
89
+ probe_cc: List[str],
90
+ test_name: List[str],
91
+ start_day: date,
92
+ end_day: date,
93
+ max_string_size: Optional[int] = None,
94
+ ):
95
+ """
96
+ Sync OONI measurements to a directory
97
+ """
98
+ click.echo(
99
+ f"Downloading measurements for {start_day} - {end_day} into {output_dir}"
100
+ )
101
+ if probe_cc:
102
+ click.echo(f"probe_cc: {','.join(probe_cc)}")
103
+ if test_name:
104
+ click.echo(f"test_name: {','.join(test_name)}")
105
+
106
+ sync_measurements(
107
+ output_dir=output_dir,
108
+ probe_cc=probe_cc,
109
+ test_name=test_name,
110
+ start_day=start_day,
111
+ end_day=end_day,
112
+ max_string_size=max_string_size,
113
+ )
114
+
115
+
116
+ if __name__ == "__main__":
117
+ cli()
oonidata/compat.py ADDED
@@ -0,0 +1,30 @@
1
+ import dataclasses
2
+ from typing import TypeVar, Any
3
+
4
+ T = TypeVar("T", bound=Any)
5
+
6
+ # Taken from: https://github.com/ericvsmith/dataclasses/blob/master/dataclass_tools.py
7
+ def add_slots(cls: T) -> T:
8
+ # Need to create a new class, since we can't set __slots__
9
+ # after a class has been created.
10
+
11
+ # Make sure __slots__ isn't already set.
12
+ if "__slots__" in cls.__dict__:
13
+ raise TypeError(f"{cls.__name__} already specifies __slots__")
14
+
15
+ # Create a new dict for our new class.
16
+ cls_dict = dict(cls.__dict__)
17
+ field_names = tuple(f.name for f in dataclasses.fields(cls))
18
+ cls_dict["__slots__"] = field_names
19
+ for field_name in field_names:
20
+ # Remove our attributes, if present. They'll still be
21
+ # available in _MARKER.
22
+ cls_dict.pop(field_name, None)
23
+ # Remove __dict__ itself.
24
+ cls_dict.pop("__dict__", None)
25
+ # And finally create the class.
26
+ qualname = getattr(cls, "__qualname__", None)
27
+ cls = type(cls)(cls.__name__, cls.__bases__, cls_dict)
28
+ if qualname is not None:
29
+ cls.__qualname__ = qualname
30
+ return cls