owid-datautils 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- owid/datautils/__init__.py +3 -0
- owid/datautils/checks.py +1 -0
- owid/datautils/common.py +44 -0
- owid/datautils/dataframes.py +739 -0
- owid/datautils/decorators.py +91 -0
- owid/datautils/format/__init__.py +7 -0
- owid/datautils/format/numbers.py +304 -0
- owid/datautils/google/__init__.py +7 -0
- owid/datautils/google/api.py +136 -0
- owid/datautils/google/config.py +117 -0
- owid/datautils/google/sheets.py +134 -0
- owid/datautils/io/__init__.py +14 -0
- owid/datautils/io/archive.py +84 -0
- owid/datautils/io/df.py +163 -0
- owid/datautils/io/json.py +78 -0
- owid/datautils/ui.py +32 -0
- owid/datautils/web.py +106 -0
- owid_datautils-0.6.1.dist-info/METADATA +20 -0
- owid_datautils-0.6.1.dist-info/RECORD +21 -0
- owid_datautils-0.6.1.dist-info/WHEEL +4 -0
- owid_datautils-0.6.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Google Sheet utils."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from typing import Any, Optional, Union
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
from gsheets import Sheets
|
|
8
|
+
from gsheets.models import SpreadSheet, WorkSheet
|
|
9
|
+
|
|
10
|
+
from owid.datautils.google.config import CLIENT_SECRETS_PATH, CREDENTIALS_PATH
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class GSheetsApi:
|
|
14
|
+
"""Interface to interact with Google sheets."""
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
clients_secrets: str = CLIENT_SECRETS_PATH,
|
|
19
|
+
credentials_path: str = CREDENTIALS_PATH,
|
|
20
|
+
) -> None:
|
|
21
|
+
self.clients_secrets = clients_secrets
|
|
22
|
+
self.credentials_path = credentials_path
|
|
23
|
+
self._init_config_folder()
|
|
24
|
+
self.__sheets = None
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def sheets(self) -> Sheets:
|
|
28
|
+
"""Access or initialize sheets attribute."""
|
|
29
|
+
if self.__sheets is None:
|
|
30
|
+
self.__sheets = Sheets.from_files(self.clients_secrets, self.credentials_path, no_webserver=True)
|
|
31
|
+
return self.__sheets
|
|
32
|
+
|
|
33
|
+
def _init_config_folder(self) -> None:
|
|
34
|
+
credentials_folder = os.path.expanduser(os.path.dirname(self.credentials_path))
|
|
35
|
+
if not os.path.isdir(credentials_folder):
|
|
36
|
+
os.makedirs(credentials_folder, exist_ok=True)
|
|
37
|
+
|
|
38
|
+
def get(self, spreadsheet_id: str, worksheet_id: Optional[int] = None) -> Union[SpreadSheet, WorkSheet]:
|
|
39
|
+
"""Get a spreadsheet or worksheet from a Google sheet.
|
|
40
|
+
|
|
41
|
+
If only `spreadsheet_id` is provided, this will return the entire spreadsheet. Otherwise,
|
|
42
|
+
the specific worksheet will be returned.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
spreadsheet_id : str
|
|
47
|
+
ID of the spreadsheet.
|
|
48
|
+
worksheet_id : int
|
|
49
|
+
ID of the worksheet.
|
|
50
|
+
|
|
51
|
+
Returns
|
|
52
|
+
-------
|
|
53
|
+
Spreadsheet or WorkSheet
|
|
54
|
+
SpreadSheet or Worksheet.
|
|
55
|
+
"""
|
|
56
|
+
ssheet = self.sheets.get(spreadsheet_id)
|
|
57
|
+
if worksheet_id:
|
|
58
|
+
return ssheet.get(worksheet_id) # type: ignore[reportReturnType]
|
|
59
|
+
return ssheet # type: ignore[reportReturnType]
|
|
60
|
+
|
|
61
|
+
def download_worksheet(
|
|
62
|
+
self,
|
|
63
|
+
spreadsheet_id: str,
|
|
64
|
+
worksheet_id: int,
|
|
65
|
+
output_path: Optional[str] = None,
|
|
66
|
+
encoding: str = "utf-8",
|
|
67
|
+
**kwargs: Any,
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Download a worksheet from a Google sheet.
|
|
70
|
+
|
|
71
|
+
Saves the downloaded worksheet as a CSV.
|
|
72
|
+
|
|
73
|
+
Parameters
|
|
74
|
+
----------
|
|
75
|
+
spreadsheet_id : str
|
|
76
|
+
ID of the spreadsheet. This is included in the URL of the spreadsheet.
|
|
77
|
+
worksheet_id : int
|
|
78
|
+
ID of the worksheet. This is included in the URL of the spreadsheet. Look for
|
|
79
|
+
"edit#gid=<worksheet_id>" section.
|
|
80
|
+
output_path : str, optional
|
|
81
|
+
Local path where to save the downloaded worksheet. By default it will save it in
|
|
82
|
+
the execution directory under the name that was given to the worksheet on Google Drive.
|
|
83
|
+
encoding : str, optional
|
|
84
|
+
Encoding of the file, by default "utf-8"
|
|
85
|
+
"""
|
|
86
|
+
sheet = self.get(spreadsheet_id, worksheet_id)
|
|
87
|
+
if output_path:
|
|
88
|
+
sheet.to_csv(output_path, encoding=encoding, **kwargs) # type: ignore[reportCallIssue]
|
|
89
|
+
else:
|
|
90
|
+
make_filename = "%(title)s.csv"
|
|
91
|
+
sheet.to_csv(make_filename=make_filename, encoding=encoding, **kwargs)
|
|
92
|
+
|
|
93
|
+
def download_spreadsheet(
|
|
94
|
+
self, spreadsheet_id: str, output_dir: str, encoding: str = "utf-8", **kwargs: Any
|
|
95
|
+
) -> None:
|
|
96
|
+
"""Download a spreadsheet from a Google sheet.
|
|
97
|
+
|
|
98
|
+
It downloads all the worksheets from the given spreadsheets and saves them as CSVs.
|
|
99
|
+
The filenames given to the worksheets are as follows:
|
|
100
|
+
|
|
101
|
+
`<output_dir>/<spreadsheet_title> - <worksheet_title>.csv`
|
|
102
|
+
|
|
103
|
+
Parameters
|
|
104
|
+
----------
|
|
105
|
+
spreadsheet_id : str
|
|
106
|
+
ID of the spreadsheet. This is included in the URL of the spreadsheet.
|
|
107
|
+
output_dir : str, optional
|
|
108
|
+
Local directory where to store all the downloaded worksheets.
|
|
109
|
+
encoding : str, optional
|
|
110
|
+
Encoding of the file, by default "utf-8"
|
|
111
|
+
"""
|
|
112
|
+
sheet = self.get(spreadsheet_id)
|
|
113
|
+
make_filename = os.path.join(output_dir, "%(title)s - %(sheet)s.csv")
|
|
114
|
+
sheet.to_csv(make_filename=make_filename, encoding=encoding, **kwargs)
|
|
115
|
+
|
|
116
|
+
def worksheet_to_df(self, spreadsheet_id: str, worksheet_id: int) -> pd.DataFrame:
|
|
117
|
+
"""Load a Worksheet as a dataframe.
|
|
118
|
+
|
|
119
|
+
Parameters
|
|
120
|
+
----------
|
|
121
|
+
spreadsheet_id : str
|
|
122
|
+
ID of the spreadsheet. This is included in the URL of the spreadsheet.
|
|
123
|
+
worksheet_id : int
|
|
124
|
+
ID of the worksheet. This is included in the URL of the spreadsheet. Look for
|
|
125
|
+
"edit#gid=<worksheet_id>" section.
|
|
126
|
+
|
|
127
|
+
Returns
|
|
128
|
+
-------
|
|
129
|
+
pd.DataFrame:
|
|
130
|
+
Dataframe with the data from the worksheet.
|
|
131
|
+
"""
|
|
132
|
+
ws = self.get(spreadsheet_id, worksheet_id)
|
|
133
|
+
df: pd.DataFrame = ws.to_frame() # type: ignore[reportAttributeAccessIssue]
|
|
134
|
+
return df
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Input/Output methods."""
|
|
2
|
+
|
|
3
|
+
from owid.datautils.io.archive import decompress_file
|
|
4
|
+
from owid.datautils.io.df import from_file as df_from_file
|
|
5
|
+
from owid.datautils.io.df import to_file as df_to_file
|
|
6
|
+
from owid.datautils.io.json import load_json, save_json
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"decompress_file",
|
|
10
|
+
"load_json",
|
|
11
|
+
"save_json",
|
|
12
|
+
"df_from_file",
|
|
13
|
+
"df_to_file",
|
|
14
|
+
]
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Input/Output functions for local files."""
|
|
2
|
+
|
|
3
|
+
import tarfile
|
|
4
|
+
import zipfile
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Union, cast
|
|
7
|
+
|
|
8
|
+
from py7zr import SevenZipFile
|
|
9
|
+
|
|
10
|
+
from owid.datautils.decorators import enable_file_download
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@enable_file_download(path_arg_name="input_file")
|
|
14
|
+
def decompress_file(
|
|
15
|
+
input_file: Union[str, Path],
|
|
16
|
+
output_folder: Union[str, Path],
|
|
17
|
+
overwrite: bool = False,
|
|
18
|
+
) -> None:
|
|
19
|
+
"""Extract a zip or tar file.
|
|
20
|
+
|
|
21
|
+
It can be a local or a remote file.
|
|
22
|
+
|
|
23
|
+
Parameters
|
|
24
|
+
----------
|
|
25
|
+
input_file : Union[str, Path]
|
|
26
|
+
Path to local zip file, or URL of a remote zip file.
|
|
27
|
+
output_folder : Union[str, Path]
|
|
28
|
+
Path to local folder.
|
|
29
|
+
overwrite : bool
|
|
30
|
+
Overwrite decompressed content if it already exists (otherwise raises an error if content already exists).
|
|
31
|
+
|
|
32
|
+
"""
|
|
33
|
+
if isinstance(input_file, str):
|
|
34
|
+
input_file = Path(input_file)
|
|
35
|
+
input_file = cast(Path, input_file)
|
|
36
|
+
|
|
37
|
+
if zipfile.is_zipfile(input_file):
|
|
38
|
+
_decompress_zip_file(input_file, output_folder, overwrite)
|
|
39
|
+
elif tarfile.is_tarfile(input_file):
|
|
40
|
+
_decompress_tar_file(input_file, output_folder, overwrite)
|
|
41
|
+
elif input_file.suffix.lower() == ".7z":
|
|
42
|
+
with SevenZipFile(input_file, mode="r") as z:
|
|
43
|
+
z.extractall(path=output_folder)
|
|
44
|
+
else:
|
|
45
|
+
raise ValueError("File is neither a zip nor a tar file.")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _decompress_zip_file(
|
|
49
|
+
input_file: Union[str, Path],
|
|
50
|
+
output_folder: Union[str, Path],
|
|
51
|
+
overwrite: bool = False,
|
|
52
|
+
) -> None:
|
|
53
|
+
"""Unpack zip file."""
|
|
54
|
+
zip_file = zipfile.ZipFile(input_file)
|
|
55
|
+
|
|
56
|
+
# If the content to be written in output folder already exists, raise an error,
|
|
57
|
+
# unless 'overwrite' is set to True, in which case the existing file will be overwritten.
|
|
58
|
+
# Path to new file to be created.
|
|
59
|
+
new_file = Path(output_folder) / Path(zip_file.namelist()[0])
|
|
60
|
+
if new_file.exists() and not overwrite:
|
|
61
|
+
raise FileExistsError("Output already exists. Either change output_folder or use overwrite=True.")
|
|
62
|
+
|
|
63
|
+
# Unzip the file and save it in the local output folder.
|
|
64
|
+
# Note that, if output_folder path does not exist, the following command will create it.
|
|
65
|
+
zip_file.extractall(output_folder)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _decompress_tar_file(
|
|
69
|
+
input_file: Union[str, Path],
|
|
70
|
+
output_folder: Union[str, Path],
|
|
71
|
+
overwrite: bool = False,
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Unpack tar file."""
|
|
74
|
+
with tarfile.open(input_file) as tar_file:
|
|
75
|
+
# If the content to be written in output folder already exists, raise an error,
|
|
76
|
+
# unless 'overwrite' is set to True, in which case the existing file will be overwritten.
|
|
77
|
+
# Path to new file to be created.
|
|
78
|
+
new_file = Path(output_folder) / Path(tar_file.getnames()[0])
|
|
79
|
+
if new_file.exists() and not overwrite:
|
|
80
|
+
raise FileExistsError("Output already exists. Either change output_folder or use" " overwrite=True.")
|
|
81
|
+
|
|
82
|
+
# Unzip the file and save it in the local output folder.
|
|
83
|
+
# Note that, if output_folder path does not exist, the following command will create it.
|
|
84
|
+
tar_file.extractall(output_folder)
|
owid/datautils/io/df.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""DataFrame io operations."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, List, Optional, Union
|
|
6
|
+
|
|
7
|
+
import pandas as pd
|
|
8
|
+
|
|
9
|
+
from owid.datautils.decorators import enable_file_download
|
|
10
|
+
|
|
11
|
+
COMPRESSION_SUPPORTED = ["gz", "bz2", "zip", "xz", "zst", "tar"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _has_index(df: pd.DataFrame) -> bool:
|
|
15
|
+
# Copy of dataframes.has_index to avoid circular imports.
|
|
16
|
+
df_has_index = True if df.index.names[0] is not None else False
|
|
17
|
+
|
|
18
|
+
return df_has_index
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@enable_file_download("file_path")
|
|
22
|
+
def from_file(
|
|
23
|
+
file_path: Union[str, Path], file_type: Optional[str] = None, **kwargs: Any
|
|
24
|
+
) -> Union[pd.DataFrame, List[pd.DataFrame]]:
|
|
25
|
+
"""Load a file as a pandas DataFrame.
|
|
26
|
+
|
|
27
|
+
It uses standard pandas function pandas.read_* but adds the ability to read from a URL in some
|
|
28
|
+
cases where pandas does not work.
|
|
29
|
+
|
|
30
|
+
The function will infer the extension of `file_path` by simply taking what follows the last ".". For example:
|
|
31
|
+
"file.csv" will be read as a csv file, and "http://my/domain/file.xlsx" will be read as an excel file.
|
|
32
|
+
|
|
33
|
+
Reading from compressed files will not work by default, unless you provide a `file_type`.
|
|
34
|
+
|
|
35
|
+
Parameters
|
|
36
|
+
----------
|
|
37
|
+
filepath : str
|
|
38
|
+
Path or url to file.
|
|
39
|
+
file_type : str
|
|
40
|
+
File type of the data file. By default is None, as it is only required when reading compressed files.
|
|
41
|
+
This is typically equivalent to the file extension. However, when reading a
|
|
42
|
+
compressed file, this refers to the actual file that is compressed (not the compressed file extension).
|
|
43
|
+
Reading from compressed files is supported for "csv", "dta" and "json".
|
|
44
|
+
kwargs :
|
|
45
|
+
pandas.read_* arguments.
|
|
46
|
+
|
|
47
|
+
Returns
|
|
48
|
+
-------
|
|
49
|
+
pandas.DataFrame:
|
|
50
|
+
Read dataframe.
|
|
51
|
+
"""
|
|
52
|
+
# Ensure file_path is a Path object.
|
|
53
|
+
file_path = Path(file_path)
|
|
54
|
+
|
|
55
|
+
# Ensure extension is lower case and does not start with '.'.
|
|
56
|
+
extension = file_path.suffix.lstrip(".").lower()
|
|
57
|
+
|
|
58
|
+
# If compressed file, raise an exception unless file_type is given
|
|
59
|
+
if extension in COMPRESSION_SUPPORTED:
|
|
60
|
+
if file_type:
|
|
61
|
+
extension = file_type
|
|
62
|
+
else:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
"To be able to read from a compressed file, you need to provide a value" " for `file_type`."
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# Check path is valid
|
|
68
|
+
if not file_path.exists():
|
|
69
|
+
raise FileNotFoundError(f"Cannot find file: {file_path}")
|
|
70
|
+
|
|
71
|
+
# Available input methods (some of them may need additional dependencies to work).
|
|
72
|
+
input_methods = {
|
|
73
|
+
"csv": pd.read_csv,
|
|
74
|
+
"dta": pd.read_stata,
|
|
75
|
+
"feather": pd.read_feather,
|
|
76
|
+
"hdf": pd.read_hdf,
|
|
77
|
+
"html": pd.read_html,
|
|
78
|
+
"json": pd.read_json,
|
|
79
|
+
"parquet": pd.read_parquet,
|
|
80
|
+
"pickle": pd.read_pickle,
|
|
81
|
+
"pkl": pd.read_pickle,
|
|
82
|
+
"xlsx": pd.read_excel,
|
|
83
|
+
"xml": pd.read_xml,
|
|
84
|
+
}
|
|
85
|
+
if extension not in input_methods:
|
|
86
|
+
raise ValueError("Failed reading dataframe because of an unknown file extension:" f" {extension}")
|
|
87
|
+
# Select the appropriate reading method.
|
|
88
|
+
read_function = input_methods[extension]
|
|
89
|
+
|
|
90
|
+
# Load file using the chosen read function and the appropriate arguments.
|
|
91
|
+
df: pd.DataFrame = read_function(file_path, **kwargs)
|
|
92
|
+
return df
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def to_file(df: pd.DataFrame, file_path: Union[str, Path], overwrite: bool = True, **kwargs: Any) -> None:
|
|
96
|
+
"""Save dataframe to file.
|
|
97
|
+
|
|
98
|
+
This function wraps all pandas df.to_* methods, e.g. df.to_csv() or df.to_parquet(), with the following advantages:
|
|
99
|
+
* The output file will have the format determined by the extension of file_path. Hence, to_file(df, "data.csv") will
|
|
100
|
+
create a csv file, and to_file(df, "data.parquet") will create a parquet file.
|
|
101
|
+
* If file_path is with one or more subfolders that do not exist, the full path will be created.
|
|
102
|
+
* It can overwrite an existing file (if overwrite is True), or raise an error if the file already exists.
|
|
103
|
+
* It will avoid creating an index column if the dataframe has a dummy index (which would be equivalent to doing
|
|
104
|
+
df.to_csv(file_path, index=False)), but it will include the index if the dataframe has one.
|
|
105
|
+
* Any additional keyword argument that would be passed on to the method to write a file can be safely added. For
|
|
106
|
+
example, to_file(df, "data.csv", na_rep="TEST") will replace missing data by "TEST" (analogous to
|
|
107
|
+
df.to_csv("data.csv", na_rep="TEST")).
|
|
108
|
+
|
|
109
|
+
Parameters
|
|
110
|
+
----------
|
|
111
|
+
df : pd.DataFrame
|
|
112
|
+
Dataframe to be stored in a file.
|
|
113
|
+
file_path : Union[str, Path]
|
|
114
|
+
Path to file to be created.
|
|
115
|
+
overwrite : bool, optional
|
|
116
|
+
True to overwrite file if it already exists. False to raise an error if file already exists.
|
|
117
|
+
|
|
118
|
+
"""
|
|
119
|
+
# Ensure file_path is a Path object.
|
|
120
|
+
file_path = Path(file_path)
|
|
121
|
+
|
|
122
|
+
# Ensure extension is lower case and does not start with '.'.
|
|
123
|
+
extension = file_path.suffix.lstrip(".").lower()
|
|
124
|
+
|
|
125
|
+
# Ensure output directory exists.
|
|
126
|
+
if not file_path.parent.exists():
|
|
127
|
+
file_path.parent.mkdir(parents=True)
|
|
128
|
+
|
|
129
|
+
# Avoid overwriting an existing file unless explicitly stated.
|
|
130
|
+
if file_path.is_file() and not overwrite:
|
|
131
|
+
raise FileExistsError("Failed to save dataframe because file exists and 'overwrite' is False.")
|
|
132
|
+
|
|
133
|
+
# Available output methods (some of them may need additional dependencies to work).
|
|
134
|
+
output_methods = {
|
|
135
|
+
"csv": df.to_csv,
|
|
136
|
+
"dta": df.to_stata,
|
|
137
|
+
"feather": df.to_feather,
|
|
138
|
+
"hdf": df.to_hdf,
|
|
139
|
+
"html": df.to_html,
|
|
140
|
+
"json": df.to_json,
|
|
141
|
+
"md": df.to_markdown,
|
|
142
|
+
"parquet": df.to_parquet,
|
|
143
|
+
"pickle": df.to_pickle,
|
|
144
|
+
"pkl": df.to_pickle,
|
|
145
|
+
"tex": df.to_latex,
|
|
146
|
+
"txt": df.to_string,
|
|
147
|
+
"xlsx": df.to_excel,
|
|
148
|
+
"xml": df.to_xml,
|
|
149
|
+
}
|
|
150
|
+
if extension not in output_methods:
|
|
151
|
+
raise ValueError(f"Failed saving dataframe because of an unknown file extension: {extension}")
|
|
152
|
+
# Select the appropriate storing method.
|
|
153
|
+
save_function = output_methods[extension]
|
|
154
|
+
|
|
155
|
+
# Decide whether dataframe should be stored with or without an index, if:
|
|
156
|
+
# * The storing method allows for an 'index' argument.
|
|
157
|
+
# * The argument "index" is not explicitly given.
|
|
158
|
+
if ("index" in inspect.signature(save_function).parameters) and ("index" not in kwargs):
|
|
159
|
+
# Make 'index' False to avoid storing index if dataframe has a dummy index.
|
|
160
|
+
kwargs["index"] = _has_index(df=df)
|
|
161
|
+
|
|
162
|
+
# Save file using the chosen save function and the appropriate arguments.
|
|
163
|
+
save_function(file_path, **kwargs)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Input/Output functions for local files."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Hashable, List, Tuple, Union
|
|
6
|
+
|
|
7
|
+
from owid.datautils.common import warn_on_list_of_entities
|
|
8
|
+
from owid.datautils.decorators import enable_file_download
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _load_json_data_and_duplicated_keys(ordered_pairs: List[Tuple[Hashable, Any]]) -> Any:
|
|
12
|
+
clean_dict = {}
|
|
13
|
+
duplicated_keys = []
|
|
14
|
+
for key, value in ordered_pairs:
|
|
15
|
+
if key in clean_dict:
|
|
16
|
+
duplicated_keys.append(key)
|
|
17
|
+
clean_dict[key] = value
|
|
18
|
+
if len(duplicated_keys) > 0:
|
|
19
|
+
warn_on_list_of_entities(
|
|
20
|
+
list_of_entities=duplicated_keys,
|
|
21
|
+
warning_message="Duplicated entities found.",
|
|
22
|
+
show_list=True,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
return clean_dict
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@enable_file_download(path_arg_name="json_file")
|
|
29
|
+
def load_json(json_file: Union[str, Path], warn_on_duplicated_keys: bool = True) -> Any:
|
|
30
|
+
"""Load data from json file, and optionally warn if there are duplicated keys.
|
|
31
|
+
|
|
32
|
+
If json file contains duplicated keys, a warning is optionally raised, and only the value of the latest duplicated
|
|
33
|
+
key is kept.
|
|
34
|
+
|
|
35
|
+
Parameters
|
|
36
|
+
----------
|
|
37
|
+
json_file : Path or str
|
|
38
|
+
Path to json file.
|
|
39
|
+
warn_on_duplicated_keys : bool
|
|
40
|
+
True to raise a warning if there are duplicated keys in json file. False to ignore.
|
|
41
|
+
|
|
42
|
+
Returns
|
|
43
|
+
-------
|
|
44
|
+
data : dict
|
|
45
|
+
Data loaded from json file.
|
|
46
|
+
|
|
47
|
+
"""
|
|
48
|
+
with open(json_file, "r") as _json_file:
|
|
49
|
+
json_content = _json_file.read()
|
|
50
|
+
if warn_on_duplicated_keys:
|
|
51
|
+
data = json.loads(json_content, object_pairs_hook=_load_json_data_and_duplicated_keys)
|
|
52
|
+
else:
|
|
53
|
+
data = json.loads(json_content)
|
|
54
|
+
|
|
55
|
+
return data
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def save_json(data: Any, json_file: Union[str, Path], **kwargs: Any) -> None:
|
|
59
|
+
"""Save data to a json file.
|
|
60
|
+
|
|
61
|
+
Parameters
|
|
62
|
+
----------
|
|
63
|
+
data : list
|
|
64
|
+
Data to be stored in a json file.
|
|
65
|
+
json_file : str
|
|
66
|
+
Path to output json file.
|
|
67
|
+
kwargs:
|
|
68
|
+
Additional keyword arguments for json.dump (e.g. indent=4, sort_keys=True).
|
|
69
|
+
|
|
70
|
+
"""
|
|
71
|
+
# Ensure json_file is a path.
|
|
72
|
+
json_file = Path(json_file)
|
|
73
|
+
|
|
74
|
+
# Ensure output directory exists.
|
|
75
|
+
json_file.parent.mkdir(parents=True, exist_ok=True)
|
|
76
|
+
|
|
77
|
+
with open(json_file, "w") as _json_file:
|
|
78
|
+
json.dump(data, _json_file, **kwargs)
|
owid/datautils/ui.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Nice console UI helpers."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
from typing import NoReturn
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def log(action: str, message: str) -> None:
|
|
10
|
+
"""Log message."""
|
|
11
|
+
action = f"{action:20s}"
|
|
12
|
+
click.echo(f"{blue(action)}{message}")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def bail(message: str) -> NoReturn:
|
|
16
|
+
"""Print an error message then exit."""
|
|
17
|
+
print(f'{red("ERROR:")} {message}', file=sys.stderr)
|
|
18
|
+
sys.exit(1)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def blue(s: str) -> str:
|
|
22
|
+
"""Add click style (blue color)."""
|
|
23
|
+
return _colorize(s, fg="blue")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def red(s: str) -> str:
|
|
27
|
+
"""Add click style (red color)."""
|
|
28
|
+
return _colorize(s, fg="red")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _colorize(s: str, fg: str) -> str:
|
|
32
|
+
return click.style(s, fg=fg)
|
owid/datautils/web.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Web utils."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import warnings
|
|
5
|
+
from urllib.parse import urlparse
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
from requests.adapters import HTTPAdapter
|
|
9
|
+
from requests.packages.urllib3.util.ssl_ import create_urllib3_context # type: ignore
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def get_base_url(url: str, include_scheme: bool = True) -> str:
|
|
13
|
+
"""Get base URL from an arbitrary URL path (e.g. "https://example.com/some/path" -> "https://example.com").
|
|
14
|
+
|
|
15
|
+
If the given URL does not start with "http(s)://"
|
|
16
|
+
|
|
17
|
+
Parameters
|
|
18
|
+
----------
|
|
19
|
+
url : str
|
|
20
|
+
Input URL.
|
|
21
|
+
include_scheme : bool, optional
|
|
22
|
+
True to include "http(s)://" at the beginning of the returned base URL.
|
|
23
|
+
False to hide the "http(s)://" (so that "https://example.com/some/path" -> "example.com").
|
|
24
|
+
|
|
25
|
+
Returns
|
|
26
|
+
-------
|
|
27
|
+
base_url : str
|
|
28
|
+
Base URL.
|
|
29
|
+
|
|
30
|
+
"""
|
|
31
|
+
# Function urlparse cannot parse a url if it does not start with http(s)://.
|
|
32
|
+
# If such a url is passed, assume "http://".
|
|
33
|
+
if not re.match(r"https?://", url):
|
|
34
|
+
warnings.warn(f"Schema not defined for url {url}; assuming http.")
|
|
35
|
+
url = f"http://{url}"
|
|
36
|
+
|
|
37
|
+
# Parse url, and return the base url either starting with "http(s)://" (if include_scheme is True) or without it.
|
|
38
|
+
parsed_url = urlparse(url)
|
|
39
|
+
if include_scheme:
|
|
40
|
+
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
|
|
41
|
+
else:
|
|
42
|
+
base_url = parsed_url.netloc
|
|
43
|
+
|
|
44
|
+
return base_url
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _DESAdapter(HTTPAdapter): # type: ignore
|
|
48
|
+
"""A TransportAdapter that re-enables Triple DES support in requests.
|
|
49
|
+
|
|
50
|
+
From: https://stackoverflow.com/a/46186957/5056599
|
|
51
|
+
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
ciphers = "HIGH:!DH:!aNULL:DEFAULT@SECLEVEL=1"
|
|
55
|
+
|
|
56
|
+
def init_poolmanager(self, *args, **kwargs): # type: ignore
|
|
57
|
+
context = create_urllib3_context(ciphers=self.ciphers)
|
|
58
|
+
kwargs["ssl_context"] = context
|
|
59
|
+
return super(_DESAdapter, self).init_poolmanager(*args, **kwargs)
|
|
60
|
+
|
|
61
|
+
def proxy_manager_for(self, *args, **kwargs): # type: ignore
|
|
62
|
+
context = create_urllib3_context(ciphers=self.ciphers)
|
|
63
|
+
kwargs["ssl_context"] = context
|
|
64
|
+
return super(_DESAdapter, self).proxy_manager_for(*args, **kwargs)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def download_file_from_url(
|
|
68
|
+
url: str,
|
|
69
|
+
local_path: str,
|
|
70
|
+
chunk_size: float = 1024 * 1024,
|
|
71
|
+
timeout: float = 30,
|
|
72
|
+
verify: bool = True,
|
|
73
|
+
ciphers_low: bool = False,
|
|
74
|
+
) -> None:
|
|
75
|
+
"""Download file from a URL into a local file.
|
|
76
|
+
|
|
77
|
+
Parameters
|
|
78
|
+
----------
|
|
79
|
+
url : str
|
|
80
|
+
URL of the file to be downloaded.
|
|
81
|
+
local_path : str
|
|
82
|
+
Path to local file that will be created.
|
|
83
|
+
chunk_size : float, optional
|
|
84
|
+
Maximum size of the chunks of the response of a request.
|
|
85
|
+
timeout : float, optional
|
|
86
|
+
Timeout for standard GET requests.
|
|
87
|
+
verify : bool, optional
|
|
88
|
+
Verify SSL certificate for HTTPS requests.
|
|
89
|
+
ciphers_low : bool, optional
|
|
90
|
+
Use less restrictive encryption for request (required by certain old web sites).
|
|
91
|
+
|
|
92
|
+
"""
|
|
93
|
+
# Create a persistent request session.
|
|
94
|
+
with requests.Session() as session:
|
|
95
|
+
if ciphers_low:
|
|
96
|
+
# Special type of request.
|
|
97
|
+
session.mount(get_base_url(url), _DESAdapter())
|
|
98
|
+
response = session.get(url, timeout=timeout)
|
|
99
|
+
else:
|
|
100
|
+
# Send a standard GET request.
|
|
101
|
+
response = session.get(url, stream=True, timeout=timeout, verify=verify)
|
|
102
|
+
|
|
103
|
+
# Save the requested data into a local file.
|
|
104
|
+
with open(local_path, "wb") as output_file:
|
|
105
|
+
for chunk in response.iter_content(chunk_size=chunk_size): # type: ignore
|
|
106
|
+
output_file.write(chunk)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: owid-datautils
|
|
3
|
+
Version: 0.6.1
|
|
4
|
+
Summary: Data utils library by the Data Team at Our World in Data
|
|
5
|
+
Author-email: Our World in Data <tech@ourworldindata.org>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: data cleaning,data processing,data utils,our world in data
|
|
9
|
+
Requires-Python: <4.0,>=3.10
|
|
10
|
+
Requires-Dist: boto3>=1.38.23
|
|
11
|
+
Requires-Dist: click>=8.1.7
|
|
12
|
+
Requires-Dist: colorama>=0.4.4
|
|
13
|
+
Requires-Dist: gdown>=4.5.2
|
|
14
|
+
Requires-Dist: gsheets>=0.6.1
|
|
15
|
+
Requires-Dist: pandas>=2.2.3
|
|
16
|
+
Requires-Dist: py7zr>=0.22.0
|
|
17
|
+
Requires-Dist: pyarrow>=18.0.0
|
|
18
|
+
Requires-Dist: pydrive2>=1.15.0
|
|
19
|
+
Requires-Dist: structlog>=21.5.0
|
|
20
|
+
Requires-Dist: urllib3<2
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
owid/datautils/__init__.py,sha256=ApGUd0h-5lbnwXtY1DLhjVBFdbdI-wFEy4LLDKg0Trw,96
|
|
2
|
+
owid/datautils/checks.py,sha256=HGw8hvYVkOrIGE19dpkDUqIyKBiEEViqK2OZr37IsOs,50
|
|
3
|
+
owid/datautils/common.py,sha256=9mSgAAL8NnG4iD6Ysj08Vw2Aw1eyxLA1atWqyDffu7M,1598
|
|
4
|
+
owid/datautils/dataframes.py,sha256=Y0rm436WMlWTK3TSmDqCBFwmyFB-FcFzh8VMDOwUAXI,30794
|
|
5
|
+
owid/datautils/decorators.py,sha256=GD_tk13b6pAjHCvp8KQPZ3zB_XhxzR3Xrfo4LYfDMew,3842
|
|
6
|
+
owid/datautils/ui.py,sha256=pNq0MN1ym7jv-dSQI0AyUo3PZZVHfGERpPEOlOxZKZo,656
|
|
7
|
+
owid/datautils/web.py,sha256=Fd62EIFIXD1PWrEaXVa-91idKtEG_SaImCRUI3kedhU,3522
|
|
8
|
+
owid/datautils/format/__init__.py,sha256=L3lFpSnMs9XyKgymLQF5HILI2q5U4TvhVPh9jNFKXDM,151
|
|
9
|
+
owid/datautils/format/numbers.py,sha256=kJBkN-boLvaOTzDynG8oE5nzsplaaXHRUMFUzMRr6Uc,7970
|
|
10
|
+
owid/datautils/google/__init__.py,sha256=iCGh97I6JhM58Y-MpHeS0zeiRUpC1c6N4FW8A3atWSY,101
|
|
11
|
+
owid/datautils/google/api.py,sha256=xPnqwBcP0Qr7hbxqyN59x0a8Sk4KhsuFrluxAfUpg5M,4380
|
|
12
|
+
owid/datautils/google/config.py,sha256=_6zLgKL-mt7AVesJmvTX0HuuwripmW8Mhwund4YmmRk,4195
|
|
13
|
+
owid/datautils/google/sheets.py,sha256=U45Kfd_--cXLTGXqX9W60VjYTN1Q1ztEKg2zs-1oNy0,4933
|
|
14
|
+
owid/datautils/io/__init__.py,sha256=O15qZVYnA28cznkxKjZX0kPH2Rfaof9-lxtO4ho5w6A,363
|
|
15
|
+
owid/datautils/io/archive.py,sha256=miF3nD8lWA0l59AEHoppGYP_eVc7cYCYE7d2VHlniJs,3087
|
|
16
|
+
owid/datautils/io/df.py,sha256=nx0jJRWsBtHpopKdJ4fKDd1X4oF2y2iUA1_2wyPgQZQ,6526
|
|
17
|
+
owid/datautils/io/json.py,sha256=QlK624POl3fmngi0rnPG_HY2-CSPl1JISYBrzSdcEDs,2334
|
|
18
|
+
owid_datautils-0.6.1.dist-info/METADATA,sha256=wsjXD3JgXdc1E0q89xQQxfyE9lZC3cRPkqZQxCR2oho,652
|
|
19
|
+
owid_datautils-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
20
|
+
owid_datautils-0.6.1.dist-info/licenses/LICENSE,sha256=h3W2OEglx2jaOj145JoWav12nVTK1kHQm_vIsHayHyc,1079
|
|
21
|
+
owid_datautils-0.6.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2021 Global Change Data Lab
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|