garf-core 0.0.11__tar.gz → 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {garf_core-0.0.11 → garf_core-0.1.0}/PKG-INFO +8 -9
- {garf_core-0.0.11 → garf_core-0.1.0}/README.md +5 -7
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core/__init__.py +1 -1
- garf_core-0.1.0/garf_core/api_clients.py +172 -0
- garf_core-0.1.0/garf_core/fetchers/__init__.py +23 -0
- garf_core-0.1.0/garf_core/fetchers/fake.py +74 -0
- garf_core-0.1.0/garf_core/fetchers/rest.py +58 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core/query_editor.py +49 -25
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core/report_fetcher.py +34 -35
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core.egg-info/PKG-INFO +8 -9
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core.egg-info/SOURCES.txt +4 -1
- garf_core-0.1.0/garf_core.egg-info/entry_points.txt +3 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core.egg-info/requires.txt +1 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/pyproject.toml +4 -1
- garf_core-0.0.11/garf_core/api_clients.py +0 -86
- garf_core-0.0.11/garf_core.egg-info/entry_points.txt +0 -2
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core/base_query.py +0 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core/exceptions.py +0 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core/parsers.py +0 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core/report.py +0 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core.egg-info/dependency_links.txt +0 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/garf_core.egg-info/top_level.txt +0 -0
- {garf_core-0.0.11 → garf_core-0.1.0}/setup.cfg +0 -0
@@ -1,8 +1,8 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: garf-core
|
3
|
-
Version: 0.0
|
3
|
+
Version: 0.1.0
|
4
4
|
Summary: Abstracts fetching data from API based on provided SQL-like query.
|
5
|
-
Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>
|
5
|
+
Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
|
6
6
|
License: Apache 2.0
|
7
7
|
Classifier: Programming Language :: Python :: 3 :: Only
|
8
8
|
Classifier: Programming Language :: Python :: 3.8
|
@@ -22,6 +22,7 @@ Requires-Dist: jinja2
|
|
22
22
|
Requires-Dist: typing-extensions
|
23
23
|
Requires-Dist: requests
|
24
24
|
Requires-Dist: pyyaml
|
25
|
+
Requires-Dist: pydantic
|
25
26
|
Provides-Extra: pandas
|
26
27
|
Requires-Dist: pandas; extra == "pandas"
|
27
28
|
Provides-Extra: polars
|
@@ -39,15 +40,13 @@ Requires-Dist: garf-core[pandas,polars]; extra == "all"
|
|
39
40
|
|
40
41
|
These abstractions are designed to be as modular and simple as possible:
|
41
42
|
|
42
|
-
* `BaseApiClient` - an interface for connecting to APIs.
|
43
|
-
* `
|
43
|
+
* `BaseApiClient` - an interface for connecting to APIs. Check [default implementations](docs/builtin-functionality.md#apiclients)
|
44
|
+
* `BaseParser` - an interface to parse results from the API. Check [default implementations](docs/builtin-functionality.md#parsers)
|
45
|
+
* `ApiReportFetcher` - responsible for fetching and parsing data from reporting API. [Default implementations](docs/builtin-functionality.md#apireportfetchers)
|
46
|
+
|
44
47
|
* `QuerySpecification` - parsed SQL-query into various elements.
|
45
|
-
* `
|
46
|
-
* `ListParser` - returns results from API as a raw list.
|
47
|
-
* `DictParser` - returns results from API as a formatted dict.
|
48
|
-
* `NumericDictParser` - returns results from API as a formatted dict with converted numeric values.
|
48
|
+
* `BaseQuery` - protocol for all class based queries.
|
49
49
|
* `GarfReport` - contains data from API in a format that is easy to write and interact with.
|
50
|
-
* `ApiReportFetcher` - responsible for fetching and parsing data from reporting API.
|
51
50
|
|
52
51
|
## Installation
|
53
52
|
|
@@ -8,15 +8,13 @@
|
|
8
8
|
|
9
9
|
These abstractions are designed to be as modular and simple as possible:
|
10
10
|
|
11
|
-
* `BaseApiClient` - an interface for connecting to APIs.
|
12
|
-
* `
|
11
|
+
* `BaseApiClient` - an interface for connecting to APIs. Check [default implementations](docs/builtin-functionality.md#apiclients)
|
12
|
+
* `BaseParser` - an interface to parse results from the API. Check [default implementations](docs/builtin-functionality.md#parsers)
|
13
|
+
* `ApiReportFetcher` - responsible for fetching and parsing data from reporting API. [Default implementations](docs/builtin-functionality.md#apireportfetchers)
|
14
|
+
|
13
15
|
* `QuerySpecification` - parsed SQL-query into various elements.
|
14
|
-
* `
|
15
|
-
* `ListParser` - returns results from API as a raw list.
|
16
|
-
* `DictParser` - returns results from API as a formatted dict.
|
17
|
-
* `NumericDictParser` - returns results from API as a formatted dict with converted numeric values.
|
16
|
+
* `BaseQuery` - protocol for all class based queries.
|
18
17
|
* `GarfReport` - contains data from API in a format that is easy to write and interact with.
|
19
|
-
* `ApiReportFetcher` - responsible for fetching and parsing data from reporting API.
|
20
18
|
|
21
19
|
## Installation
|
22
20
|
|
@@ -0,0 +1,172 @@
|
|
1
|
+
# Copyright 2025 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
"""Module for defining client to interact with API."""
|
15
|
+
|
16
|
+
from __future__ import annotations
|
17
|
+
|
18
|
+
import abc
|
19
|
+
import contextlib
|
20
|
+
import csv
|
21
|
+
import dataclasses
|
22
|
+
import json
|
23
|
+
import os
|
24
|
+
import pathlib
|
25
|
+
from collections.abc import Sequence
|
26
|
+
from typing import Any
|
27
|
+
|
28
|
+
import requests
|
29
|
+
from typing_extensions import override
|
30
|
+
|
31
|
+
from garf_core import exceptions
|
32
|
+
|
33
|
+
|
34
|
+
@dataclasses.dataclass
|
35
|
+
class GarfApiRequest:
|
36
|
+
"""Base class for specifying request."""
|
37
|
+
|
38
|
+
|
39
|
+
@dataclasses.dataclass
|
40
|
+
class GarfApiResponse:
|
41
|
+
"""Base class for specifying response."""
|
42
|
+
|
43
|
+
results: list
|
44
|
+
|
45
|
+
|
46
|
+
class GarfApiError(exceptions.GarfError):
|
47
|
+
"""API specific exception."""
|
48
|
+
|
49
|
+
|
50
|
+
class BaseClient(abc.ABC):
|
51
|
+
"""Base API client class."""
|
52
|
+
|
53
|
+
@abc.abstractmethod
|
54
|
+
def get_response(
|
55
|
+
self, request: GarfApiRequest = GarfApiRequest(), **kwargs: str
|
56
|
+
) -> GarfApiResponse:
|
57
|
+
"""Method for getting response."""
|
58
|
+
|
59
|
+
|
60
|
+
class RestApiClient(BaseClient):
|
61
|
+
"""Specifies REST client."""
|
62
|
+
|
63
|
+
OK = 200
|
64
|
+
|
65
|
+
def __init__(self, endpoint: str, **kwargs: str) -> None:
|
66
|
+
"""Initializes RestApiClient."""
|
67
|
+
self.endpoint = endpoint
|
68
|
+
self.query_args = kwargs
|
69
|
+
|
70
|
+
@override
|
71
|
+
def get_response(
|
72
|
+
self, request: GarfApiRequest = GarfApiRequest(), **kwargs: str
|
73
|
+
) -> GarfApiResponse:
|
74
|
+
response = requests.get(f'{self.endpoint}/{request.resource_name}')
|
75
|
+
if response.status_code == self.OK:
|
76
|
+
return GarfApiResponse(response.json())
|
77
|
+
raise GarfApiError('Failed to get data from API')
|
78
|
+
|
79
|
+
|
80
|
+
class FakeApiClient(BaseClient):
|
81
|
+
"""Fake class for specifying API client."""
|
82
|
+
|
83
|
+
def __init__(self, results: Sequence[dict[str, Any]], **kwargs: str) -> None:
|
84
|
+
"""Initializes FakeApiClient."""
|
85
|
+
self.results = list(results)
|
86
|
+
self.kwargs = kwargs
|
87
|
+
|
88
|
+
@override
|
89
|
+
def get_response(
|
90
|
+
self, request: GarfApiRequest = GarfApiRequest(), **kwargs: str
|
91
|
+
) -> GarfApiResponse:
|
92
|
+
del request
|
93
|
+
return GarfApiResponse(results=self.results)
|
94
|
+
|
95
|
+
@classmethod
|
96
|
+
def from_file(cls, file_location: str | os.PathLike[str]) -> FakeApiClient:
|
97
|
+
"""Initializes FakeApiClient from json or csv files.
|
98
|
+
|
99
|
+
Args:
|
100
|
+
file_location: Path of file with data.
|
101
|
+
|
102
|
+
Returns:
|
103
|
+
Initialized client.
|
104
|
+
|
105
|
+
Raises:
|
106
|
+
GarfApiError: When file with unsupported extension is provided.
|
107
|
+
"""
|
108
|
+
if str(file_location).endswith('.json'):
|
109
|
+
return FakeApiClient.from_json(file_location)
|
110
|
+
if str(file_location).endswith('.csv'):
|
111
|
+
return FakeApiClient.from_csv(file_location)
|
112
|
+
raise GarfApiError(
|
113
|
+
'Unsupported file extension, only csv and json are supported.'
|
114
|
+
)
|
115
|
+
|
116
|
+
@classmethod
|
117
|
+
def from_json(cls, file_location: str | os.PathLike[str]) -> FakeApiClient:
|
118
|
+
"""Initializes FakeApiClient from json file.
|
119
|
+
|
120
|
+
Args:
|
121
|
+
file_location: Path of file with data.
|
122
|
+
|
123
|
+
Returns:
|
124
|
+
Initialized client.
|
125
|
+
|
126
|
+
Raises:
|
127
|
+
GarfApiError: When file with data not found.
|
128
|
+
"""
|
129
|
+
try:
|
130
|
+
with pathlib.Path.open(file_location, 'r', encoding='utf-8') as f:
|
131
|
+
data = json.load(f)
|
132
|
+
return FakeApiClient(data)
|
133
|
+
except FileNotFoundError as e:
|
134
|
+
raise GarfApiError(f'Failed to open {file_location}') from e
|
135
|
+
|
136
|
+
@classmethod
|
137
|
+
def from_csv(cls, file_location: str | os.PathLike[str]) -> FakeApiClient:
|
138
|
+
"""Initializes FakeApiClient from csv file.
|
139
|
+
|
140
|
+
Args:
|
141
|
+
file_location: Path of file with data.
|
142
|
+
|
143
|
+
Returns:
|
144
|
+
Initialized client.
|
145
|
+
|
146
|
+
Raises:
|
147
|
+
GarfApiError: When file with data not found.
|
148
|
+
"""
|
149
|
+
try:
|
150
|
+
with pathlib.Path.open(file_location, 'r', encoding='utf-8') as f:
|
151
|
+
reader = csv.DictReader(f)
|
152
|
+
data = []
|
153
|
+
for row in reader:
|
154
|
+
data.append(
|
155
|
+
{key: _field_converter(value) for key, value in row.items()}
|
156
|
+
)
|
157
|
+
return FakeApiClient(data)
|
158
|
+
except FileNotFoundError as e:
|
159
|
+
raise GarfApiError(f'Failed to open {file_location}') from e
|
160
|
+
|
161
|
+
|
162
|
+
def _field_converter(field: str):
|
163
|
+
if isinstance(field, str) and (lower_field := field.lower()) in (
|
164
|
+
'true',
|
165
|
+
'false',
|
166
|
+
):
|
167
|
+
return lower_field == 'true'
|
168
|
+
with contextlib.suppress(ValueError):
|
169
|
+
return int(field)
|
170
|
+
with contextlib.suppress(ValueError):
|
171
|
+
return float(field)
|
172
|
+
return field
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Copyright 2025 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
"""Built-in fetchers."""
|
16
|
+
|
17
|
+
from garf_core.fetchers.fake import FakeApiReportFetcher
|
18
|
+
from garf_core.fetchers.rest import RestApiReportFetcher
|
19
|
+
|
20
|
+
__all__ = [
|
21
|
+
'FakeApiReportFetcher',
|
22
|
+
'RestApiReportFetcher',
|
23
|
+
]
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# Copyright 2025 Google LLf
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
# pylint: disable=C0330, g-bad-import-order, g-multiple-import
|
16
|
+
|
17
|
+
"""Getting fake data from memory or a file."""
|
18
|
+
|
19
|
+
from __future__ import annotations
|
20
|
+
|
21
|
+
import logging
|
22
|
+
import os
|
23
|
+
|
24
|
+
from garf_core import (
|
25
|
+
api_clients,
|
26
|
+
parsers,
|
27
|
+
query_editor,
|
28
|
+
report_fetcher,
|
29
|
+
)
|
30
|
+
|
31
|
+
logger = logging.getLogger(__name__)
|
32
|
+
|
33
|
+
|
34
|
+
class FakeApiReportFetcher(report_fetcher.ApiReportFetcher):
|
35
|
+
"""Returns simulated data."""
|
36
|
+
|
37
|
+
def __init__(
|
38
|
+
self,
|
39
|
+
data: list[dict[str, parsers.ApiRowElement]] | None = None,
|
40
|
+
parser: parsers.BaseParser = parsers.DictParser,
|
41
|
+
query_specification_builder: query_editor.QuerySpecification = (
|
42
|
+
query_editor.QuerySpecification
|
43
|
+
),
|
44
|
+
data_location: str | os.PathLike[str] | None = None,
|
45
|
+
csv_location: str | os.PathLike[str] | None = None,
|
46
|
+
json_location: str | os.PathLike[str] | None = None,
|
47
|
+
**kwargs: str,
|
48
|
+
) -> None:
|
49
|
+
if not data and not (
|
50
|
+
data_location := json_location or csv_location or data_location
|
51
|
+
):
|
52
|
+
raise report_fetcher.ApiReportFetcherError(
|
53
|
+
'Missing fake data for the fetcher.'
|
54
|
+
)
|
55
|
+
api_client = (
|
56
|
+
api_clients.FakeApiClient(data)
|
57
|
+
if data
|
58
|
+
else api_clients.FakeApiClient.from_file(data_location)
|
59
|
+
)
|
60
|
+
super().__init__(api_client, parser, query_specification_builder, **kwargs)
|
61
|
+
|
62
|
+
@classmethod
|
63
|
+
def from_csv(
|
64
|
+
cls, file_location: str | os.PathLike[str]
|
65
|
+
) -> FakeApiReportFetcher:
|
66
|
+
"""Initialized FakeApiReportFetcher from a csv file."""
|
67
|
+
return FakeApiReportFetcher(csv_location=file_location)
|
68
|
+
|
69
|
+
@classmethod
|
70
|
+
def from_json(
|
71
|
+
cls, file_location: str | os.PathLike[str]
|
72
|
+
) -> FakeApiReportFetcher:
|
73
|
+
"""Initialized FakeApiReportFetcher from a json file."""
|
74
|
+
return FakeApiReportFetcher(json_location=file_location)
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# Copyright 2025 Google LLf
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
# pylint: disable=C0330, g-bad-import-order, g-multiple-import
|
16
|
+
|
17
|
+
"""Module for getting data from Rest APIs based on a query."""
|
18
|
+
|
19
|
+
from __future__ import annotations
|
20
|
+
|
21
|
+
import logging
|
22
|
+
|
23
|
+
from garf_core import (
|
24
|
+
api_clients,
|
25
|
+
parsers,
|
26
|
+
query_editor,
|
27
|
+
report_fetcher,
|
28
|
+
)
|
29
|
+
|
30
|
+
logger = logging.getLogger(__name__)
|
31
|
+
|
32
|
+
|
33
|
+
class RestApiReportFetcher(report_fetcher.ApiReportFetcher):
|
34
|
+
"""Fetches data from an REST API endpoint.
|
35
|
+
|
36
|
+
Attributes:
|
37
|
+
api_client: Initialized RestApiClient.
|
38
|
+
parser: Type of parser to convert API response.
|
39
|
+
"""
|
40
|
+
|
41
|
+
def __init__(
|
42
|
+
self,
|
43
|
+
endpoint: str,
|
44
|
+
parser: parsers.BaseParser = parsers.DictParser,
|
45
|
+
query_specification_builder: query_editor.QuerySpecification = (
|
46
|
+
query_editor.QuerySpecification
|
47
|
+
),
|
48
|
+
**kwargs: str,
|
49
|
+
) -> None:
|
50
|
+
"""Instantiates RestApiReportFetcher.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
endpoint: URL of API endpoint.
|
54
|
+
parser: Type of parser to convert API response.
|
55
|
+
query_specification_builder: Class to perform query parsing.
|
56
|
+
"""
|
57
|
+
api_client = api_clients.RestApiClient(endpoint)
|
58
|
+
super().__init__(api_client, parser, query_specification_builder, **kwargs)
|
@@ -23,11 +23,27 @@ import re
|
|
23
23
|
from typing import Generator
|
24
24
|
|
25
25
|
import jinja2
|
26
|
+
import pydantic
|
26
27
|
from dateutil import relativedelta
|
27
28
|
from typing_extensions import Self
|
28
29
|
|
29
30
|
from garf_core import exceptions
|
30
31
|
|
32
|
+
QueryParameters = dict[str, str | float | int]
|
33
|
+
|
34
|
+
|
35
|
+
class GarfQueryParameters(pydantic.BaseModel):
|
36
|
+
"""Parameters for dynamically changing text of a query."""
|
37
|
+
|
38
|
+
macro: QueryParameters | None = None
|
39
|
+
template: QueryParameters | None = None
|
40
|
+
|
41
|
+
def model_post_init(self, __context__) -> None:
|
42
|
+
if self.macro is None:
|
43
|
+
self.macro = {}
|
44
|
+
if self.template is None:
|
45
|
+
self.template = {}
|
46
|
+
|
31
47
|
|
32
48
|
class GarfQueryError(exceptions.GarfError):
|
33
49
|
"""Base exception for Garf queries."""
|
@@ -53,9 +69,12 @@ class GarfResourceError(GarfQueryError):
|
|
53
69
|
"""Specifies incorrect resource name in the query."""
|
54
70
|
|
55
71
|
|
56
|
-
|
57
|
-
|
58
|
-
|
72
|
+
class GarfBuiltInQueryError(GarfQueryError):
|
73
|
+
"""Specifies non-existing builtin query."""
|
74
|
+
|
75
|
+
|
76
|
+
class ProcessedField(pydantic.BaseModel):
|
77
|
+
"""Sore field with its customizers.
|
59
78
|
|
60
79
|
Attributes:
|
61
80
|
field: Extractable field.
|
@@ -136,7 +155,7 @@ class VirtualColumn:
|
|
136
155
|
substitute_expression: str | None = None
|
137
156
|
|
138
157
|
@classmethod
|
139
|
-
def from_raw(cls, field: str, macros:
|
158
|
+
def from_raw(cls, field: str, macros: QueryParameters) -> VirtualColumn:
|
140
159
|
"""Converts a field to virtual column."""
|
141
160
|
if field.isdigit():
|
142
161
|
field = int(field)
|
@@ -189,16 +208,16 @@ class ExtractedLineElements:
|
|
189
208
|
|
190
209
|
@classmethod
|
191
210
|
def from_query_line(
|
192
|
-
cls,
|
211
|
+
cls,
|
212
|
+
line: str,
|
213
|
+
macros: QueryParameters | None = None,
|
193
214
|
) -> ExtractedLineElements:
|
215
|
+
if macros is None:
|
216
|
+
macros = {}
|
194
217
|
field, *alias = re.split(' [Aa][Ss] ', line)
|
195
218
|
processed_field = ProcessedField.from_raw(field)
|
196
219
|
field = processed_field.field
|
197
|
-
if field
|
198
|
-
# if field.is_valid:
|
199
|
-
virtual_column = None
|
200
|
-
else:
|
201
|
-
virtual_column = VirtualColumn.from_raw(field, macros)
|
220
|
+
virtual_column = None if field else VirtualColumn.from_raw(field, macros)
|
202
221
|
if alias and processed_field.customizer_type:
|
203
222
|
customizer = {
|
204
223
|
'type': processed_field.customizer_type,
|
@@ -300,11 +319,11 @@ class CommonParametersMixin:
|
|
300
319
|
|
301
320
|
class TemplateProcessorMixin:
|
302
321
|
def replace_params_template(
|
303
|
-
self, query_text: str, params:
|
322
|
+
self, query_text: str, params: GarfQueryParameters | None = None
|
304
323
|
) -> str:
|
305
324
|
logging.debug('Original query text:\n%s', query_text)
|
306
325
|
if params:
|
307
|
-
if templates := params.
|
326
|
+
if templates := params.template:
|
308
327
|
query_templates = {
|
309
328
|
name: value for name, value in templates.items() if name in query_text
|
310
329
|
}
|
@@ -315,7 +334,7 @@ class TemplateProcessorMixin:
|
|
315
334
|
query_text = self.expand_jinja(query_text, {})
|
316
335
|
else:
|
317
336
|
query_text = self.expand_jinja(query_text, {})
|
318
|
-
if macros := params.
|
337
|
+
if macros := params.macro:
|
319
338
|
query_text = query_text.format(**macros)
|
320
339
|
logging.debug('Query text after macro substitution:\n%s', query_text)
|
321
340
|
else:
|
@@ -323,7 +342,7 @@ class TemplateProcessorMixin:
|
|
323
342
|
return query_text
|
324
343
|
|
325
344
|
def expand_jinja(
|
326
|
-
self, query_text: str, template_params:
|
345
|
+
self, query_text: str, template_params: QueryParameters | None = None
|
327
346
|
) -> str:
|
328
347
|
file_inclusions = ('% include', '% import', '% extend')
|
329
348
|
if any(file_inclusion in query_text for file_inclusion in file_inclusions):
|
@@ -360,7 +379,7 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
|
|
360
379
|
self,
|
361
380
|
text: str,
|
362
381
|
title: str | None = None,
|
363
|
-
args:
|
382
|
+
args: GarfQueryParameters | None = GarfQueryParameters(),
|
364
383
|
**kwargs: str,
|
365
384
|
) -> None:
|
366
385
|
"""Instantiates QuerySpecification based on text, title and optional args.
|
@@ -371,35 +390,40 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
|
|
371
390
|
args: Optional parameters to be dynamically injected into query text.
|
372
391
|
api_version: Version of Google Ads API.
|
373
392
|
"""
|
374
|
-
self.args = args or
|
393
|
+
self.args = args or GarfQueryParameters()
|
375
394
|
self.query = BaseQueryElements(title=title, text=text)
|
376
395
|
|
377
396
|
@property
|
378
|
-
def macros(self) ->
|
397
|
+
def macros(self) -> QueryParameters:
|
379
398
|
"""Returns macros with injected common parameters."""
|
380
399
|
common_params = dict(self.common_params)
|
381
|
-
if macros := self.args.
|
400
|
+
if macros := self.args.macro:
|
382
401
|
common_params.update(macros)
|
383
402
|
return common_params
|
384
403
|
|
385
404
|
def generate(self) -> BaseQueryElements:
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
405
|
+
self.remove_comments().expand().extract_resource_name()
|
406
|
+
if self.query.resource_name.startswith('builtin'):
|
407
|
+
return BaseQueryElements(
|
408
|
+
title=self.query.resource_name.replace('builtin.', ''),
|
409
|
+
text=self.query.text,
|
410
|
+
resource_name=self.query.resource_name,
|
411
|
+
is_builtin_query=True,
|
412
|
+
)
|
413
|
+
(
|
414
|
+
self.remove_trailing_comma()
|
391
415
|
.extract_fields()
|
392
416
|
.extract_filters()
|
393
417
|
.extract_sorts()
|
394
418
|
.extract_column_names()
|
395
419
|
.extract_virtual_columns()
|
396
420
|
.extract_customizers()
|
397
|
-
.query
|
398
421
|
)
|
422
|
+
return self.query
|
399
423
|
|
400
424
|
def expand(self) -> Self:
|
401
425
|
"""Applies necessary transformations to query."""
|
402
|
-
query_text = self.expand_jinja(self.query.text, self.args.
|
426
|
+
query_text = self.expand_jinja(self.query.text, self.args.template)
|
403
427
|
try:
|
404
428
|
self.query.text = query_text.format(**self.macros).strip()
|
405
429
|
except KeyError as e:
|
@@ -11,20 +11,22 @@
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
|
+
|
15
|
+
# pylint: disable=C0330, g-bad-import-order, g-multiple-import
|
16
|
+
|
14
17
|
"""Module for getting data from API based on a query.
|
15
18
|
|
16
|
-
ApiReportFetcher
|
17
|
-
and returning GarfReport.
|
19
|
+
ApiReportFetcher fetches data from API, parses it and returns GarfReport.
|
18
20
|
"""
|
19
|
-
# pylint: disable=C0330, g-bad-import-order, g-multiple-import
|
20
21
|
|
21
22
|
from __future__ import annotations
|
22
23
|
|
23
24
|
import logging
|
24
|
-
from typing import
|
25
|
+
from typing import Callable
|
25
26
|
|
26
27
|
from garf_core import (
|
27
28
|
api_clients,
|
29
|
+
exceptions,
|
28
30
|
parsers,
|
29
31
|
query_editor,
|
30
32
|
report,
|
@@ -33,6 +35,10 @@ from garf_core import (
|
|
33
35
|
logger = logging.getLogger(__name__)
|
34
36
|
|
35
37
|
|
38
|
+
class ApiReportFetcherError(exceptions.GarfError):
|
39
|
+
"""Base exception for all ApiReportFetchers."""
|
40
|
+
|
41
|
+
|
36
42
|
class ApiReportFetcher:
|
37
43
|
"""Class responsible for getting data from report API.
|
38
44
|
|
@@ -45,10 +51,12 @@ class ApiReportFetcher:
|
|
45
51
|
def __init__(
|
46
52
|
self,
|
47
53
|
api_client: api_clients.BaseApiClient,
|
48
|
-
parser: parsers.BaseParser = parsers.
|
54
|
+
parser: parsers.BaseParser = parsers.DictParser,
|
49
55
|
query_specification_builder: query_editor.QuerySpecification = (
|
50
56
|
query_editor.QuerySpecification
|
51
57
|
),
|
58
|
+
builtin_queries: dict[str, Callable[[ApiReportFetcher], report.GarfReport]]
|
59
|
+
| None = None,
|
52
60
|
**kwargs: str,
|
53
61
|
) -> None:
|
54
62
|
"""Instantiates ApiReportFetcher based on provided api client.
|
@@ -57,16 +65,26 @@ class ApiReportFetcher:
|
|
57
65
|
api_client: Instantiated api client.
|
58
66
|
parser: Type of parser to convert API response.
|
59
67
|
query_specification_builder: Class to perform query parsing.
|
68
|
+
builtin_queries:
|
69
|
+
Mapping between query name and function for generating GarfReport.
|
60
70
|
"""
|
61
71
|
self.api_client = api_client
|
62
72
|
self.parser = parser()
|
63
73
|
self.query_specification_builder = query_specification_builder
|
64
74
|
self.query_args = kwargs
|
75
|
+
self.builtin_queries = builtin_queries or {}
|
76
|
+
|
77
|
+
def add_builtin_queries(
|
78
|
+
self,
|
79
|
+
builtin_queries: dict[str, Callable[[ApiReportFetcher], report.GarfReport]],
|
80
|
+
) -> None:
|
81
|
+
"""Adds new built-in queries to the fetcher."""
|
82
|
+
self.builtin_queries.update(builtin_queries)
|
65
83
|
|
66
84
|
async def afetch(
|
67
85
|
self,
|
68
86
|
query_specification: str | query_editor.QueryElements,
|
69
|
-
args:
|
87
|
+
args: query_editor.GarfQueryParameters | None = None,
|
70
88
|
**kwargs: str,
|
71
89
|
) -> report.GarfReport:
|
72
90
|
"""Asynchronously fetches data from API based on query_specification.
|
@@ -84,7 +102,7 @@ class ApiReportFetcher:
|
|
84
102
|
def fetch(
|
85
103
|
self,
|
86
104
|
query_specification: str | query_editor.QuerySpecification,
|
87
|
-
args:
|
105
|
+
args: query_editor.GarfQueryParameters | None = None,
|
88
106
|
**kwargs: str,
|
89
107
|
) -> report.GarfReport:
|
90
108
|
"""Fetches data from API based on query_specification.
|
@@ -101,42 +119,23 @@ class ApiReportFetcher:
|
|
101
119
|
GarfExecutorException:
|
102
120
|
When customer_ids are not provided or API returned error.
|
103
121
|
"""
|
122
|
+
if args is None:
|
123
|
+
args = query_editor.GarfQueryParameters()
|
104
124
|
if not isinstance(query_specification, query_editor.QuerySpecification):
|
105
125
|
query_specification = self.query_specification_builder(
|
106
126
|
text=str(query_specification),
|
107
127
|
args=args,
|
108
128
|
)
|
109
129
|
query = query_specification.generate()
|
130
|
+
if query.is_builtin_query:
|
131
|
+
if not (builtin_report := self.builtin_queries.get(query.title)):
|
132
|
+
raise query_editor.GarfBuiltInQueryError(
|
133
|
+
f'Cannot find the built-in query "{query.title}"'
|
134
|
+
)
|
135
|
+
return builtin_report(self, **kwargs)
|
136
|
+
|
110
137
|
response = self.api_client.get_response(query, **kwargs)
|
111
138
|
parsed_response = self.parser.parse_response(response, query)
|
112
139
|
return report.GarfReport(
|
113
140
|
results=parsed_response, column_names=query.column_names
|
114
141
|
)
|
115
|
-
|
116
|
-
|
117
|
-
class RestApiReportFetcher(ApiReportFetcher):
|
118
|
-
"""Fetches data from an REST API endpoint.
|
119
|
-
|
120
|
-
Attributes:
|
121
|
-
api_client: Initialized RestApiClient.
|
122
|
-
parser: Type of parser to convert API response.
|
123
|
-
"""
|
124
|
-
|
125
|
-
def __init__(
|
126
|
-
self,
|
127
|
-
endpoint: str,
|
128
|
-
parser: parsers.BaseParser = parsers.DictParser,
|
129
|
-
query_specification_builder: query_editor.QuerySpecification = (
|
130
|
-
query_editor.QuerySpecification
|
131
|
-
),
|
132
|
-
**kwargs: str,
|
133
|
-
) -> None:
|
134
|
-
"""Instantiates RestApiReportFetcher.
|
135
|
-
|
136
|
-
Args:
|
137
|
-
endpoint: URL of API endpoint.
|
138
|
-
parser: Type of parser to convert API response.
|
139
|
-
query_specification_builder: Class to perform query parsing.
|
140
|
-
"""
|
141
|
-
api_client = api_clients.RestApiClient(endpoint)
|
142
|
-
super().__init__(api_client, parser, query_specification_builder, **kwargs)
|
@@ -1,8 +1,8 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: garf-core
|
3
|
-
Version: 0.0
|
3
|
+
Version: 0.1.0
|
4
4
|
Summary: Abstracts fetching data from API based on provided SQL-like query.
|
5
|
-
Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>
|
5
|
+
Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
|
6
6
|
License: Apache 2.0
|
7
7
|
Classifier: Programming Language :: Python :: 3 :: Only
|
8
8
|
Classifier: Programming Language :: Python :: 3.8
|
@@ -22,6 +22,7 @@ Requires-Dist: jinja2
|
|
22
22
|
Requires-Dist: typing-extensions
|
23
23
|
Requires-Dist: requests
|
24
24
|
Requires-Dist: pyyaml
|
25
|
+
Requires-Dist: pydantic
|
25
26
|
Provides-Extra: pandas
|
26
27
|
Requires-Dist: pandas; extra == "pandas"
|
27
28
|
Provides-Extra: polars
|
@@ -39,15 +40,13 @@ Requires-Dist: garf-core[pandas,polars]; extra == "all"
|
|
39
40
|
|
40
41
|
These abstractions are designed to be as modular and simple as possible:
|
41
42
|
|
42
|
-
* `BaseApiClient` - an interface for connecting to APIs.
|
43
|
-
* `
|
43
|
+
* `BaseApiClient` - an interface for connecting to APIs. Check [default implementations](docs/builtin-functionality.md#apiclients)
|
44
|
+
* `BaseParser` - an interface to parse results from the API. Check [default implementations](docs/builtin-functionality.md#parsers)
|
45
|
+
* `ApiReportFetcher` - responsible for fetching and parsing data from reporting API. [Default implementations](docs/builtin-functionality.md#apireportfetchers)
|
46
|
+
|
44
47
|
* `QuerySpecification` - parsed SQL-query into various elements.
|
45
|
-
* `
|
46
|
-
* `ListParser` - returns results from API as a raw list.
|
47
|
-
* `DictParser` - returns results from API as a formatted dict.
|
48
|
-
* `NumericDictParser` - returns results from API as a formatted dict with converted numeric values.
|
48
|
+
* `BaseQuery` - protocol for all class based queries.
|
49
49
|
* `GarfReport` - contains data from API in a format that is easy to write and interact with.
|
50
|
-
* `ApiReportFetcher` - responsible for fetching and parsing data from reporting API.
|
51
50
|
|
52
51
|
## Installation
|
53
52
|
|
@@ -13,4 +13,7 @@ garf_core.egg-info/SOURCES.txt
|
|
13
13
|
garf_core.egg-info/dependency_links.txt
|
14
14
|
garf_core.egg-info/entry_points.txt
|
15
15
|
garf_core.egg-info/requires.txt
|
16
|
-
garf_core.egg-info/top_level.txt
|
16
|
+
garf_core.egg-info/top_level.txt
|
17
|
+
garf_core/fetchers/__init__.py
|
18
|
+
garf_core/fetchers/fake.py
|
19
|
+
garf_core/fetchers/rest.py
|
@@ -10,9 +10,11 @@ dependencies = [
|
|
10
10
|
"typing-extensions",
|
11
11
|
"requests",
|
12
12
|
"pyyaml",
|
13
|
+
"pydantic",
|
13
14
|
]
|
14
15
|
authors = [
|
15
16
|
{name = "Google Inc. (gTech gPS CSE team)", email = "no-reply@google.com"},
|
17
|
+
{name = "Andrei Markin", email = "andrey.markin.ppc@gmail.com"},
|
16
18
|
]
|
17
19
|
requires-python = ">=3.8"
|
18
20
|
description = "Abstracts fetching data from API based on provided SQL-like query."
|
@@ -37,7 +39,8 @@ dynamic=["version"]
|
|
37
39
|
version = {attr = "garf_core.__version__"}
|
38
40
|
|
39
41
|
[project.entry-points.garf]
|
40
|
-
rest = "garf_core.
|
42
|
+
rest = "garf_core.fetchers.rest"
|
43
|
+
fake = "garf_core.fetchers.fake"
|
41
44
|
|
42
45
|
[project.optional-dependencies]
|
43
46
|
pandas=[
|
@@ -1,86 +0,0 @@
|
|
1
|
-
# Copyright 2025 Google LLC
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
"""Module for defining client to interact with API."""
|
15
|
-
|
16
|
-
from __future__ import annotations
|
17
|
-
|
18
|
-
import abc
|
19
|
-
import dataclasses
|
20
|
-
from collections.abc import Sequence
|
21
|
-
|
22
|
-
import requests
|
23
|
-
from typing_extensions import override
|
24
|
-
|
25
|
-
from garf_core import exceptions
|
26
|
-
|
27
|
-
|
28
|
-
@dataclasses.dataclass
|
29
|
-
class GarfApiRequest:
|
30
|
-
"""Base class for specifying request."""
|
31
|
-
|
32
|
-
|
33
|
-
@dataclasses.dataclass
|
34
|
-
class GarfApiResponse:
|
35
|
-
"""Base class for specifying response."""
|
36
|
-
|
37
|
-
results: list
|
38
|
-
|
39
|
-
|
40
|
-
class GarfApiError(exceptions.GarfError):
|
41
|
-
"""API specific exception."""
|
42
|
-
|
43
|
-
|
44
|
-
class BaseClient(abc.ABC):
|
45
|
-
"""Base API client class."""
|
46
|
-
|
47
|
-
@abc.abstractmethod
|
48
|
-
def get_response(
|
49
|
-
self, request: GarfApiRequest = GarfApiRequest(), **kwargs: str
|
50
|
-
) -> GarfApiResponse:
|
51
|
-
"""Method for getting response."""
|
52
|
-
|
53
|
-
|
54
|
-
class RestApiClient(BaseClient):
|
55
|
-
"""Specifies REST client."""
|
56
|
-
|
57
|
-
OK = 200
|
58
|
-
|
59
|
-
def __init__(self, endpoint: str, **kwargs: str) -> None:
|
60
|
-
"""Initializes RestApiClient."""
|
61
|
-
self.endpoint = endpoint
|
62
|
-
self.query_args = kwargs
|
63
|
-
|
64
|
-
@override
|
65
|
-
def get_response(
|
66
|
-
self, request: GarfApiRequest = GarfApiRequest(), **kwargs: str
|
67
|
-
) -> GarfApiResponse:
|
68
|
-
response = requests.get(f'{self.endpoint}/{request.resource_name}')
|
69
|
-
if response.status_code == self.OK:
|
70
|
-
return GarfApiResponse(response.json())
|
71
|
-
raise GarfApiError('Failed to get data from API')
|
72
|
-
|
73
|
-
|
74
|
-
class FakeApiClient(BaseClient):
|
75
|
-
"""Fake class for specifying API client."""
|
76
|
-
|
77
|
-
def __init__(self, results: Sequence) -> None:
|
78
|
-
"""Initializes FakeApiClient."""
|
79
|
-
self.results = list(results)
|
80
|
-
|
81
|
-
@override
|
82
|
-
def get_response(
|
83
|
-
self, request: GarfApiRequest = GarfApiRequest()
|
84
|
-
) -> GarfApiResponse:
|
85
|
-
del request
|
86
|
-
return GarfApiResponse(results=self.results)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|