api-to-dataframe 1.5.2__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {api_to_dataframe-1.5.2 → api_to_dataframe-2.0.0}/PKG-INFO +1 -2
- {api_to_dataframe-1.5.2 → api_to_dataframe-2.0.0}/pyproject.toml +1 -2
- api_to_dataframe-2.0.0/src/api_to_dataframe/controller/client_builder.py +95 -0
- api_to_dataframe-2.0.0/src/api_to_dataframe/models/get_data.py +27 -0
- api_to_dataframe-2.0.0/src/api_to_dataframe/models/retainer.py +39 -0
- {api_to_dataframe-1.5.2 → api_to_dataframe-2.0.0}/src/api_to_dataframe/utils/logger.py +0 -5
- api_to_dataframe-1.5.2/src/api_to_dataframe/controller/client_builder.py +0 -257
- api_to_dataframe-1.5.2/src/api_to_dataframe/models/get_data.py +0 -221
- api_to_dataframe-1.5.2/src/api_to_dataframe/models/retainer.py +0 -191
- {api_to_dataframe-1.5.2 → api_to_dataframe-2.0.0}/LICENSE +0 -0
- {api_to_dataframe-1.5.2 → api_to_dataframe-2.0.0}/README.md +0 -0
- {api_to_dataframe-1.5.2 → api_to_dataframe-2.0.0}/src/api_to_dataframe/__init__.py +0 -0
- {api_to_dataframe-1.5.2 → api_to_dataframe-2.0.0}/src/api_to_dataframe/utils/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: api-to-dataframe
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: A package to convert API responses to pandas dataframe
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: IvanildoBarauna
|
|
@@ -15,7 +15,6 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.11
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.8
|
|
17
17
|
Requires-Dist: logging (>=0.4.9.6,<0.5.0.0)
|
|
18
|
-
Requires-Dist: otel-wrapper (>=0.1.1,<0.2.0)
|
|
19
18
|
Requires-Dist: pandas (>=2.2.3,<3.0.0)
|
|
20
19
|
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
|
21
20
|
Project-URL: Documentation, https://github.com/IvanildoBarauna/api-to-dataframe/blob/main/README.md
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "api-to-dataframe"
|
|
3
|
-
version = "
|
|
3
|
+
version = "2.0.0"
|
|
4
4
|
description = "A package to convert API responses to pandas dataframe"
|
|
5
5
|
authors = ["IvanildoBarauna <ivanildo.jnr@outlook.com>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -27,7 +27,6 @@ python = "^3.9"
|
|
|
27
27
|
pandas = "^2.2.3"
|
|
28
28
|
requests = "^2.32.3"
|
|
29
29
|
logging = "^0.4.9.6"
|
|
30
|
-
otel-wrapper = "^0.1.1"
|
|
31
30
|
|
|
32
31
|
[tool.poetry.group.dev.dependencies]
|
|
33
32
|
poetry-dynamic-versioning = "^1.3.0"
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
from api_to_dataframe.models.retainer import retry_strategies, Strategies
|
|
2
|
+
from api_to_dataframe.models.get_data import GetData
|
|
3
|
+
from api_to_dataframe.utils.logger import logger
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ClientBuilder:
|
|
7
|
+
def __init__( # pylint: disable=too-many-positional-arguments,too-many-arguments
|
|
8
|
+
self,
|
|
9
|
+
endpoint: str,
|
|
10
|
+
headers: dict = None,
|
|
11
|
+
retry_strategy: Strategies = Strategies.NO_RETRY_STRATEGY,
|
|
12
|
+
retries: int = 3,
|
|
13
|
+
initial_delay: int = 1,
|
|
14
|
+
connection_timeout: int = 1,
|
|
15
|
+
):
|
|
16
|
+
"""
|
|
17
|
+
Initializes the ClientBuilder object.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
endpoint (str): The API endpoint to connect to.
|
|
21
|
+
headers (dict, optional): The headers to use for the API request. Defaults to None.
|
|
22
|
+
retry_strategy (Strategies, optional): Defaults to Strategies.NO_RETRY_STRATEGY.
|
|
23
|
+
retries (int): The number of times to retry a failed request. Defaults to 3.
|
|
24
|
+
initial_delay (int): The delay between retries in seconds. Defaults to 1.
|
|
25
|
+
connection_timeout (int): The timeout for the connection in seconds. Defaults to 1.
|
|
26
|
+
|
|
27
|
+
Raises:
|
|
28
|
+
ValueError: If endpoint is an empty string.
|
|
29
|
+
ValueError: If retries is not a non-negative integer.
|
|
30
|
+
ValueError: If delay is not a non-negative integer.
|
|
31
|
+
ValueError: If connection_timeout is not a non-negative integer.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
if headers is None:
|
|
35
|
+
headers = {}
|
|
36
|
+
if endpoint == "":
|
|
37
|
+
error_msg = "endpoint cannot be an empty string"
|
|
38
|
+
logger.error(error_msg)
|
|
39
|
+
raise ValueError
|
|
40
|
+
if not isinstance(retries, int) or retries < 0:
|
|
41
|
+
error_msg = "retries must be a non-negative integer"
|
|
42
|
+
logger.error(error_msg)
|
|
43
|
+
raise ValueError
|
|
44
|
+
if not isinstance(initial_delay, int) or initial_delay < 0:
|
|
45
|
+
error_msg = "initial_delay must be a non-negative integer"
|
|
46
|
+
logger.error(error_msg)
|
|
47
|
+
raise ValueError
|
|
48
|
+
if not isinstance(connection_timeout, int) or connection_timeout < 0:
|
|
49
|
+
error_msg = "connection_timeout must be a non-negative integer"
|
|
50
|
+
logger.error(error_msg)
|
|
51
|
+
raise ValueError
|
|
52
|
+
|
|
53
|
+
self.endpoint = endpoint
|
|
54
|
+
self.retry_strategy = retry_strategy
|
|
55
|
+
self.connection_timeout = connection_timeout
|
|
56
|
+
self.headers = headers
|
|
57
|
+
self.retries = retries
|
|
58
|
+
self.delay = initial_delay
|
|
59
|
+
|
|
60
|
+
@retry_strategies
|
|
61
|
+
def get_api_data(self):
|
|
62
|
+
"""
|
|
63
|
+
Retrieves data from the API using the defined endpoint and retry strategy.
|
|
64
|
+
|
|
65
|
+
This function sends a request to the API using the endpoint, headers, and
|
|
66
|
+
connection timeout specified in the instance attributes. It uses the
|
|
67
|
+
defined retry strategy to handle potential failures and retries.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
dict: The JSON response from the API as a dictionary.
|
|
71
|
+
"""
|
|
72
|
+
response = GetData.get_response(
|
|
73
|
+
endpoint=self.endpoint,
|
|
74
|
+
headers=self.headers,
|
|
75
|
+
connection_timeout=self.connection_timeout,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
return response.json()
|
|
79
|
+
|
|
80
|
+
@staticmethod
|
|
81
|
+
def api_to_dataframe(response: dict):
|
|
82
|
+
"""
|
|
83
|
+
Converts an API response to a DataFrame.
|
|
84
|
+
|
|
85
|
+
This function takes a dictionary response from an API,
|
|
86
|
+
uses the `to_dataframe` function from the `GetData` class
|
|
87
|
+
to convert it into a DataFrame, and logs the operation as successful.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
response (dict): The dictionary containing the API response.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
DataFrame: A pandas DataFrame containing the data from the API response.
|
|
94
|
+
"""
|
|
95
|
+
return GetData.to_dataframe(response)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
import pandas as pd
|
|
3
|
+
from api_to_dataframe.utils.logger import logger
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class GetData:
|
|
7
|
+
@staticmethod
|
|
8
|
+
def get_response(endpoint: str, headers: dict, connection_timeout: int):
|
|
9
|
+
# Make the request
|
|
10
|
+
response = requests.get(endpoint, timeout=connection_timeout, headers=headers)
|
|
11
|
+
|
|
12
|
+
# Attempt to raise for status to catch errors
|
|
13
|
+
response.raise_for_status()
|
|
14
|
+
|
|
15
|
+
return response
|
|
16
|
+
|
|
17
|
+
@staticmethod
|
|
18
|
+
def to_dataframe(response):
|
|
19
|
+
df = pd.DataFrame(response)
|
|
20
|
+
|
|
21
|
+
# Check if DataFrame is empty
|
|
22
|
+
if df.empty:
|
|
23
|
+
error_msg = "::: DataFrame is empty :::"
|
|
24
|
+
logger.error(error_msg)
|
|
25
|
+
raise ValueError(error_msg)
|
|
26
|
+
|
|
27
|
+
return df
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from enum import Enum
|
|
3
|
+
|
|
4
|
+
from requests.exceptions import RequestException
|
|
5
|
+
from api_to_dataframe.utils.logger import logger
|
|
6
|
+
from api_to_dataframe.utils import Constants
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Strategies(Enum):
|
|
10
|
+
NO_RETRY_STRATEGY = 0
|
|
11
|
+
LINEAR_RETRY_STRATEGY = 1
|
|
12
|
+
EXPONENTIAL_RETRY_STRATEGY = 2
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def retry_strategies(func):
|
|
16
|
+
def wrapper(*args, **kwargs): # pylint: disable=inconsistent-return-statements
|
|
17
|
+
retry_number = 0
|
|
18
|
+
while retry_number < args[0].retries:
|
|
19
|
+
try:
|
|
20
|
+
if retry_number > 0:
|
|
21
|
+
logger.info(
|
|
22
|
+
f"Trying for the {retry_number} of {Constants.MAX_OF_RETRIES} retries. Using {args[0].retry_strategy}"
|
|
23
|
+
)
|
|
24
|
+
return func(*args, **kwargs)
|
|
25
|
+
except RequestException as e:
|
|
26
|
+
retry_number += 1
|
|
27
|
+
|
|
28
|
+
if args[0].retry_strategy == Strategies.NO_RETRY_STRATEGY:
|
|
29
|
+
raise e
|
|
30
|
+
if args[0].retry_strategy == Strategies.LINEAR_RETRY_STRATEGY:
|
|
31
|
+
time.sleep(args[0].delay)
|
|
32
|
+
elif args[0].retry_strategy == Strategies.EXPONENTIAL_RETRY_STRATEGY:
|
|
33
|
+
time.sleep(args[0].delay * retry_number)
|
|
34
|
+
|
|
35
|
+
if retry_number in (args[0].retries, Constants.MAX_OF_RETRIES):
|
|
36
|
+
logger.error(f"Failed after {retry_number} retries")
|
|
37
|
+
raise e
|
|
38
|
+
|
|
39
|
+
return wrapper
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
from otel_wrapper.deps_injector import wrapper_builder
|
|
3
2
|
|
|
4
|
-
# Configure logging once at the start of your program
|
|
5
3
|
logging.basicConfig(
|
|
6
4
|
encoding="utf-8",
|
|
7
5
|
format="%(asctime)s :: api-to-dataframe[%(levelname)s] :: %(message)s",
|
|
@@ -11,6 +9,3 @@ logging.basicConfig(
|
|
|
11
9
|
|
|
12
10
|
# Initialize traditional logger
|
|
13
11
|
logger = logging.getLogger("api-to-dataframe")
|
|
14
|
-
|
|
15
|
-
# Initialize OpenTelemetry wrapper
|
|
16
|
-
telemetry = wrapper_builder("api-to-dataframe")
|
|
@@ -1,257 +0,0 @@
|
|
|
1
|
-
from api_to_dataframe.models.retainer import retry_strategies, Strategies
|
|
2
|
-
from api_to_dataframe.models.get_data import GetData
|
|
3
|
-
from api_to_dataframe.utils.logger import logger, telemetry
|
|
4
|
-
import time
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class ClientBuilder:
|
|
8
|
-
def __init__( # pylint: disable=too-many-positional-arguments,too-many-arguments
|
|
9
|
-
self,
|
|
10
|
-
endpoint: str,
|
|
11
|
-
headers: dict = None,
|
|
12
|
-
retry_strategy: Strategies = Strategies.NO_RETRY_STRATEGY,
|
|
13
|
-
retries: int = 3,
|
|
14
|
-
initial_delay: int = 1,
|
|
15
|
-
connection_timeout: int = 1,
|
|
16
|
-
):
|
|
17
|
-
"""
|
|
18
|
-
Initializes the ClientBuilder object.
|
|
19
|
-
|
|
20
|
-
Args:
|
|
21
|
-
endpoint (str): The API endpoint to connect to.
|
|
22
|
-
headers (dict, optional): The headers to use for the API request. Defaults to None.
|
|
23
|
-
retry_strategy (Strategies, optional): Defaults to Strategies.NO_RETRY_STRATEGY.
|
|
24
|
-
retries (int): The number of times to retry a failed request. Defaults to 3.
|
|
25
|
-
initial_delay (int): The delay between retries in seconds. Defaults to 1.
|
|
26
|
-
connection_timeout (int): The timeout for the connection in seconds. Defaults to 1.
|
|
27
|
-
|
|
28
|
-
Raises:
|
|
29
|
-
ValueError: If endpoint is an empty string.
|
|
30
|
-
ValueError: If retries is not a non-negative integer.
|
|
31
|
-
ValueError: If delay is not a non-negative integer.
|
|
32
|
-
ValueError: If connection_timeout is not a non-negative integer.
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
if headers is None:
|
|
36
|
-
headers = {}
|
|
37
|
-
if endpoint == "":
|
|
38
|
-
error_msg = "endpoint cannot be an empty string"
|
|
39
|
-
logger.error(error_msg)
|
|
40
|
-
telemetry.logs().new_log(
|
|
41
|
-
msg=error_msg,
|
|
42
|
-
tags={"component": "ClientBuilder", "method": "__init__"},
|
|
43
|
-
level=40 # ERROR level
|
|
44
|
-
)
|
|
45
|
-
raise ValueError
|
|
46
|
-
if not isinstance(retries, int) or retries < 0:
|
|
47
|
-
error_msg = "retries must be a non-negative integer"
|
|
48
|
-
logger.error(error_msg)
|
|
49
|
-
telemetry.logs().new_log(
|
|
50
|
-
msg=error_msg,
|
|
51
|
-
tags={"component": "ClientBuilder", "method": "__init__"},
|
|
52
|
-
level=40 # ERROR level
|
|
53
|
-
)
|
|
54
|
-
raise ValueError
|
|
55
|
-
if not isinstance(initial_delay, int) or initial_delay < 0:
|
|
56
|
-
error_msg = "initial_delay must be a non-negative integer"
|
|
57
|
-
logger.error(error_msg)
|
|
58
|
-
telemetry.logs().new_log(
|
|
59
|
-
msg=error_msg,
|
|
60
|
-
tags={"component": "ClientBuilder", "method": "__init__"},
|
|
61
|
-
level=40 # ERROR level
|
|
62
|
-
)
|
|
63
|
-
raise ValueError
|
|
64
|
-
if not isinstance(connection_timeout, int) or connection_timeout < 0:
|
|
65
|
-
error_msg = "connection_timeout must be a non-negative integer"
|
|
66
|
-
logger.error(error_msg)
|
|
67
|
-
telemetry.logs().new_log(
|
|
68
|
-
msg=error_msg,
|
|
69
|
-
tags={"component": "ClientBuilder", "method": "__init__"},
|
|
70
|
-
level=40 # ERROR level
|
|
71
|
-
)
|
|
72
|
-
raise ValueError
|
|
73
|
-
|
|
74
|
-
self.endpoint = endpoint
|
|
75
|
-
self.retry_strategy = retry_strategy
|
|
76
|
-
self.connection_timeout = connection_timeout
|
|
77
|
-
self.headers = headers
|
|
78
|
-
self.retries = retries
|
|
79
|
-
self.delay = initial_delay
|
|
80
|
-
|
|
81
|
-
# Record client initialization metric
|
|
82
|
-
telemetry.metrics().metric_increment(
|
|
83
|
-
name="client.initialization",
|
|
84
|
-
tags={
|
|
85
|
-
"endpoint": endpoint,
|
|
86
|
-
"retry_strategy": retry_strategy.name,
|
|
87
|
-
"connection_timeout": str(connection_timeout)
|
|
88
|
-
}
|
|
89
|
-
)
|
|
90
|
-
|
|
91
|
-
# Log initialization
|
|
92
|
-
telemetry.logs().new_log(
|
|
93
|
-
msg=f"ClientBuilder initialized with endpoint {endpoint}",
|
|
94
|
-
tags={
|
|
95
|
-
"endpoint": endpoint,
|
|
96
|
-
"retry_strategy": retry_strategy.name,
|
|
97
|
-
"connection_timeout": str(connection_timeout),
|
|
98
|
-
"component": "ClientBuilder"
|
|
99
|
-
},
|
|
100
|
-
level=20 # INFO level
|
|
101
|
-
)
|
|
102
|
-
|
|
103
|
-
@retry_strategies
|
|
104
|
-
def get_api_data(self):
|
|
105
|
-
"""
|
|
106
|
-
Retrieves data from the API using the defined endpoint and retry strategy.
|
|
107
|
-
|
|
108
|
-
This function sends a request to the API using the endpoint, headers, and
|
|
109
|
-
connection timeout specified in the instance attributes. It uses the
|
|
110
|
-
defined retry strategy to handle potential failures and retries.
|
|
111
|
-
|
|
112
|
-
Returns:
|
|
113
|
-
dict: The JSON response from the API as a dictionary.
|
|
114
|
-
"""
|
|
115
|
-
# Use the telemetry spans with new API
|
|
116
|
-
span = telemetry.traces().new_span("get_api_data")
|
|
117
|
-
try:
|
|
118
|
-
# Add span attributes
|
|
119
|
-
span.set_attribute("endpoint", self.endpoint)
|
|
120
|
-
span.set_attribute("retry_strategy", self.retry_strategy.name)
|
|
121
|
-
span.set_attribute("connection_timeout", self.connection_timeout)
|
|
122
|
-
|
|
123
|
-
# Log the API request
|
|
124
|
-
telemetry.logs().new_log(
|
|
125
|
-
msg=f"Making API request to {self.endpoint}",
|
|
126
|
-
tags={
|
|
127
|
-
"endpoint": self.endpoint,
|
|
128
|
-
"component": "ClientBuilder",
|
|
129
|
-
"method": "get_api_data"
|
|
130
|
-
},
|
|
131
|
-
level=20 # INFO level
|
|
132
|
-
)
|
|
133
|
-
|
|
134
|
-
# Record the start time for response time measurement
|
|
135
|
-
start_time = time.time()
|
|
136
|
-
|
|
137
|
-
# Make the API request
|
|
138
|
-
response = GetData.get_response(
|
|
139
|
-
endpoint=self.endpoint,
|
|
140
|
-
headers=self.headers,
|
|
141
|
-
connection_timeout=self.connection_timeout,
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
# Calculate response time
|
|
145
|
-
response_time = time.time() - start_time
|
|
146
|
-
|
|
147
|
-
# Record response time as histogram
|
|
148
|
-
telemetry.metrics().record_histogram(
|
|
149
|
-
name="api.response_time",
|
|
150
|
-
tags={"endpoint": self.endpoint},
|
|
151
|
-
value=response_time
|
|
152
|
-
)
|
|
153
|
-
|
|
154
|
-
# Record successful request metric
|
|
155
|
-
telemetry.metrics().metric_increment(
|
|
156
|
-
name="api.request.success",
|
|
157
|
-
tags={"endpoint": self.endpoint}
|
|
158
|
-
)
|
|
159
|
-
|
|
160
|
-
# Log success
|
|
161
|
-
telemetry.logs().new_log(
|
|
162
|
-
msg=f"API request to {self.endpoint} successful",
|
|
163
|
-
tags={
|
|
164
|
-
"endpoint": self.endpoint,
|
|
165
|
-
"response_status": response.status_code,
|
|
166
|
-
"response_time": response_time,
|
|
167
|
-
"component": "ClientBuilder",
|
|
168
|
-
"method": "get_api_data"
|
|
169
|
-
},
|
|
170
|
-
level=20 # INFO level
|
|
171
|
-
)
|
|
172
|
-
|
|
173
|
-
return response.json()
|
|
174
|
-
finally:
|
|
175
|
-
span.end()
|
|
176
|
-
|
|
177
|
-
@staticmethod
|
|
178
|
-
def api_to_dataframe(response: dict):
|
|
179
|
-
"""
|
|
180
|
-
Converts an API response to a DataFrame.
|
|
181
|
-
|
|
182
|
-
This function takes a dictionary response from an API,
|
|
183
|
-
uses the `to_dataframe` function from the `GetData` class
|
|
184
|
-
to convert it into a DataFrame, and logs the operation as successful.
|
|
185
|
-
|
|
186
|
-
Args:
|
|
187
|
-
response (dict): The dictionary containing the API response.
|
|
188
|
-
|
|
189
|
-
Returns:
|
|
190
|
-
DataFrame: A pandas DataFrame containing the data from the API response.
|
|
191
|
-
"""
|
|
192
|
-
# Use telemetry with new API
|
|
193
|
-
span = telemetry.traces().new_span("api_to_dataframe")
|
|
194
|
-
try:
|
|
195
|
-
response_size = len(response) if isinstance(response, list) else 1
|
|
196
|
-
span.set_attribute("response_size", response_size)
|
|
197
|
-
|
|
198
|
-
# Log conversion start
|
|
199
|
-
telemetry.logs().new_log(
|
|
200
|
-
msg="Converting API response to DataFrame",
|
|
201
|
-
tags={
|
|
202
|
-
"response_size": response_size,
|
|
203
|
-
"response_type": type(response).__name__,
|
|
204
|
-
"component": "ClientBuilder",
|
|
205
|
-
"method": "api_to_dataframe"
|
|
206
|
-
},
|
|
207
|
-
level=20 # INFO level
|
|
208
|
-
)
|
|
209
|
-
|
|
210
|
-
try:
|
|
211
|
-
# Convert to dataframe
|
|
212
|
-
df = GetData.to_dataframe(response)
|
|
213
|
-
|
|
214
|
-
# Record metrics
|
|
215
|
-
telemetry.metrics().metric_increment(
|
|
216
|
-
name="dataframe.conversion.success",
|
|
217
|
-
tags={"size": len(df)}
|
|
218
|
-
)
|
|
219
|
-
|
|
220
|
-
# Log success
|
|
221
|
-
telemetry.logs().new_log(
|
|
222
|
-
msg="Successfully converted API response to DataFrame",
|
|
223
|
-
tags={
|
|
224
|
-
"dataframe_rows": len(df),
|
|
225
|
-
"dataframe_columns": len(df.columns),
|
|
226
|
-
"component": "ClientBuilder",
|
|
227
|
-
"method": "api_to_dataframe"
|
|
228
|
-
},
|
|
229
|
-
level=20 # INFO level
|
|
230
|
-
)
|
|
231
|
-
|
|
232
|
-
return df
|
|
233
|
-
|
|
234
|
-
except Exception as e:
|
|
235
|
-
# Record failure metric
|
|
236
|
-
telemetry.metrics().metric_increment(
|
|
237
|
-
name="dataframe.conversion.failure",
|
|
238
|
-
tags={"error_type": type(e).__name__}
|
|
239
|
-
)
|
|
240
|
-
|
|
241
|
-
# Log error
|
|
242
|
-
error_msg = f"Failed to convert API response to DataFrame: {str(e)}"
|
|
243
|
-
telemetry.logs().new_log(
|
|
244
|
-
msg=error_msg,
|
|
245
|
-
tags={
|
|
246
|
-
"error": str(e),
|
|
247
|
-
"error_type": type(e).__name__,
|
|
248
|
-
"component": "ClientBuilder",
|
|
249
|
-
"method": "api_to_dataframe"
|
|
250
|
-
},
|
|
251
|
-
level=40 # ERROR level
|
|
252
|
-
)
|
|
253
|
-
|
|
254
|
-
# Re-raise the exception
|
|
255
|
-
raise
|
|
256
|
-
finally:
|
|
257
|
-
span.end()
|
|
@@ -1,221 +0,0 @@
|
|
|
1
|
-
import requests
|
|
2
|
-
import pandas as pd
|
|
3
|
-
from api_to_dataframe.utils.logger import logger, telemetry
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class GetData:
|
|
7
|
-
@staticmethod
|
|
8
|
-
def get_response(endpoint: str, headers: dict, connection_timeout: int):
|
|
9
|
-
# Start a span for the API request
|
|
10
|
-
span = telemetry.traces().new_span("http_request")
|
|
11
|
-
try:
|
|
12
|
-
span.set_attribute("http.url", endpoint)
|
|
13
|
-
span.set_attribute("http.method", "GET")
|
|
14
|
-
span.set_attribute("http.timeout", connection_timeout)
|
|
15
|
-
|
|
16
|
-
# Log the request
|
|
17
|
-
telemetry.logs().new_log(
|
|
18
|
-
msg=f"Sending HTTP GET request to {endpoint}",
|
|
19
|
-
tags={
|
|
20
|
-
"endpoint": endpoint,
|
|
21
|
-
"timeout": connection_timeout,
|
|
22
|
-
"component": "GetData",
|
|
23
|
-
"method": "get_response"
|
|
24
|
-
},
|
|
25
|
-
level=20 # INFO level
|
|
26
|
-
)
|
|
27
|
-
|
|
28
|
-
try:
|
|
29
|
-
# Make the request
|
|
30
|
-
response = requests.get(endpoint, timeout=connection_timeout, headers=headers)
|
|
31
|
-
|
|
32
|
-
# Set response attributes on span
|
|
33
|
-
span.set_attribute("http.status_code", response.status_code)
|
|
34
|
-
span.set_attribute("http.response_content_length", len(response.content))
|
|
35
|
-
|
|
36
|
-
# Attempt to raise for status to catch errors
|
|
37
|
-
response.raise_for_status()
|
|
38
|
-
|
|
39
|
-
# Log successful response
|
|
40
|
-
telemetry.logs().new_log(
|
|
41
|
-
msg=f"Received HTTP {response.status_code} response from {endpoint}",
|
|
42
|
-
tags={
|
|
43
|
-
"endpoint": endpoint,
|
|
44
|
-
"status_code": response.status_code,
|
|
45
|
-
"response_size": len(response.content),
|
|
46
|
-
"component": "GetData",
|
|
47
|
-
"method": "get_response"
|
|
48
|
-
},
|
|
49
|
-
level=20 # INFO level
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
# Record successful request metric
|
|
53
|
-
telemetry.metrics().metric_increment(
|
|
54
|
-
name="http.request.success",
|
|
55
|
-
tags={
|
|
56
|
-
"endpoint": endpoint,
|
|
57
|
-
"status_code": response.status_code
|
|
58
|
-
}
|
|
59
|
-
)
|
|
60
|
-
|
|
61
|
-
return response
|
|
62
|
-
|
|
63
|
-
except requests.exceptions.RequestException as e:
|
|
64
|
-
# Record the exception on the span
|
|
65
|
-
span.record_exception(e)
|
|
66
|
-
span.set_attribute("error", True)
|
|
67
|
-
span.set_attribute("error.type", type(e).__name__)
|
|
68
|
-
span.set_attribute("error.message", str(e))
|
|
69
|
-
|
|
70
|
-
# Log the error
|
|
71
|
-
telemetry.logs().new_log(
|
|
72
|
-
msg=f"HTTP request failed: {str(e)}",
|
|
73
|
-
tags={
|
|
74
|
-
"endpoint": endpoint,
|
|
75
|
-
"error": str(e),
|
|
76
|
-
"error_type": type(e).__name__,
|
|
77
|
-
"component": "GetData",
|
|
78
|
-
"method": "get_response"
|
|
79
|
-
},
|
|
80
|
-
level=40 # ERROR level
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
# Record failure metric
|
|
84
|
-
telemetry.metrics().metric_increment(
|
|
85
|
-
name="http.request.failure",
|
|
86
|
-
tags={
|
|
87
|
-
"endpoint": endpoint,
|
|
88
|
-
"error_type": type(e).__name__
|
|
89
|
-
}
|
|
90
|
-
)
|
|
91
|
-
|
|
92
|
-
# Re-raise the exception
|
|
93
|
-
raise
|
|
94
|
-
finally:
|
|
95
|
-
span.end()
|
|
96
|
-
|
|
97
|
-
@staticmethod
|
|
98
|
-
def to_dataframe(response):
|
|
99
|
-
# Start a span for dataframe conversion
|
|
100
|
-
span = telemetry.traces().new_span("convert_to_dataframe")
|
|
101
|
-
try:
|
|
102
|
-
# Set attributes about the data
|
|
103
|
-
data_size = len(response) if isinstance(response, list) else 1
|
|
104
|
-
span.set_attribute("data.size", data_size)
|
|
105
|
-
span.set_attribute("data.type", type(response).__name__)
|
|
106
|
-
|
|
107
|
-
# Log conversion attempt
|
|
108
|
-
telemetry.logs().new_log(
|
|
109
|
-
msg="Converting data to DataFrame",
|
|
110
|
-
tags={
|
|
111
|
-
"data_size": data_size,
|
|
112
|
-
"data_type": type(response).__name__,
|
|
113
|
-
"component": "GetData",
|
|
114
|
-
"method": "to_dataframe"
|
|
115
|
-
},
|
|
116
|
-
level=20 # INFO level
|
|
117
|
-
)
|
|
118
|
-
|
|
119
|
-
try:
|
|
120
|
-
# Convert to DataFrame
|
|
121
|
-
df = pd.DataFrame(response)
|
|
122
|
-
|
|
123
|
-
# Check if DataFrame is empty
|
|
124
|
-
if df.empty:
|
|
125
|
-
error_msg = "::: DataFrame is empty :::"
|
|
126
|
-
logger.error(error_msg)
|
|
127
|
-
|
|
128
|
-
# Log the error with OpenTelemetry
|
|
129
|
-
telemetry.logs().new_log(
|
|
130
|
-
msg=error_msg,
|
|
131
|
-
tags={
|
|
132
|
-
"data_size": data_size,
|
|
133
|
-
"data_type": type(response).__name__,
|
|
134
|
-
"component": "GetData",
|
|
135
|
-
"method": "to_dataframe"
|
|
136
|
-
},
|
|
137
|
-
level=40 # ERROR level
|
|
138
|
-
)
|
|
139
|
-
|
|
140
|
-
# Record empty DataFrame metric
|
|
141
|
-
telemetry.metrics().metric_increment(
|
|
142
|
-
name="dataframe.empty",
|
|
143
|
-
tags={"data_type": type(response).__name__}
|
|
144
|
-
)
|
|
145
|
-
|
|
146
|
-
# Set span as error
|
|
147
|
-
span.set_attribute("error", True)
|
|
148
|
-
span.set_attribute("error.type", "ValueError")
|
|
149
|
-
span.set_attribute("error.message", error_msg)
|
|
150
|
-
|
|
151
|
-
raise ValueError(error_msg)
|
|
152
|
-
|
|
153
|
-
# Log success
|
|
154
|
-
telemetry.logs().new_log(
|
|
155
|
-
msg="Successfully converted data to DataFrame",
|
|
156
|
-
tags={
|
|
157
|
-
"rows": len(df),
|
|
158
|
-
"columns": len(df.columns),
|
|
159
|
-
"component": "GetData",
|
|
160
|
-
"method": "to_dataframe"
|
|
161
|
-
},
|
|
162
|
-
level=20 # INFO level
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
# Record dataframe metrics
|
|
166
|
-
telemetry.metrics().record_gauge(
|
|
167
|
-
name="dataframe.rows",
|
|
168
|
-
tags={"data_type": type(response).__name__},
|
|
169
|
-
value=float(len(df))
|
|
170
|
-
)
|
|
171
|
-
|
|
172
|
-
telemetry.metrics().record_gauge(
|
|
173
|
-
name="dataframe.columns",
|
|
174
|
-
tags={"data_type": type(response).__name__},
|
|
175
|
-
value=float(len(df.columns))
|
|
176
|
-
)
|
|
177
|
-
|
|
178
|
-
# Set additional span attributes
|
|
179
|
-
span.set_attribute("dataframe.rows", len(df))
|
|
180
|
-
span.set_attribute("dataframe.columns", len(df.columns))
|
|
181
|
-
|
|
182
|
-
return df
|
|
183
|
-
|
|
184
|
-
except ValueError:
|
|
185
|
-
# Re-raise ValueErrors (like empty DataFrame)
|
|
186
|
-
raise
|
|
187
|
-
|
|
188
|
-
except Exception as err:
|
|
189
|
-
# Log the error
|
|
190
|
-
error_msg = f"Invalid response for transform in dataframe: {err}"
|
|
191
|
-
logger.error(error_msg)
|
|
192
|
-
|
|
193
|
-
# Log with OpenTelemetry
|
|
194
|
-
telemetry.logs().new_log(
|
|
195
|
-
msg=error_msg,
|
|
196
|
-
tags={
|
|
197
|
-
"error": str(err),
|
|
198
|
-
"error_type": type(err).__name__,
|
|
199
|
-
"data_type": type(response).__name__,
|
|
200
|
-
"component": "GetData",
|
|
201
|
-
"method": "to_dataframe"
|
|
202
|
-
},
|
|
203
|
-
level=40 # ERROR level
|
|
204
|
-
)
|
|
205
|
-
|
|
206
|
-
# Record conversion failure metric
|
|
207
|
-
telemetry.metrics().metric_increment(
|
|
208
|
-
name="dataframe.conversion.error",
|
|
209
|
-
tags={"error_type": type(err).__name__}
|
|
210
|
-
)
|
|
211
|
-
|
|
212
|
-
# Record the exception on the span
|
|
213
|
-
span.record_exception(err)
|
|
214
|
-
span.set_attribute("error", True)
|
|
215
|
-
span.set_attribute("error.type", type(err).__name__)
|
|
216
|
-
span.set_attribute("error.message", str(err))
|
|
217
|
-
|
|
218
|
-
# Raise TypeError with original error as cause
|
|
219
|
-
raise TypeError(error_msg) from err
|
|
220
|
-
finally:
|
|
221
|
-
span.end()
|
|
@@ -1,191 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
from enum import Enum
|
|
3
|
-
|
|
4
|
-
from requests.exceptions import RequestException
|
|
5
|
-
from api_to_dataframe.utils.logger import logger, telemetry
|
|
6
|
-
from api_to_dataframe.utils import Constants
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class Strategies(Enum):
|
|
10
|
-
NO_RETRY_STRATEGY = 0
|
|
11
|
-
LINEAR_RETRY_STRATEGY = 1
|
|
12
|
-
EXPONENTIAL_RETRY_STRATEGY = 2
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def retry_strategies(func):
|
|
16
|
-
def wrapper(*args, **kwargs): # pylint: disable=inconsistent-return-statements
|
|
17
|
-
retry_number = 0
|
|
18
|
-
|
|
19
|
-
# Get the endpoint for better observability context
|
|
20
|
-
endpoint = args[0].endpoint if hasattr(args[0], 'endpoint') else 'unknown'
|
|
21
|
-
|
|
22
|
-
# Start a span for this entire retry operation
|
|
23
|
-
span = telemetry.traces().new_span("retry_operation")
|
|
24
|
-
try:
|
|
25
|
-
span.set_attribute("endpoint", endpoint)
|
|
26
|
-
span.set_attribute("retry_strategy", args[0].retry_strategy.name)
|
|
27
|
-
span.set_attribute("max_retries", args[0].retries)
|
|
28
|
-
|
|
29
|
-
while retry_number < args[0].retries:
|
|
30
|
-
try:
|
|
31
|
-
# Log retry attempt if not the first attempt
|
|
32
|
-
if retry_number > 0:
|
|
33
|
-
# Log using traditional logger
|
|
34
|
-
logger.info(f"Trying for the {retry_number} of {Constants.MAX_OF_RETRIES} retries. Using {args[0].retry_strategy}")
|
|
35
|
-
|
|
36
|
-
# Log using OpenTelemetry
|
|
37
|
-
telemetry.logs().new_log(
|
|
38
|
-
msg=f"Retry attempt {retry_number} of {args[0].retries}",
|
|
39
|
-
tags={
|
|
40
|
-
"endpoint": endpoint,
|
|
41
|
-
"retry_number": retry_number,
|
|
42
|
-
"max_retries": args[0].retries,
|
|
43
|
-
"retry_strategy": args[0].retry_strategy.name,
|
|
44
|
-
"component": "RetryStrategy"
|
|
45
|
-
},
|
|
46
|
-
level=20 # INFO level
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
# Record retry metric
|
|
50
|
-
telemetry.metrics().metric_increment(
|
|
51
|
-
name="api.request.retry",
|
|
52
|
-
tags={
|
|
53
|
-
"endpoint": endpoint,
|
|
54
|
-
"retry_strategy": args[0].retry_strategy.name
|
|
55
|
-
}
|
|
56
|
-
)
|
|
57
|
-
|
|
58
|
-
# Update span with current retry count
|
|
59
|
-
span.set_attribute("current_retry", retry_number)
|
|
60
|
-
|
|
61
|
-
# Execute the wrapped function
|
|
62
|
-
result = func(*args, **kwargs)
|
|
63
|
-
|
|
64
|
-
# If we got here, it succeeded - record success metric after retries
|
|
65
|
-
if retry_number > 0:
|
|
66
|
-
telemetry.metrics().metric_increment(
|
|
67
|
-
name="api.request.retry.success",
|
|
68
|
-
tags={
|
|
69
|
-
"endpoint": endpoint,
|
|
70
|
-
"retry_count": retry_number,
|
|
71
|
-
"retry_strategy": args[0].retry_strategy.name
|
|
72
|
-
}
|
|
73
|
-
)
|
|
74
|
-
|
|
75
|
-
telemetry.logs().new_log(
|
|
76
|
-
msg=f"Request succeeded after {retry_number} retries",
|
|
77
|
-
tags={
|
|
78
|
-
"endpoint": endpoint,
|
|
79
|
-
"retry_count": retry_number,
|
|
80
|
-
"retry_strategy": args[0].retry_strategy.name,
|
|
81
|
-
"component": "RetryStrategy"
|
|
82
|
-
},
|
|
83
|
-
level=20 # INFO level
|
|
84
|
-
)
|
|
85
|
-
|
|
86
|
-
return result
|
|
87
|
-
|
|
88
|
-
except RequestException as e:
|
|
89
|
-
retry_number += 1
|
|
90
|
-
|
|
91
|
-
# Record exception in the span
|
|
92
|
-
span.record_exception(e)
|
|
93
|
-
span.set_attribute("failed_attempt", retry_number)
|
|
94
|
-
|
|
95
|
-
# Handle different retry strategies
|
|
96
|
-
if args[0].retry_strategy == Strategies.NO_RETRY_STRATEGY:
|
|
97
|
-
# Log failure with OpenTelemetry
|
|
98
|
-
telemetry.logs().new_log(
|
|
99
|
-
msg=f"Request failed with {type(e).__name__}, no retry strategy configured",
|
|
100
|
-
tags={
|
|
101
|
-
"endpoint": endpoint,
|
|
102
|
-
"error": str(e),
|
|
103
|
-
"error_type": type(e).__name__,
|
|
104
|
-
"component": "RetryStrategy"
|
|
105
|
-
},
|
|
106
|
-
level=40 # ERROR level
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
# Record failure metric
|
|
110
|
-
telemetry.metrics().metric_increment(
|
|
111
|
-
name="api.request.failure",
|
|
112
|
-
tags={
|
|
113
|
-
"endpoint": endpoint,
|
|
114
|
-
"error_type": type(e).__name__,
|
|
115
|
-
"retry_strategy": "none"
|
|
116
|
-
}
|
|
117
|
-
)
|
|
118
|
-
|
|
119
|
-
raise e
|
|
120
|
-
|
|
121
|
-
# Apply delay based on strategy
|
|
122
|
-
if args[0].retry_strategy == Strategies.LINEAR_RETRY_STRATEGY:
|
|
123
|
-
delay = args[0].delay
|
|
124
|
-
strategy_name = "linear"
|
|
125
|
-
elif args[0].retry_strategy == Strategies.EXPONENTIAL_RETRY_STRATEGY:
|
|
126
|
-
delay = args[0].delay * retry_number
|
|
127
|
-
strategy_name = "exponential"
|
|
128
|
-
|
|
129
|
-
# Log retry delay
|
|
130
|
-
telemetry.logs().new_log(
|
|
131
|
-
msg=f"Request failed, retrying in {delay}s (strategy: {strategy_name})",
|
|
132
|
-
tags={
|
|
133
|
-
"endpoint": endpoint,
|
|
134
|
-
"retry_count": retry_number,
|
|
135
|
-
"delay": delay,
|
|
136
|
-
"strategy": strategy_name,
|
|
137
|
-
"error": str(e),
|
|
138
|
-
"error_type": type(e).__name__,
|
|
139
|
-
"component": "RetryStrategy"
|
|
140
|
-
},
|
|
141
|
-
level=30 # WARNING level
|
|
142
|
-
)
|
|
143
|
-
|
|
144
|
-
# Record the delay time in metrics
|
|
145
|
-
telemetry.metrics().record_gauge(
|
|
146
|
-
name="api.retry.delay",
|
|
147
|
-
tags={
|
|
148
|
-
"endpoint": endpoint,
|
|
149
|
-
"retry_count": retry_number,
|
|
150
|
-
"strategy": strategy_name
|
|
151
|
-
},
|
|
152
|
-
value=float(delay)
|
|
153
|
-
)
|
|
154
|
-
|
|
155
|
-
# Sleep for the calculated delay
|
|
156
|
-
time.sleep(delay)
|
|
157
|
-
|
|
158
|
-
# Check if we've reached max retries
|
|
159
|
-
if retry_number in (args[0].retries, Constants.MAX_OF_RETRIES):
|
|
160
|
-
# Log final failure
|
|
161
|
-
error_msg = f"Failed after {retry_number} retries"
|
|
162
|
-
logger.error(error_msg)
|
|
163
|
-
|
|
164
|
-
telemetry.logs().new_log(
|
|
165
|
-
msg=error_msg,
|
|
166
|
-
tags={
|
|
167
|
-
"endpoint": endpoint,
|
|
168
|
-
"retry_count": retry_number,
|
|
169
|
-
"max_retries": args[0].retries,
|
|
170
|
-
"error": str(e),
|
|
171
|
-
"error_type": type(e).__name__,
|
|
172
|
-
"component": "RetryStrategy"
|
|
173
|
-
},
|
|
174
|
-
level=40 # ERROR level
|
|
175
|
-
)
|
|
176
|
-
|
|
177
|
-
# Record final failure metric
|
|
178
|
-
telemetry.metrics().metric_increment(
|
|
179
|
-
name="api.request.retry.exhausted",
|
|
180
|
-
tags={
|
|
181
|
-
"endpoint": endpoint,
|
|
182
|
-
"retry_strategy": args[0].retry_strategy.name,
|
|
183
|
-
"error_type": type(e).__name__
|
|
184
|
-
}
|
|
185
|
-
)
|
|
186
|
-
|
|
187
|
-
raise e
|
|
188
|
-
finally:
|
|
189
|
-
span.end()
|
|
190
|
-
|
|
191
|
-
return wrapper
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|