api-to-dataframe 1.3.13__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/PKG-INFO +2 -2
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/pyproject.toml +2 -2
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/src/api_to_dataframe/controller/client_builder.py +16 -20
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/src/api_to_dataframe/models/get_data.py +0 -4
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/src/api_to_dataframe/models/retainer.py +3 -10
- api_to_dataframe-1.4.0/src/api_to_dataframe/utils/logger.py +11 -0
- api_to_dataframe-1.3.13/src/api_to_dataframe/utils/logger.py +0 -22
- api_to_dataframe-1.3.13/src/api_to_dataframe/utils/metrics.py +0 -19
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/LICENSE +0 -0
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/README.md +0 -0
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/src/api_to_dataframe/__init__.py +0 -0
- {api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/src/api_to_dataframe/utils/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: api-to-dataframe
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: A package to convert API responses to pandas dataframe
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: IvanildoBarauna
|
|
@@ -16,8 +16,8 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.12
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.13
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.8
|
|
19
|
-
Requires-Dist: datadog (>=0.50.2,<0.51.0)
|
|
20
19
|
Requires-Dist: logging (>=0.4.9.6,<0.5.0.0)
|
|
20
|
+
Requires-Dist: otel-wrapper (>=0.0.1,<0.0.2)
|
|
21
21
|
Requires-Dist: pandas (>=2.2.3,<3.0.0)
|
|
22
22
|
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
|
23
23
|
Project-URL: Documentation, https://github.com/IvanildoBarauna/api-to-dataframe/blob/main/README.md
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "api-to-dataframe"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.4.0"
|
|
4
4
|
description = "A package to convert API responses to pandas dataframe"
|
|
5
5
|
authors = ["IvanildoBarauna <ivanildo.jnr@outlook.com>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -27,7 +27,7 @@ python = "^3.9"
|
|
|
27
27
|
pandas = "^2.2.3"
|
|
28
28
|
requests = "^2.32.3"
|
|
29
29
|
logging = "^0.4.9.6"
|
|
30
|
-
|
|
30
|
+
otel-wrapper = "^0.0.1"
|
|
31
31
|
|
|
32
32
|
[tool.poetry.group.dev.dependencies]
|
|
33
33
|
poetry-dynamic-versioning = "^1.3.0"
|
{api_to_dataframe-1.3.13 → api_to_dataframe-1.4.0}/src/api_to_dataframe/controller/client_builder.py
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from api_to_dataframe.models.retainer import retry_strategies, Strategies
|
|
2
2
|
from api_to_dataframe.models.get_data import GetData
|
|
3
|
-
from api_to_dataframe.utils.logger import
|
|
4
|
-
from
|
|
3
|
+
from api_to_dataframe.utils.logger import logger
|
|
4
|
+
from otel_wrapper import OpenObservability
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
class ClientBuilder:
|
|
@@ -35,16 +35,16 @@ class ClientBuilder:
|
|
|
35
35
|
if headers is None:
|
|
36
36
|
headers = {}
|
|
37
37
|
if endpoint == "":
|
|
38
|
-
|
|
38
|
+
logger.error("endpoint cannot be an empty string")
|
|
39
39
|
raise ValueError
|
|
40
40
|
if not isinstance(retries, int) or retries < 0:
|
|
41
|
-
|
|
41
|
+
logger.error("retries must be a non-negative integer")
|
|
42
42
|
raise ValueError
|
|
43
43
|
if not isinstance(initial_delay, int) or initial_delay < 0:
|
|
44
|
-
|
|
44
|
+
logger.error("initial_delay must be a non-negative integer")
|
|
45
45
|
raise ValueError
|
|
46
46
|
if not isinstance(connection_timeout, int) or connection_timeout < 0:
|
|
47
|
-
|
|
47
|
+
logger.error("connection_timeout must be a non-negative integer")
|
|
48
48
|
raise ValueError
|
|
49
49
|
|
|
50
50
|
self.endpoint = endpoint
|
|
@@ -53,10 +53,9 @@ class ClientBuilder:
|
|
|
53
53
|
self.headers = headers
|
|
54
54
|
self.retries = retries
|
|
55
55
|
self.delay = initial_delay
|
|
56
|
-
self.
|
|
57
|
-
|
|
58
|
-
self.
|
|
59
|
-
|
|
56
|
+
self._o11y_wrapper = OpenObservability(application_name="api-to-dataframe").get_wrapper()
|
|
57
|
+
self._traces = self._o11y_wrapper.traces()
|
|
58
|
+
self._tracer = self._traces.get_tracer()
|
|
60
59
|
|
|
61
60
|
@retry_strategies
|
|
62
61
|
def get_api_data(self):
|
|
@@ -71,16 +70,15 @@ class ClientBuilder:
|
|
|
71
70
|
dict: The JSON response from the API as a dictionary.
|
|
72
71
|
"""
|
|
73
72
|
|
|
74
|
-
self.
|
|
75
|
-
|
|
76
|
-
response = GetData.get_response(
|
|
77
|
-
endpoint=self.endpoint,
|
|
78
|
-
headers=self.headers,
|
|
79
|
-
connection_timeout=self.connection_timeout,
|
|
80
|
-
)
|
|
73
|
+
with self._tracer.start_as_current_span("get_last_quote") as span:
|
|
74
|
+
span.set_attribute("endpoint", self.endpoint)
|
|
81
75
|
|
|
76
|
+
response = GetData.get_response(
|
|
77
|
+
endpoint=self.endpoint,
|
|
78
|
+
headers=self.headers,
|
|
79
|
+
connection_timeout=self.connection_timeout,
|
|
80
|
+
)
|
|
82
81
|
|
|
83
|
-
self.metrics_client.increment("get_api_data_success")
|
|
84
82
|
|
|
85
83
|
return response.json()
|
|
86
84
|
|
|
@@ -99,9 +97,7 @@ class ClientBuilder:
|
|
|
99
97
|
Returns:
|
|
100
98
|
DataFrame: A pandas DataFrame containing the data from the API response.
|
|
101
99
|
"""
|
|
102
|
-
MetricsClient().increment("api_to_dataframe_attempt")
|
|
103
100
|
|
|
104
101
|
df = GetData.to_dataframe(response)
|
|
105
102
|
|
|
106
|
-
MetricsClient().increment("api_to_dataframe_success")
|
|
107
103
|
return df
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import requests
|
|
2
2
|
import pandas as pd
|
|
3
3
|
|
|
4
|
-
from api_to_dataframe.utils.logger import log, LogLevel
|
|
5
|
-
|
|
6
4
|
|
|
7
5
|
class GetData:
|
|
8
6
|
@staticmethod
|
|
@@ -16,13 +14,11 @@ class GetData:
|
|
|
16
14
|
try:
|
|
17
15
|
df = pd.DataFrame(response)
|
|
18
16
|
except Exception as err:
|
|
19
|
-
log(f"Error serializing to dataframe: {err}", LogLevel.ERROR)
|
|
20
17
|
raise TypeError(
|
|
21
18
|
f"Invalid response for transform in dataframe: {err}"
|
|
22
19
|
) from err
|
|
23
20
|
|
|
24
21
|
if df.empty:
|
|
25
|
-
log("DataFrame is empty", LogLevel.ERROR)
|
|
26
22
|
raise ValueError("::: DataFrame is empty :::")
|
|
27
23
|
|
|
28
24
|
return df
|
|
@@ -2,7 +2,7 @@ import time
|
|
|
2
2
|
from enum import Enum
|
|
3
3
|
|
|
4
4
|
from requests.exceptions import RequestException
|
|
5
|
-
from api_to_dataframe.utils.logger import
|
|
5
|
+
from api_to_dataframe.utils.logger import logger
|
|
6
6
|
from api_to_dataframe.utils import Constants
|
|
7
7
|
|
|
8
8
|
|
|
@@ -18,11 +18,7 @@ def retry_strategies(func):
|
|
|
18
18
|
while retry_number < args[0].retries:
|
|
19
19
|
try:
|
|
20
20
|
if retry_number > 0:
|
|
21
|
-
|
|
22
|
-
f"Trying for the {retry_number} of {Constants.MAX_OF_RETRIES} retries. "
|
|
23
|
-
f"Using {args[0].retry_strategy}",
|
|
24
|
-
LogLevel.INFO,
|
|
25
|
-
)
|
|
21
|
+
logger.info(f"Trying for the {retry_number} of {Constants.MAX_OF_RETRIES} retries. Using {args[0].retry_strategy}")
|
|
26
22
|
return func(*args, **kwargs)
|
|
27
23
|
except RequestException as e:
|
|
28
24
|
retry_number += 1
|
|
@@ -35,10 +31,7 @@ def retry_strategies(func):
|
|
|
35
31
|
time.sleep(args[0].delay * retry_number)
|
|
36
32
|
|
|
37
33
|
if retry_number in (args[0].retries, Constants.MAX_OF_RETRIES):
|
|
38
|
-
|
|
39
|
-
f"Failed after {retry_number} retries using {args[0].retry_strategy}",
|
|
40
|
-
LogLevel.ERROR,
|
|
41
|
-
)
|
|
34
|
+
logger.error(f"Failed after {retry_number} retries")
|
|
42
35
|
raise e
|
|
43
36
|
|
|
44
37
|
return wrapper
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
# Configure logging once at the start of your program
|
|
4
|
+
logging.basicConfig(
|
|
5
|
+
encoding="utf-8",
|
|
6
|
+
format="%(asctime)s :: api-to-dataframe[%(levelname)s] :: %(message)s",
|
|
7
|
+
datefmt="%Y-%m-%d %H:%M:%S %Z",
|
|
8
|
+
level=logging.INFO,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger("api-to-dataframe")
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from enum import Enum
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
class LogLevel(Enum):
|
|
6
|
-
INFO = logging.INFO
|
|
7
|
-
WARNING = logging.WARNING
|
|
8
|
-
ERROR = logging.ERROR
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
# Configure logging once at the start of your program
|
|
12
|
-
logging.basicConfig(
|
|
13
|
-
encoding="utf-8",
|
|
14
|
-
format="%(asctime)s :: api-to-dataframe[%(levelname)s] :: %(message)s",
|
|
15
|
-
datefmt="%Y-%m-%d %H:%M:%S %Z",
|
|
16
|
-
level=logging.INFO,
|
|
17
|
-
)
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def log(message: str, level: LogLevel):
|
|
21
|
-
logger = logging.getLogger("api-to-dataframe")
|
|
22
|
-
logger.log(level.value, message)
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
from datadog import initialize, statsd
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
class MetricsClient:
|
|
5
|
-
def __init__(self) -> None:
|
|
6
|
-
self.options = options = {
|
|
7
|
-
"statsd_host": "node-metrics-ba28.ivanildobarauna.dev",
|
|
8
|
-
"statsd_port": 8125,
|
|
9
|
-
}
|
|
10
|
-
self.client = initialize(**options)
|
|
11
|
-
self.application_name = "api-to-dataframe"
|
|
12
|
-
|
|
13
|
-
def increment(
|
|
14
|
-
self, action: str, tags: list[str] = ["env:production"], value: float = 1.0
|
|
15
|
-
):
|
|
16
|
-
|
|
17
|
-
tags.append(f"action:{action}")
|
|
18
|
-
|
|
19
|
-
statsd.increment(metric=self.application_name, tags=tags, value=value)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|