tagreader 6.2.0__tar.gz → 6.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tagreader-6.2.0 → tagreader-6.3.0}/PKG-INFO +6 -5
- {tagreader-6.2.0 → tagreader-6.3.0}/README.md +4 -3
- {tagreader-6.2.0 → tagreader-6.3.0}/pyproject.toml +2 -2
- {tagreader-6.2.0 → tagreader-6.3.0}/tagreader/cache.py +2 -3
- {tagreader-6.2.0 → tagreader-6.3.0}/tagreader/clients.py +41 -21
- {tagreader-6.2.0 → tagreader-6.3.0}/tagreader/utils.py +17 -5
- {tagreader-6.2.0 → tagreader-6.3.0}/tagreader/web_handlers.py +179 -81
- {tagreader-6.2.0 → tagreader-6.3.0}/LICENSE +0 -0
- {tagreader-6.2.0 → tagreader-6.3.0}/tagreader/__init__.py +0 -0
- {tagreader-6.2.0 → tagreader-6.3.0}/tagreader/__version__.py +0 -0
- {tagreader-6.2.0 → tagreader-6.3.0}/tagreader/logger.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tagreader
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.0
|
|
4
4
|
Summary: Tagreader is a Python package for reading trend data from the OSIsoft PI and Aspen Infoplus.21 IMS systems.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -25,7 +25,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
25
25
|
Classifier: Topic :: Utilities
|
|
26
26
|
Requires-Dist: certifi (>=2024.12.14,<2026.0.0)
|
|
27
27
|
Requires-Dist: diskcache (>=5.6.1)
|
|
28
|
-
Requires-Dist: msal-bearer (>=1.
|
|
28
|
+
Requires-Dist: msal-bearer (>=1.5.2)
|
|
29
29
|
Requires-Dist: pandas (>=1)
|
|
30
30
|
Requires-Dist: platformdirs (>=4.3.7)
|
|
31
31
|
Requires-Dist: pycryptodome (>=3.20.0)
|
|
@@ -37,6 +37,10 @@ Project-URL: Homepage, https://github.com/equinor/tagreader-python
|
|
|
37
37
|
Project-URL: Repository, https://github.com/equinor/tagreader-python
|
|
38
38
|
Description-Content-Type: text/markdown
|
|
39
39
|
|
|
40
|
+
⚠️ **Important:** **We are phasing out TagReader, and the code will no longer be maintained.**
|
|
41
|
+
|
|
42
|
+
For employees in Equinor we recommend using Omnia Timeseries
|
|
43
|
+
|
|
40
44
|
# tagreader-python <!-- omit in toc -->
|
|
41
45
|
|
|
42
46
|

|
|
@@ -86,6 +90,3 @@ For more details, see the [Tagreader Docs](https://equinor.github.io/tagreader-p
|
|
|
86
90
|
## Documentation
|
|
87
91
|
The full documentation can be found in [Tagreader Docs](https://equinor.github.io/tagreader-python/)
|
|
88
92
|
|
|
89
|
-
## Contribute
|
|
90
|
-
To starting contributing, please see [Tagreader Docs - Contribute](https://equinor.github.io/tagreader-python/docs/contribute/overview)
|
|
91
|
-
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
⚠️ **Important:** **We are phasing out TagReader, and the code will no longer be maintained.**
|
|
2
|
+
|
|
3
|
+
For employees in Equinor we recommend using Omnia Timeseries
|
|
4
|
+
|
|
1
5
|
# tagreader-python <!-- omit in toc -->
|
|
2
6
|
|
|
3
7
|

|
|
@@ -46,6 +50,3 @@ For more details, see the [Tagreader Docs](https://equinor.github.io/tagreader-p
|
|
|
46
50
|
|
|
47
51
|
## Documentation
|
|
48
52
|
The full documentation can be found in [Tagreader Docs](https://equinor.github.io/tagreader-python/)
|
|
49
|
-
|
|
50
|
-
## Contribute
|
|
51
|
-
To starting contributing, please see [Tagreader Docs - Contribute](https://equinor.github.io/tagreader-python/docs/contribute/overview)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "tagreader"
|
|
3
|
-
version = "6.
|
|
3
|
+
version = "6.3.0"
|
|
4
4
|
description = "Tagreader is a Python package for reading trend data from the OSIsoft PI and Aspen Infoplus.21 IMS systems."
|
|
5
5
|
authors = [
|
|
6
6
|
{ name = "Einar S. Idsø", email = "eiids@equinor.com" },
|
|
@@ -32,7 +32,7 @@ dependencies = [
|
|
|
32
32
|
"certifi>=2024.12.14,<2026.0.0",
|
|
33
33
|
"requests>=2.32.5",
|
|
34
34
|
"requests-kerberos>=0",
|
|
35
|
-
"msal-bearer>=1.
|
|
35
|
+
"msal-bearer>=1.5.2",
|
|
36
36
|
"diskcache>=5.6.1",
|
|
37
37
|
"pycryptodome>=3.20.0",
|
|
38
38
|
"requests-ntlm>=1.1,<=2.0",
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
from datetime import datetime, timedelta
|
|
1
|
+
from datetime import datetime, timedelta, timezone
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import Dict, List, Optional, Tuple, Union, cast
|
|
4
4
|
|
|
5
5
|
import pandas as pd
|
|
6
|
-
import pytz
|
|
7
6
|
from diskcache import Cache
|
|
8
7
|
|
|
9
8
|
from tagreader.logger import logger
|
|
@@ -21,7 +20,7 @@ def safe_tagname(tagname: str) -> str:
|
|
|
21
20
|
def timestamp_to_epoch(timestamp: datetime) -> int:
|
|
22
21
|
origin = datetime(1970, 1, 1)
|
|
23
22
|
if timestamp.tzinfo is not None:
|
|
24
|
-
timestamp = timestamp.astimezone(
|
|
23
|
+
timestamp = timestamp.astimezone(timezone.utc).replace(tzinfo=None)
|
|
25
24
|
return (timestamp - origin) // timedelta(seconds=1)
|
|
26
25
|
|
|
27
26
|
|
|
@@ -2,11 +2,11 @@ from datetime import datetime, timedelta, timezone, tzinfo
|
|
|
2
2
|
from itertools import groupby
|
|
3
3
|
from operator import itemgetter
|
|
4
4
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
5
|
-
from
|
|
5
|
+
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
6
6
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
import pandas as pd
|
|
9
|
-
import
|
|
9
|
+
import requests
|
|
10
10
|
|
|
11
11
|
from tagreader.cache import BucketCache, SmartCache
|
|
12
12
|
from tagreader.logger import logger
|
|
@@ -21,11 +21,12 @@ from tagreader.web_handlers import (
|
|
|
21
21
|
PIHandlerWeb,
|
|
22
22
|
get_auth_aspen,
|
|
23
23
|
get_auth_pi,
|
|
24
|
+
get_url_aspen,
|
|
24
25
|
list_aspenone_sources,
|
|
25
26
|
list_piwebapi_sources,
|
|
26
27
|
)
|
|
27
28
|
|
|
28
|
-
NONE_START_TIME = datetime(1970, 1, 1, tzinfo=
|
|
29
|
+
NONE_START_TIME = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
|
29
30
|
|
|
30
31
|
|
|
31
32
|
def list_sources(
|
|
@@ -49,8 +50,6 @@ def list_sources(
|
|
|
49
50
|
auth = get_auth_pi()
|
|
50
51
|
return list_piwebapi_sources(url=url, auth=auth, verify_ssl=verify_ssl)
|
|
51
52
|
elif imstype == IMSType.ASPENONE:
|
|
52
|
-
if auth is None:
|
|
53
|
-
auth = get_auth_aspen()
|
|
54
53
|
return list_aspenone_sources(url=url, auth=auth, verify_ssl=verify_ssl)
|
|
55
54
|
elif imstype in [IMSType.PI, IMSType.ASPEN, IMSType.IP21]:
|
|
56
55
|
raise ValueError(
|
|
@@ -121,33 +120,50 @@ def get_handler(
|
|
|
121
120
|
url: Optional[str],
|
|
122
121
|
options: Dict[str, Union[int, float, str]],
|
|
123
122
|
verify_ssl: Optional[Union[bool, str]],
|
|
124
|
-
auth: Optional[Any],
|
|
123
|
+
auth: Optional[Any] = None,
|
|
125
124
|
cache: Optional[Union[SmartCache, BucketCache]] = None,
|
|
126
125
|
):
|
|
127
126
|
if imstype is None:
|
|
127
|
+
orig_auth = auth
|
|
128
|
+
orig_url = url
|
|
128
129
|
try:
|
|
129
|
-
|
|
130
|
+
aspen_source = list_aspenone_sources(
|
|
130
131
|
url=None, auth=None, verify_ssl=verify_ssl
|
|
131
|
-
)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
132
|
+
)
|
|
133
|
+
except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError):
|
|
134
|
+
# Try again using app registration auth
|
|
135
|
+
try:
|
|
136
|
+
auth = get_auth_aspen(False)
|
|
137
|
+
url = get_url_aspen(False)
|
|
138
|
+
aspen_source = list_aspenone_sources(
|
|
139
|
+
auth=auth, url=url, verify_ssl=verify_ssl
|
|
140
|
+
)
|
|
141
|
+
except requests.exceptions.HTTPError as e:
|
|
142
|
+
logger.debug(f"Could not list Aspenone sources: {e}")
|
|
143
|
+
aspen_source = []
|
|
144
|
+
|
|
145
|
+
if datasource in aspen_source:
|
|
146
|
+
imstype = IMSType.ASPENONE
|
|
147
|
+
else:
|
|
148
|
+
auth = orig_auth
|
|
149
|
+
url = orig_url
|
|
150
|
+
|
|
135
151
|
if imstype is None:
|
|
136
152
|
try:
|
|
137
153
|
if datasource in list_piwebapi_sources(
|
|
138
154
|
url=None, auth=None, verify_ssl=verify_ssl
|
|
139
155
|
):
|
|
140
156
|
imstype = IMSType.PIWEBAPI
|
|
141
|
-
except HTTPError as e:
|
|
157
|
+
except requests.exceptions.HTTPError as e:
|
|
142
158
|
logger.debug(f"Could not list PI sources: {e}")
|
|
143
159
|
|
|
144
160
|
if imstype == IMSType.PIWEBAPI:
|
|
145
161
|
return PIHandlerWeb(
|
|
146
|
-
url=url,
|
|
147
162
|
datasource=datasource,
|
|
148
|
-
|
|
149
|
-
verify_ssl=verify_ssl,
|
|
163
|
+
url=url,
|
|
150
164
|
auth=auth,
|
|
165
|
+
verify_ssl=verify_ssl,
|
|
166
|
+
options=options,
|
|
151
167
|
cache=cache,
|
|
152
168
|
)
|
|
153
169
|
|
|
@@ -175,7 +191,7 @@ class IMSClient:
|
|
|
175
191
|
self,
|
|
176
192
|
datasource: str,
|
|
177
193
|
imstype: Optional[Union[str, IMSType]] = None,
|
|
178
|
-
tz: Union[tzinfo, str] =
|
|
194
|
+
tz: Optional[Union[tzinfo, str]] = None,
|
|
179
195
|
url: Optional[str] = None,
|
|
180
196
|
handler_options: Dict[str, Union[int, float, str]] = {}, # noqa:
|
|
181
197
|
verify_ssl: Optional[Union[bool, str]] = True,
|
|
@@ -191,11 +207,15 @@ class IMSClient:
|
|
|
191
207
|
f" We suggest to use the tagreader.IMSType enumerator when initiating a client."
|
|
192
208
|
)
|
|
193
209
|
|
|
210
|
+
if tz is None or tz == "":
|
|
211
|
+
tz = ZoneInfo("Europe/Oslo")
|
|
194
212
|
if isinstance(tz, str):
|
|
195
|
-
|
|
196
|
-
self.tz =
|
|
197
|
-
|
|
198
|
-
raise ValueError(
|
|
213
|
+
try:
|
|
214
|
+
self.tz = ZoneInfo(tz)
|
|
215
|
+
except ZoneInfoNotFoundError:
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"Invalid timezone string. Given tz was {tz}"
|
|
218
|
+
) from None
|
|
199
219
|
elif isinstance(tz, tzinfo):
|
|
200
220
|
self.tz = tz
|
|
201
221
|
else:
|
|
@@ -345,7 +365,7 @@ class IMSClient:
|
|
|
345
365
|
end=end,
|
|
346
366
|
)
|
|
347
367
|
frames.append(df)
|
|
348
|
-
if len(df) < self.handler.
|
|
368
|
+
if len(df) < self.handler.max_rows:
|
|
349
369
|
break
|
|
350
370
|
start = df.index[-1]
|
|
351
371
|
|
|
@@ -3,14 +3,14 @@ import hashlib
|
|
|
3
3
|
import platform
|
|
4
4
|
import socket
|
|
5
5
|
import ssl
|
|
6
|
-
from datetime import datetime, tzinfo
|
|
6
|
+
from datetime import datetime, timezone, tzinfo
|
|
7
7
|
from enum import Enum
|
|
8
8
|
from pathlib import Path
|
|
9
|
-
from typing import Union
|
|
9
|
+
from typing import Optional, Union
|
|
10
|
+
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
10
11
|
|
|
11
12
|
import certifi
|
|
12
13
|
import pandas as pd
|
|
13
|
-
import pytz
|
|
14
14
|
import requests
|
|
15
15
|
from platformdirs import user_data_dir
|
|
16
16
|
|
|
@@ -50,12 +50,24 @@ def convert_to_pydatetime(date_stamp: Union[datetime, str, pd.Timestamp]) -> dat
|
|
|
50
50
|
|
|
51
51
|
def ensure_datetime_with_tz(
|
|
52
52
|
date_stamp: Union[datetime, str, pd.Timestamp],
|
|
53
|
-
tz: tzinfo =
|
|
53
|
+
tz: Optional[tzinfo] = None,
|
|
54
54
|
) -> datetime:
|
|
55
|
+
if tz is None:
|
|
56
|
+
try:
|
|
57
|
+
tz = ZoneInfo("Europe/Oslo")
|
|
58
|
+
except ZoneInfoNotFoundError:
|
|
59
|
+
logger.warning(
|
|
60
|
+
"Unable to load timezone 'Europe/Oslo'. Falling back to UTC."
|
|
61
|
+
)
|
|
62
|
+
tz = timezone.utc
|
|
55
63
|
date_stamp = convert_to_pydatetime(date_stamp)
|
|
56
64
|
|
|
57
65
|
if not date_stamp.tzinfo:
|
|
58
|
-
|
|
66
|
+
localize = getattr(tz, "localize", None)
|
|
67
|
+
if callable(localize):
|
|
68
|
+
date_stamp = localize(date_stamp)
|
|
69
|
+
else:
|
|
70
|
+
date_stamp = date_stamp.replace(tzinfo=tz)
|
|
59
71
|
|
|
60
72
|
return date_stamp
|
|
61
73
|
|
|
@@ -3,14 +3,13 @@ import json
|
|
|
3
3
|
import re
|
|
4
4
|
import urllib.parse
|
|
5
5
|
from abc import ABC, abstractmethod
|
|
6
|
-
from datetime import datetime, timedelta
|
|
6
|
+
from datetime import datetime, timedelta, timezone
|
|
7
7
|
from hashlib import new as hashlib_new_method
|
|
8
8
|
from json.decoder import JSONDecodeError
|
|
9
9
|
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
10
10
|
|
|
11
11
|
import numpy as np
|
|
12
12
|
import pandas as pd
|
|
13
|
-
import pytz
|
|
14
13
|
import requests
|
|
15
14
|
import urllib3
|
|
16
15
|
from Crypto.Hash import MD4 as _MD4
|
|
@@ -87,17 +86,11 @@ def get_url_aspen(use_internal: bool = True) -> str:
|
|
|
87
86
|
return r"https://ewepwapa1pep04-statoilsrm.msappproxy.net/ProcessExplorer/ProcessData/AtProcessDataREST.dll"
|
|
88
87
|
|
|
89
88
|
|
|
90
|
-
def
|
|
91
|
-
url:
|
|
92
|
-
auth:
|
|
93
|
-
verify_ssl: Optional[Union[bool, str]]
|
|
89
|
+
def _fetch_aspenone_sources(
|
|
90
|
+
url: str,
|
|
91
|
+
auth: Any,
|
|
92
|
+
verify_ssl: Optional[Union[bool, str]],
|
|
94
93
|
) -> List[str]:
|
|
95
|
-
if url is None:
|
|
96
|
-
url = get_url_aspen()
|
|
97
|
-
|
|
98
|
-
if auth is None:
|
|
99
|
-
auth = get_auth_aspen()
|
|
100
|
-
|
|
101
94
|
if verify_ssl is None:
|
|
102
95
|
verify_ssl = get_verify_ssl()
|
|
103
96
|
|
|
@@ -115,18 +108,38 @@ def list_aspenone_sources(
|
|
|
115
108
|
except JSONDecodeError as e:
|
|
116
109
|
logger.error(f"Could not decode JSON response: {e}")
|
|
117
110
|
|
|
111
|
+
return []
|
|
118
112
|
|
|
119
|
-
|
|
113
|
+
|
|
114
|
+
# Keyed on url only so default-auth calls share the cache regardless of auth or verify_ssl.
|
|
115
|
+
_aspenone_sources_cache: Dict[str, List[str]] = {}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def list_aspenone_sources(
|
|
120
119
|
url: Optional[str] = None,
|
|
121
120
|
auth: Optional[Any] = None,
|
|
122
121
|
verify_ssl: Optional[Union[bool, str]] = True,
|
|
123
122
|
) -> List[str]:
|
|
124
123
|
if url is None:
|
|
125
|
-
url =
|
|
124
|
+
url = get_url_aspen()
|
|
126
125
|
|
|
127
|
-
if auth is None:
|
|
128
|
-
auth
|
|
126
|
+
if auth is not None:
|
|
127
|
+
return _fetch_aspenone_sources(url, auth, verify_ssl)
|
|
128
|
+
|
|
129
|
+
if url not in _aspenone_sources_cache:
|
|
130
|
+
url_sources = _fetch_aspenone_sources(url, get_auth_aspen(), verify_ssl)
|
|
131
|
+
if len(url_sources) > 0:
|
|
132
|
+
_aspenone_sources_cache[url] = url_sources
|
|
133
|
+
else:
|
|
134
|
+
return []
|
|
135
|
+
return list(_aspenone_sources_cache[url])
|
|
129
136
|
|
|
137
|
+
|
|
138
|
+
def _fetch_piwebapi_sources(
|
|
139
|
+
url: str,
|
|
140
|
+
auth: Any,
|
|
141
|
+
verify_ssl: Optional[Union[bool, str]],
|
|
142
|
+
) -> List[str]:
|
|
130
143
|
if verify_ssl is None:
|
|
131
144
|
verify_ssl = get_verify_ssl()
|
|
132
145
|
|
|
@@ -143,6 +156,30 @@ def list_piwebapi_sources(
|
|
|
143
156
|
except JSONDecodeError as e:
|
|
144
157
|
logger.error(f"Could not decode JSON response: {e}")
|
|
145
158
|
|
|
159
|
+
return []
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# Keyed on url only so default-auth calls share the cache regardless of auth or verify_ssl.
|
|
163
|
+
_piwebapi_sources_cache: Dict[str, List[str]] = {}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def list_piwebapi_sources(
|
|
167
|
+
url: Optional[str] = None,
|
|
168
|
+
auth: Optional[Any] = None,
|
|
169
|
+
verify_ssl: Optional[Union[bool, str]] = True,
|
|
170
|
+
) -> List[str]:
|
|
171
|
+
if url is None:
|
|
172
|
+
url = get_url_pi()
|
|
173
|
+
|
|
174
|
+
if auth is not None:
|
|
175
|
+
return _fetch_piwebapi_sources(url, auth, verify_ssl)
|
|
176
|
+
|
|
177
|
+
if url not in _piwebapi_sources_cache:
|
|
178
|
+
url_sources = _fetch_piwebapi_sources(url, get_auth_pi(), verify_ssl)
|
|
179
|
+
if len(url_sources) > 0:
|
|
180
|
+
_piwebapi_sources_cache[url] = url_sources
|
|
181
|
+
return list(_piwebapi_sources_cache[url])
|
|
182
|
+
|
|
146
183
|
|
|
147
184
|
def get_piwebapi_source_to_webid_dict(
|
|
148
185
|
url: Optional[str] = None,
|
|
@@ -170,6 +207,8 @@ def get_piwebapi_source_to_webid_dict(
|
|
|
170
207
|
except JSONDecodeError as e:
|
|
171
208
|
logger.error(f"Could not decode JSON response: {e}")
|
|
172
209
|
|
|
210
|
+
return []
|
|
211
|
+
|
|
173
212
|
|
|
174
213
|
class BaseHandlerWeb(ABC):
|
|
175
214
|
def __init__(
|
|
@@ -181,6 +220,7 @@ class BaseHandlerWeb(ABC):
|
|
|
181
220
|
):
|
|
182
221
|
self.datasource = datasource
|
|
183
222
|
self.base_url = url
|
|
223
|
+
self.max_rows = 10000
|
|
184
224
|
self.session = requests.Session()
|
|
185
225
|
self.auth = auth
|
|
186
226
|
self.session.auth = auth if auth is not None else get_auth_aspen()
|
|
@@ -188,20 +228,66 @@ class BaseHandlerWeb(ABC):
|
|
|
188
228
|
urllib3.disable_warnings(InsecureRequestWarning)
|
|
189
229
|
self.session.verify = verify_ssl if verify_ssl is not None else get_verify_ssl()
|
|
190
230
|
|
|
231
|
+
@property
|
|
232
|
+
def datasource(self) -> str:
|
|
233
|
+
return self._datasource
|
|
234
|
+
|
|
235
|
+
@datasource.setter
|
|
236
|
+
def datasource(self, value: Optional[str]) -> None:
|
|
237
|
+
self._datasource = value if value is not None else ""
|
|
238
|
+
|
|
239
|
+
@property
|
|
240
|
+
def base_url(self) -> str:
|
|
241
|
+
return self._base_url
|
|
242
|
+
|
|
243
|
+
@base_url.setter
|
|
244
|
+
def base_url(self, value: Optional[str]) -> None:
|
|
245
|
+
self._base_url = value if value is not None else ""
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def max_rows(self) -> int:
|
|
249
|
+
return self._max_rows
|
|
250
|
+
|
|
251
|
+
@max_rows.setter
|
|
252
|
+
def max_rows(self, value: int) -> None:
|
|
253
|
+
self._max_rows = value
|
|
254
|
+
|
|
191
255
|
def fetch(
|
|
192
256
|
self,
|
|
193
257
|
url,
|
|
194
|
-
params: Optional[Union[str, Dict[str, str]]] = None,
|
|
258
|
+
params: Optional[Union[bytes, str, Dict[str, str]]] = None,
|
|
195
259
|
timeout: Optional[int] = None,
|
|
260
|
+
use_post: bool = False,
|
|
196
261
|
) -> Dict:
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
params
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
262
|
+
if use_post:
|
|
263
|
+
headers = {}
|
|
264
|
+
if isinstance(params, dict):
|
|
265
|
+
params = urllib.parse.urlencode(
|
|
266
|
+
params, safe="*", quote_via=urllib.parse.quote
|
|
267
|
+
).encode("utf-8")
|
|
268
|
+
headers["Content-Type"] = (
|
|
269
|
+
"application/x-www-form-urlencoded; charset=utf-8"
|
|
270
|
+
)
|
|
271
|
+
elif isinstance(params, str):
|
|
272
|
+
params = params.encode("utf-8")
|
|
273
|
+
headers["Content-Type"] = "text/xml; charset=utf-8"
|
|
274
|
+
|
|
275
|
+
res = self.session.post(
|
|
276
|
+
url,
|
|
277
|
+
data=params,
|
|
278
|
+
headers=headers,
|
|
279
|
+
timeout=(None, timeout),
|
|
280
|
+
)
|
|
281
|
+
else:
|
|
282
|
+
res = self.session.get(
|
|
283
|
+
url,
|
|
284
|
+
params=params,
|
|
285
|
+
timeout=(
|
|
286
|
+
None,
|
|
287
|
+
timeout,
|
|
288
|
+
),
|
|
289
|
+
) # Noqa. Read timeout, No connect timeout.
|
|
290
|
+
|
|
205
291
|
res.raise_for_status()
|
|
206
292
|
|
|
207
293
|
if len(res.text) == 0:
|
|
@@ -223,15 +309,18 @@ class BaseHandlerWeb(ABC):
|
|
|
223
309
|
return json.loads(txt)
|
|
224
310
|
|
|
225
311
|
def connect(self):
|
|
312
|
+
if self.datasource == "":
|
|
313
|
+
raise ValueError("Datasource must be specified")
|
|
314
|
+
|
|
226
315
|
try:
|
|
227
316
|
self.verify_connection(self.datasource)
|
|
228
317
|
except requests.ConnectionError:
|
|
229
318
|
raise ConnectionError(
|
|
230
|
-
f"Not able to connect to {self.base_url}. Check network connection."
|
|
319
|
+
f"Not able to connect to api at {self.base_url}. Check network connection."
|
|
231
320
|
) from None
|
|
232
321
|
|
|
233
322
|
@abstractmethod
|
|
234
|
-
def verify_connection(self, datasource: str): ...
|
|
323
|
+
def verify_connection(self, datasource: str) -> bool: ...
|
|
235
324
|
|
|
236
325
|
|
|
237
326
|
class AspenHandlerWeb(BaseHandlerWeb):
|
|
@@ -243,7 +332,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
243
332
|
verify_ssl: Optional[Union[bool, str]] = True,
|
|
244
333
|
options: Dict[str, Any] = dict(),
|
|
245
334
|
):
|
|
246
|
-
if url is None:
|
|
335
|
+
if url is None or url == "":
|
|
247
336
|
url = get_url_aspen()
|
|
248
337
|
if auth is None:
|
|
249
338
|
auth = get_auth_aspen()
|
|
@@ -254,7 +343,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
254
343
|
auth=auth,
|
|
255
344
|
verify_ssl=verify_ssl,
|
|
256
345
|
)
|
|
257
|
-
self.
|
|
346
|
+
self.max_rows = int(options.get("max_rows", 100000))
|
|
258
347
|
self._connection_string = "" # Used for raw SQL queries
|
|
259
348
|
|
|
260
349
|
@staticmethod
|
|
@@ -266,6 +355,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
266
355
|
) -> Dict[str, Any]:
|
|
267
356
|
if not datasource:
|
|
268
357
|
raise ValueError("Data source is required argument")
|
|
358
|
+
|
|
269
359
|
# Aspen Web API expects single space instead of consecutive spaces.
|
|
270
360
|
tag = " ".join(tag.split())
|
|
271
361
|
params = {"datasource": datasource, "tag": tag, "max": max, "getTrendable": 0}
|
|
@@ -333,7 +423,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
333
423
|
f"<RT>{rt}</RT>"
|
|
334
424
|
)
|
|
335
425
|
if read_type in [ReaderType.RAW, ReaderType.SHAPEPRESERVING]:
|
|
336
|
-
query += f"<X>{self.
|
|
426
|
+
query += f"<X>{self.max_rows}</X>"
|
|
337
427
|
if read_type not in [ReaderType.INT, ReaderType.SNAPSHOT]:
|
|
338
428
|
query += f"<O>{outsiders}</O>"
|
|
339
429
|
if read_type not in [ReaderType.RAW]:
|
|
@@ -367,7 +457,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
367
457
|
|
|
368
458
|
return query
|
|
369
459
|
|
|
370
|
-
def verify_connection(self, datasource: str):
|
|
460
|
+
def verify_connection(self, datasource: str) -> bool:
|
|
371
461
|
"""Connects to the URL and verifies that the provided data source exists.
|
|
372
462
|
|
|
373
463
|
:param datasource: Data source to look for
|
|
@@ -424,7 +514,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
424
514
|
def _get_maps(self, tagname: str):
|
|
425
515
|
params = self.generate_get_map_query(tagname)
|
|
426
516
|
url = urljoin(self.base_url, "TagInfo")
|
|
427
|
-
data = self.fetch(url, params=params)
|
|
517
|
+
data = self.fetch(url, params=params, use_post=not tagname.isascii())
|
|
428
518
|
|
|
429
519
|
if "tags" not in data["data"]:
|
|
430
520
|
return {}
|
|
@@ -437,7 +527,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
437
527
|
return ret
|
|
438
528
|
|
|
439
529
|
def _get_default_mapname(self, tagname: str):
|
|
440
|
-
|
|
530
|
+
tagname, _ = self.split_tagmap(tagname)
|
|
441
531
|
all_maps = self._get_maps(tagname)
|
|
442
532
|
for k, v in all_maps.items():
|
|
443
533
|
if v:
|
|
@@ -452,18 +542,12 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
452
542
|
) -> Union[List[Tuple[str, str]], List[str]]:
|
|
453
543
|
ret = []
|
|
454
544
|
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
elif tag is None and len(desc) == 0:
|
|
459
|
-
raise ValueError("Tag is a required argument")
|
|
460
|
-
# return ret
|
|
461
|
-
elif desc is None and len(tag) == 0:
|
|
462
|
-
raise ValueError("Tag is a required argument")
|
|
463
|
-
# return ret
|
|
545
|
+
# Currently not possible to search only using description.
|
|
546
|
+
if not tag:
|
|
547
|
+
raise ValueError("Input tag is a required argument")
|
|
464
548
|
|
|
465
|
-
if tag
|
|
466
|
-
raise ValueError("
|
|
549
|
+
if not tag and not desc:
|
|
550
|
+
raise ValueError("Either tag or desc must be provided.")
|
|
467
551
|
|
|
468
552
|
tag = tag.replace("%", "*") if isinstance(tag, str) else None
|
|
469
553
|
# Prepare for regex
|
|
@@ -513,7 +597,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
513
597
|
def _get_tag_unit(self, tag: str):
|
|
514
598
|
query = self.generate_get_unit_query(tag)
|
|
515
599
|
url = urljoin(self.base_url, "TagInfo")
|
|
516
|
-
data = self.fetch(url, params=query)
|
|
600
|
+
data = self.fetch(url, params=query, use_post=not tag.isascii())
|
|
517
601
|
# try:
|
|
518
602
|
attr_data = data["data"]["tags"][0]["attrData"]
|
|
519
603
|
# except KeyError as e:
|
|
@@ -548,7 +632,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
548
632
|
query = self.generate_get_description_query(tag)
|
|
549
633
|
url = urljoin(self.base_url, "TagInfo")
|
|
550
634
|
try:
|
|
551
|
-
data = self.fetch(url, params=query)
|
|
635
|
+
data = self.fetch(url, params=query, use_post=not tag.isascii())
|
|
552
636
|
desc = data["data"]["tags"][0]["attrData"][0]["samples"][0]["v"]
|
|
553
637
|
# except KeyError:
|
|
554
638
|
# desc = ""
|
|
@@ -565,6 +649,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
565
649
|
read_type: ReaderType,
|
|
566
650
|
metadata: Optional[Dict[str, str]],
|
|
567
651
|
get_status: bool = False,
|
|
652
|
+
use_post: bool = False,
|
|
568
653
|
):
|
|
569
654
|
if read_type not in [
|
|
570
655
|
ReaderType.INT,
|
|
@@ -592,9 +677,13 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
592
677
|
# so we need to limit the range. Note -1 because INT normally includes
|
|
593
678
|
# both start and end time.
|
|
594
679
|
if read_type == ReaderType.INT:
|
|
595
|
-
end = min(end, start + sample_time * (self.
|
|
680
|
+
end = min(end, start + sample_time * (self.max_rows - 1))
|
|
596
681
|
|
|
597
682
|
tag_name, map_name = self.split_tagmap(tag)
|
|
683
|
+
if tag_name is None or len(tag_name) == 0:
|
|
684
|
+
raise ValueError(f"Input tag: {tag} returns no tag_name. Invalid input.")
|
|
685
|
+
if map_name is None:
|
|
686
|
+
map_name = ""
|
|
598
687
|
|
|
599
688
|
params = self.generate_read_query(
|
|
600
689
|
tagname=tag_name,
|
|
@@ -606,15 +695,29 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
606
695
|
metadata={},
|
|
607
696
|
)
|
|
608
697
|
|
|
609
|
-
|
|
698
|
+
# Consider use_post = use_post or not tag.isascii()
|
|
699
|
+
data = self.fetch(url, params=params, use_post=use_post)
|
|
610
700
|
|
|
611
701
|
if len(data) == 0: # Normally for timestamps in future
|
|
612
702
|
return pd.DataFrame(columns=[tag])
|
|
613
703
|
|
|
614
704
|
if "er" in data["data"][0]["samples"][0]:
|
|
705
|
+
if not use_post and data["data"][0]["samples"][0]["er"] == 5:
|
|
706
|
+
return self.read_tag(
|
|
707
|
+
tag=tag,
|
|
708
|
+
start=start,
|
|
709
|
+
end=end,
|
|
710
|
+
sample_time=sample_time,
|
|
711
|
+
read_type=read_type,
|
|
712
|
+
metadata=metadata,
|
|
713
|
+
get_status=get_status,
|
|
714
|
+
use_post=True,
|
|
715
|
+
)
|
|
716
|
+
|
|
615
717
|
logger.warning(
|
|
616
718
|
f"API error for {tag}: {data['data'][0]['samples'][0]['es']} params: {params}"
|
|
617
719
|
)
|
|
720
|
+
|
|
618
721
|
return pd.DataFrame(columns=[tag])
|
|
619
722
|
if get_status:
|
|
620
723
|
# The "l" field maps 1:1 to ODBC status field values 0, 1, 2, 4, 5, 6
|
|
@@ -640,12 +743,17 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
640
743
|
|
|
641
744
|
@staticmethod
|
|
642
745
|
def generate_sql_query(
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
746
|
+
query: str,
|
|
747
|
+
connection_string: Optional[str] = None,
|
|
748
|
+
datasource: Optional[str] = None,
|
|
646
749
|
max_rows: int = 100000,
|
|
647
|
-
):
|
|
648
|
-
if
|
|
750
|
+
) -> str:
|
|
751
|
+
if not query:
|
|
752
|
+
raise ValueError("Query must not be empty")
|
|
753
|
+
|
|
754
|
+
if not connection_string and not datasource:
|
|
755
|
+
raise ValueError("Either connection_string or datasource must be provided")
|
|
756
|
+
if connection_string:
|
|
649
757
|
connection_string = (
|
|
650
758
|
f'<SQL c="{connection_string}" m="{max_rows}" to="30" s="1">'
|
|
651
759
|
)
|
|
@@ -655,31 +763,20 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
655
763
|
'dso="CHARINT=N;CHARFLOAT=N;CHARTIME=N;CONVERTERRORS=N" '
|
|
656
764
|
f'm="{max_rows}" to="30" s="1">'
|
|
657
765
|
)
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
) # Replace new lines and tabs that are typical in formatted SQL queries with spaces.
|
|
661
|
-
|
|
766
|
+
# Replace new lines and tabs that are typical in formatted SQL queries with spaces.
|
|
767
|
+
query = query.replace("\t", " ").replace("\n", " ")
|
|
662
768
|
# Need a solution to LIKE comments. These have a % symbol in the query that does not seem to pass through the request
|
|
663
|
-
|
|
664
769
|
connection_string += f"<![CDATA[{query}]]></SQL>"
|
|
665
770
|
return connection_string
|
|
666
771
|
|
|
667
772
|
def query_sql(self, query: str, parse: bool = True) -> Union[str, pd.DataFrame]:
|
|
668
773
|
url = urljoin(self.base_url, "SQL")
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
)
|
|
676
|
-
else:
|
|
677
|
-
params = self.generate_sql_query(
|
|
678
|
-
connection_string=self._connection_string,
|
|
679
|
-
query=query,
|
|
680
|
-
max_rows=self._max_rows,
|
|
681
|
-
datasource=None,
|
|
682
|
-
)
|
|
774
|
+
params = self.generate_sql_query(
|
|
775
|
+
query=query,
|
|
776
|
+
connection_string=self._connection_string,
|
|
777
|
+
datasource=self.datasource,
|
|
778
|
+
max_rows=self.max_rows,
|
|
779
|
+
)
|
|
683
780
|
|
|
684
781
|
res = self.session.get(url, params=params)
|
|
685
782
|
res.raise_for_status()
|
|
@@ -706,15 +803,14 @@ class AspenHandlerWeb(BaseHandlerWeb):
|
|
|
706
803
|
class PIHandlerWeb(BaseHandlerWeb):
|
|
707
804
|
def __init__(
|
|
708
805
|
self,
|
|
709
|
-
url: Optional[str],
|
|
710
806
|
datasource: Optional[str],
|
|
807
|
+
url: Optional[str],
|
|
711
808
|
auth: Optional[Any],
|
|
712
809
|
verify_ssl: Optional[Union[bool, str]],
|
|
713
810
|
options: Dict[str, Union[int, float, str]],
|
|
714
811
|
cache: Optional[Union[SmartCache, BucketCache]],
|
|
715
812
|
):
|
|
716
|
-
|
|
717
|
-
if url is None:
|
|
813
|
+
if url is None or url == "":
|
|
718
814
|
url = get_url_pi()
|
|
719
815
|
if auth is None:
|
|
720
816
|
auth = get_auth_pi()
|
|
@@ -724,12 +820,12 @@ class PIHandlerWeb(BaseHandlerWeb):
|
|
|
724
820
|
auth=auth,
|
|
725
821
|
verify_ssl=verify_ssl,
|
|
726
822
|
)
|
|
727
|
-
self.
|
|
823
|
+
self.max_rows = int(options.get("max_rows", 10000))
|
|
728
824
|
self.web_id_cache = cache
|
|
729
825
|
|
|
730
826
|
@staticmethod
|
|
731
827
|
def _time_to_UTC_string(time: datetime) -> str:
|
|
732
|
-
return time.astimezone(
|
|
828
|
+
return time.astimezone(timezone.utc).strftime("%d-%b-%y %H:%M:%S")
|
|
733
829
|
|
|
734
830
|
@staticmethod
|
|
735
831
|
def escape(s: str) -> str:
|
|
@@ -774,7 +870,7 @@ class PIHandlerWeb(BaseHandlerWeb):
|
|
|
774
870
|
query = " AND ".join(q)
|
|
775
871
|
params = {"query": f"{query}"}
|
|
776
872
|
|
|
777
|
-
if datasource
|
|
873
|
+
if isinstance(datasource, str) and len(datasource) > 0:
|
|
778
874
|
params["dataserverwebid"] = (
|
|
779
875
|
f"{get_piwebapi_source_to_webid_dict(auth=auth)[datasource]}"
|
|
780
876
|
)
|
|
@@ -801,6 +897,9 @@ class PIHandlerWeb(BaseHandlerWeb):
|
|
|
801
897
|
]:
|
|
802
898
|
raise NotImplementedError
|
|
803
899
|
|
|
900
|
+
if not tag:
|
|
901
|
+
raise ValueError("Tag must be provided.")
|
|
902
|
+
|
|
804
903
|
web_id = tag
|
|
805
904
|
|
|
806
905
|
seconds = 0
|
|
@@ -813,7 +912,6 @@ class PIHandlerWeb(BaseHandlerWeb):
|
|
|
813
912
|
ReaderType.SNAPSHOT: "value",
|
|
814
913
|
ReaderType.SHAPEPRESERVING: "plot",
|
|
815
914
|
}.get(read_type, "summary")
|
|
816
|
-
|
|
817
915
|
url = f"streams/{web_id}/{get_action}"
|
|
818
916
|
params = {}
|
|
819
917
|
|
|
@@ -860,7 +958,7 @@ class PIHandlerWeb(BaseHandlerWeb):
|
|
|
860
958
|
params["selectedFields"] += ";Good;Questionable;Substituted"
|
|
861
959
|
|
|
862
960
|
if read_type == ReaderType.RAW:
|
|
863
|
-
params["maxCount"] = self.
|
|
961
|
+
params["maxCount"] = self.max_rows
|
|
864
962
|
|
|
865
963
|
return url, params
|
|
866
964
|
|
|
@@ -994,7 +1092,7 @@ class PIHandlerWeb(BaseHandlerWeb):
|
|
|
994
1092
|
if not web_id:
|
|
995
1093
|
return pd.DataFrame()
|
|
996
1094
|
|
|
997
|
-
|
|
1095
|
+
url, params = self.generate_read_query(
|
|
998
1096
|
tag=web_id,
|
|
999
1097
|
start=start,
|
|
1000
1098
|
end=end,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|