tagreader 6.0.3__tar.gz → 6.1.0__tar.gz

{tagreader-6.0.3 → tagreader-6.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: tagreader
-Version: 6.0.3
+Version: 6.1.0
 Summary: Tagreader is a Python package for reading trend data from the OSIsoft PI and Aspen Infoplus.21 IMS systems.
 License: MIT
 Keywords: Aspen InfoPlus.21,OSIsoft PI
@@ -15,27 +15,23 @@ Classifier: Natural Language :: English
 Classifier: Operating System :: MacOS
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
-Provides-Extra: notebooks
 Requires-Dist: certifi (>=2024.12.14,<2026.0.0)
-Requires-Dist: diskcache (>=5.6.1,<6.0.0)
-Requires-Dist: matplotlib (>=3.7.5,<4.0.0) ; extra == "notebooks"
-Requires-Dist: msal-bearer (>=1.3.0,<2.0.0)
-Requires-Dist: notebook (>=7.2.2,<8.0.0) ; extra == "notebooks"
+Requires-Dist: diskcache (>=5.6.1)
+Requires-Dist: msal-bearer (>=1.3.3)
 Requires-Dist: pandas (>=1)
-Requires-Dist: pycryptodome (>=3.20.0,<4.0.0)
-Requires-Dist: requests (>=2,<3)
-Requires-Dist: requests-kerberos (>=0,<1)
+Requires-Dist: platformdirs (>=4.3.7)
+Requires-Dist: pycryptodome (>=3.20.0)
+Requires-Dist: requests (>=2.32.5)
+Requires-Dist: requests-kerberos (>=0)
 Requires-Dist: requests-ntlm (>=1.1,<=2.0)
+Requires-Dist: urllib3 (>=2.5.0)
 Project-URL: Homepage, https://github.com/equinor/tagreader-python
 Project-URL: Repository, https://github.com/equinor/tagreader-python
 Description-Content-Type: text/markdown

{tagreader-6.0.3 → tagreader-6.1.0}/pyproject.toml

@@ -1,14 +1,15 @@
-[tool.poetry]
+[project]
 name = "tagreader"
-version = "6.0.3"
+version = "6.1.0"
 description = "Tagreader is a Python package for reading trend data from the OSIsoft PI and Aspen Infoplus.21 IMS systems."
-authors = ["Einar S. Idsø <eiids@equinor.com>", "Morten Dæhli Aslesen <masl@equinor.com"]
+authors = [
+    { name = "Einar S. Idsø", email = "eiids@equinor.com" },
+    { name = "Morten Dæhli Aslesen", email = "masl@equinor.com" }
+]
 license = "MIT"
 readme = "README.md"
-packages = [{include = "tagreader"}]
 keywords=["Aspen InfoPlus.21", "OSIsoft PI"]
-homepage = "https://github.com/equinor/tagreader-python"
-repository = "https://github.com/equinor/tagreader-python"
+requires-python = ">=3.9.2,<4.0.0"
 classifiers=[
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
@@ -18,35 +19,39 @@ classifiers=[
     "Operating System :: MacOS",
     "Operating System :: Microsoft :: Windows",
     "Operating System :: POSIX :: Linux",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Topic :: Utilities",
 ]
+dependencies = [
+    "pandas>=1",
+    "certifi>=2024.12.14,<2026.0.0",
+    "requests>=2.32.5",
+    "requests-kerberos>=0",
+    "msal-bearer>=1.3.3",
+    "diskcache>=5.6.1",
+    "pycryptodome>=3.20.0",
+    "requests-ntlm>=1.1,<=2.0",
+    "platformdirs>=4.3.7",
+    "urllib3>=2.5.0" # To avoid security issue https://www.cve.org/CVERecord?id=CVE-2025-50182
+]
 
-[tool.poetry.dependencies]
-python = "^3.9.2"
-pandas = ">=1"
-certifi = ">=2024.12.14,<2026.0.0"
-requests = "^2"
-requests-kerberos = "^0"
-msal-bearer = "^1.3.0"
-notebook = { version = "^7.2.2", optional = true }
-matplotlib = { version = "^3.7.5", optional = true }
-diskcache = "^5.6.1"
-pycryptodome = "^3.20.0"
-requests-ntlm = ">=1.1,<=2.0"
+[tool.poetry.group.test.dependencies]
+pytest = ">=7,<9"
+# pytest-cov = ">=6.0.0"
 
 [tool.poetry.group.dev.dependencies]
-pre-commit = "^3"
-pytest = ">=7,<9"
+# "black>=24.1.1",
+pre-commit = ">=3.0"
 
-[tool.poetry.extras]
-notebooks = ["notebook", "matplotlib"]
+[project.urls]
+homepage = "https://github.com/equinor/tagreader-python"
+repository = "https://github.com/equinor/tagreader-python"
 
 [build-system]
-requires = ["poetry-core"]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
 build-backend = "poetry.core.masonry.api"

{tagreader-6.0.3 → tagreader-6.1.0}/tagreader/__init__.py

@@ -4,11 +4,18 @@ from tagreader.utils import (
     ReaderType,
     add_equinor_root_certificate,
     is_equinor,
-    is_mac,
-    is_windows,
 )
 
 if is_equinor():
     add_equinor_root_certificate()
 
 from tagreader.__version__ import version as __version__
+
+__all__ = [
+    "IMSClient",
+    "list_sources",
+    "IMSType",
+    "ReaderType",
+    "add_equinor_root_certificate",
+    "__version__",
+]

{tagreader-6.0.3 → tagreader-6.1.0}/tagreader/clients.py

@@ -1,4 +1,3 @@
-import os
 from datetime import datetime, timedelta, timezone, tzinfo
 from itertools import groupby
 from operator import itemgetter
@@ -16,9 +15,6 @@ from tagreader.utils import (
     ReaderType,
     convert_to_pydatetime,
     ensure_datetime_with_tz,
-    find_registry_key,
-    find_registry_key_from_name,
-    is_windows,
 )
 from tagreader.web_handlers import (
     AspenHandlerWeb,
@@ -36,7 +32,7 @@ def list_sources(
     imstype: Union[IMSType, str],
     url: Optional[str] = None,
     auth: Optional[Any] = None,
-    verifySSL: bool = True,
+    verify_ssl: Optional[Union[bool, str]] = True,
 ) -> List[str]:
     if isinstance(imstype, str):
         try:
@@ -51,11 +47,11 @@ def list_sources(
     if imstype == IMSType.PIWEBAPI:
         if auth is None:
             auth = get_auth_pi()
-        return list_piwebapi_sources(url=url, auth=auth, verify_ssl=verifySSL)
+        return list_piwebapi_sources(url=url, auth=auth, verify_ssl=verify_ssl)
     elif imstype == IMSType.ASPENONE:
         if auth is None:
             auth = get_auth_aspen()
-        return list_aspenone_sources(url=url, auth=auth, verify_ssl=verifySSL)
+        return list_aspenone_sources(url=url, auth=auth, verify_ssl=verify_ssl)
     elif imstype in [IMSType.PI, IMSType.ASPEN, IMSType.IP21]:
         raise ValueError(
             f"ODBC clients are no longer supported. Given ims client type: {imstype}."
@@ -119,96 +115,19 @@ def get_next_timeslice(
     return start, calc_end
 
 
-def get_server_address_aspen(datasource: str) -> Optional[Tuple[str, int]]:
-    """Data sources are listed under
-    HKEY_CURRENT_USER\\Software\\AspenTech\\ADSA\\Caches\\AspenADSA\\username.
-    For each data source there are multiple keys with Host entries. We start by
-    identifying the correct key to use by locating the UUID for Aspen SQLplus
-    services located under Aspen SQLplus service component. Then we find the
-    host and port based on the path above and the UUID.
-    """
-
-    # todo: is obsolete after removing ODBC
-
-    if not is_windows():
-        return None
-    import winreg
-
-    regkey_clsid = winreg.OpenKey(
-        winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Classes\Wow6432Node\CLSID"
-    )
-    regkey, _ = find_registry_key_from_name(
-        regkey_clsid, "Aspen SQLplus service component"
-    )
-    regkey_implemented_categories = winreg.OpenKeyEx(regkey, "Implemented Categories")
-
-    _, aspen_uuid = find_registry_key_from_name(
-        regkey_implemented_categories, "Aspen SQLplus services"
-    )
-
-    reg_adsa = winreg.OpenKey(
-        winreg.HKEY_CURRENT_USER,
-        r"Software\AspenTech\ADSA\Caches\AspenADSA\\" + os.getlogin(),
-    )
-
-    try:
-        reg_site_key = winreg.OpenKey(reg_adsa, datasource + "\\" + aspen_uuid)
-        host = winreg.QueryValueEx(reg_site_key, "Host")[0]
-        port = int(winreg.QueryValueEx(reg_site_key, "Port")[0])
-        return host, port
-    except FileNotFoundError:
-        return None
-
-
-def get_server_address_pi(datasource: str) -> Optional[Tuple[str, int]]:
-    """
-    PI data sources are listed under
-    HKEY_LOCAL_MACHINE\\Software\\Wow6432Node\\PISystem\\PI-SDK\\x.x\\ServerHandles or
-    \\Software\\PISystem\\PI-SDK\\x.x\\ServerHandles.
-
-    :param datasource: Name of data source
-    :return: host, port
-    :type: tuple(string, int)
-    """
-    # todo: is obsolete after removing ODBC
-
-    if not is_windows():
-        return None
-    import winreg
-
-    try:
-        reg_key = winreg.OpenKey(
-            winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Wow6432Node\PISystem\PI-SDK"
-        )
-        reg_key_handles = find_registry_key(reg_key, "ServerHandles")
-        reg_site_key = find_registry_key(reg_key_handles, datasource)
-        if reg_site_key is None:
-            reg_key = winreg.OpenKey(
-                winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\PISystem\PI-SDK"
-            )
-            reg_key_handles = find_registry_key(reg_key, "ServerHandles")
-            reg_site_key = find_registry_key(reg_key_handles, datasource)
-        if reg_site_key is not None:
-            host = winreg.QueryValueEx(reg_site_key, "path")[0]
-            port = int(winreg.QueryValueEx(reg_site_key, "port")[0])
-            return host, port
-    except FileNotFoundError:
-        return None
-
-
 def get_handler(
     imstype: Optional[IMSType],
     datasource: str,
     url: Optional[str],
     options: Dict[str, Union[int, float, str]],
-    verifySSL: Optional[bool],
+    verify_ssl: Optional[Union[bool, str]],
     auth: Optional[Any],
     cache: Optional[Union[SmartCache, BucketCache]] = None,
 ):
     if imstype is None:
         try:
             if datasource in list_aspenone_sources(
-                url=None, auth=None, verify_ssl=verifySSL
+                url=None, auth=None, verify_ssl=verify_ssl
             ):
                 imstype = IMSType.ASPENONE
         except HTTPError as e:
@@ -216,7 +135,7 @@ def get_handler(
     if imstype is None:
         try:
             if datasource in list_piwebapi_sources(
-                url=None, auth=None, verify_ssl=verifySSL
+                url=None, auth=None, verify_ssl=verify_ssl
             ):
                 imstype = IMSType.PIWEBAPI
         except HTTPError as e:
@@ -227,7 +146,7 @@ def get_handler(
             url=url,
             datasource=datasource,
             options=options,
-            verify_ssl=verifySSL,
+            verify_ssl=verify_ssl,
             auth=auth,
             cache=cache,
         )
@@ -237,7 +156,7 @@ def get_handler(
             datasource=datasource,
             url=url,
             options=options,
-            verify_ssl=verifySSL,
+            verify_ssl=verify_ssl,
             auth=auth,
         )
     elif imstype in [IMSType.PI, IMSType.ASPEN, IMSType.IP21]:
@@ -259,7 +178,7 @@ class IMSClient:
         tz: Union[tzinfo, str] = pytz.timezone("Europe/Oslo"),
         url: Optional[str] = None,
         handler_options: Dict[str, Union[int, float, str]] = {},  # noqa:
-        verifySSL: bool = True,
+        verify_ssl: Optional[Union[bool, str]] = True,
         auth: Optional[Any] = None,
         cache: Optional[Union[SmartCache, BucketCache]] = None,
     ):
@@ -290,7 +209,7 @@ class IMSClient:
             datasource=datasource,
             url=url,
             options=handler_options,
-            verifySSL=verifySSL,
+            verify_ssl=verify_ssl,
             auth=auth,
             cache=self.cache,
         )
@@ -493,7 +412,7 @@ class IMSClient:
     ):
         start = start_time
         end = stop_time
-        logger.warn(
+        logger.warning(
             (
                 "This function has been renamed to read() and is deprecated. "
                 "Please call 'read()' instead"

tagreader-6.1.0/tagreader/utils.py

@@ -0,0 +1,306 @@
+import enum
+import hashlib
+import platform
+import socket
+import ssl
+from datetime import datetime, tzinfo
+from enum import Enum
+from pathlib import Path
+from typing import Union
+
+import certifi
+import pandas as pd
+import pytz
+import requests
+from platformdirs import user_data_dir
+
+from tagreader.logger import logger
+
+
+def is_windows() -> bool:
+    return platform.system() == "Windows"
+
+
+def is_mac() -> bool:
+    return platform.system() == "Darwin"
+
+
+def is_linux() -> bool:
+    return platform.system() == "Linux"
+
+
+if is_windows():
+    import winreg
+
+if is_mac():
+    import subprocess
+
+
+def convert_to_pydatetime(date_stamp: Union[datetime, str, pd.Timestamp]) -> datetime:
+    if isinstance(date_stamp, datetime):
+        return date_stamp
+    elif isinstance(date_stamp, pd.Timestamp):
+        return date_stamp.to_pydatetime()
+    else:
+        try:
+            return pd.to_datetime(str(date_stamp), format="ISO8601").to_pydatetime()
+        except ValueError:
+            return pd.to_datetime(str(date_stamp), dayfirst=True).to_pydatetime()
+
+
+def ensure_datetime_with_tz(
+    date_stamp: Union[datetime, str, pd.Timestamp],
+    tz: tzinfo = pytz.timezone("Europe/Oslo"),
+) -> datetime:
+    date_stamp = convert_to_pydatetime(date_stamp)
+
+    if not date_stamp.tzinfo:
+        date_stamp = tz.localize(date_stamp)
+
+    return date_stamp
+
+
+def urljoin(*args) -> str:
+    """
+    Joins components of URL. Ensures slashes are inserted or removed where
+    needed, and does not strip trailing slash of last element.
+
+    Returns:
+        str -- Generated URL
+    """
+    trailing_slash = "/" if args[-1].endswith("/") else ""
+    return "/".join(map(lambda x: str(x).strip("/"), args)) + trailing_slash
+
+
+class ReaderType(enum.IntEnum):
+    """Enumerates available types of data to read.
+
+    For members with more than one name per value, the first member (the
+    original) needs to be untouched since it may be used as back-reference
+    (specifically for cache hierarchies).
+    """
+
+    RAW = SAMPLED = ACTUAL = enum.auto()  # Raw sampled data
+    SHAPEPRESERVING = BESTFIT = enum.auto()  # Minimum data points for preserving shape
+    INT = INTERPOLATE = INTERPOLATED = enum.auto()  # Interpolated data
+    MIN = MINIMUM = enum.auto()  # Min value
+    MAX = MAXIMUM = enum.auto()  # Max value
+    AVG = AVERAGE = AVERAGED = enum.auto()  # Averaged data
+    VAR = VARIANCE = enum.auto()  # Variance of data
+    STD = STDDEV = enum.auto()  # Standard deviation of data
+    RNG = RANGE = enum.auto()  # Range of data
+    COUNT = enum.auto()  # Number of data points
+    GOOD = enum.auto()  # Number of good data points
+    BAD = NOTGOOD = enum.auto()  # Number of not good data points
+    TOTAL = enum.auto()  # Number of total data
+    SUM = enum.auto()  # Sum of data
+    SNAPSHOT = FINAL = LAST = enum.auto()  # Last sampled value
+
+
+def add_equinor_root_certificate() -> bool:
+    """
+    This is a utility function for Equinor employees on Equinor managed machines.
+
+    The function searches for the Equinor Root certificate in the
+    cert store and imports it to the cacert bundle. Does nothing if not
+    running on Equinor host.
+
+    NB! This needs to be repeated after updating the cacert module.
+
+    Returns:
+        bool: True if function completes successfully
+    """
+    certificate = find_local_equinor_root_certificate()
+
+    # If certificate is not found locally, we download it from the Equinor server
+    if certificate == "":
+        logger.debug(
+            "Unable to locate Equinor Root CA certificate on this host. Downloading from Equinor server."
+        )
+        response = requests.get("http://pki.equinor.com/aia/ecpr.crt")
+
+        if response.status_code != 200:
+            logger.error(
+                "Unable to find Equinor Root CA certificate locally and on Equinor server."
+            )
+            return False
+
+        certificate = response.text.replace("\r", "")
+
+        # Write result to user data so we can read the cert from there next time
+        filepath = Path(user_data_dir("tagreader")) / "equinor_root_ca.crt"
+        try:
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            filepath.write_text(certificate)
+            logger.debug("Equinor Root CA certificate written to cache")
+        except Exception as e:
+            logger.debug(f"Failed to write Equinor Root CA certificate to cache: {e}")
+
+    if certificate in certifi.contents():
+        logger.debug("Equinor Root Certificate already exists in certifi store")
+        return True
+
+    ca_file = certifi.where()
+    with open(ca_file, "ab") as f:
+        f.write(bytes(certificate, "ascii"))
+    logger.debug("Equinor Root Certificate added to certifi store")
+    return True
+
+
+def find_local_equinor_root_certificate() -> str:
+    equinor_root_pem_hash = "5A206332CE73CED1D44C8A99C4C43B7CEE03DF5F"
+    ca_search = "Equinor Root CA"
+
+    if is_windows():
+        logger.debug("Checking for Equinor Root CA in Windows certificate store")
+        for cert in ssl.enum_certificates("CA"):  # type: ignore
+            found_cert = cert[0]
+            # deepcode ignore InsecureHash: <Only hashes to compare with known hash>
+            if hashlib.sha1(found_cert).hexdigest().upper() == equinor_root_pem_hash:
+                return ssl.DER_cert_to_PEM_cert(found_cert)
+
+    elif is_mac():
+        logger.debug("Checking for Equinor Root CA in MacOS certificate store")
+        macos_ca_certs = subprocess.run(
+            ["security", "find-certificate", "-a", "-c", ca_search, "-Z"],
+            stdout=subprocess.PIPE,
+        ).stdout
+
+        if equinor_root_pem_hash in str(macos_ca_certs).upper():
+            c = get_macos_equinor_certificates()
+            for cert in c:
+                # deepcode ignore InsecureHash: <Only hashes to compare with known hash>
+                if hashlib.sha1(cert).hexdigest().upper() == equinor_root_pem_hash:
+                    return ssl.DER_cert_to_PEM_cert(cert)
+
+    # If the certificate is not found in the local cert store, look in the tagreader cache
+    filepath = Path(user_data_dir("tagreader")) / "equinor_root_ca.crt"
+
+    try:
+        if filepath.exists():
+            return filepath.read_text()
+    except Exception as e:
+        logger.debug(f"Failed to read Equinor Root CA certificate from cache: {e}")
+
+    return ""
+
+
+def get_macos_equinor_certificates():
+    import ssl
+    import tempfile
+
+    if not is_mac():
+        raise OSError("Function only works on MacOS")
+
+    ca_search = "Equinor Root CA"
+
+    ctx = ssl.create_default_context()
+    macos_ca_certs = subprocess.run(
+        ["security", "find-certificate", "-a", "-c", ca_search, "-p"],
+        stdout=subprocess.PIPE,
+    ).stdout
+    with tempfile.NamedTemporaryFile("w+b", delete=False) as tmp_file:
+        tmp_file.write(macos_ca_certs)
+
+    ctx.load_verify_locations(tmp_file.name)
+
+    return ctx.get_ca_certs(binary_form=True)
+
+
+def is_equinor() -> bool:
+    """Determines whether code is running on an Equinor host
+
+    If Windows host:
+    Finds host's domain in Windows Registry at
+    HKLM\\SYSTEM\\ControlSet001\\Services\\Tcpip\\Parameters\\Domain
+    or check if hostname starts with eqdev or eqpc
+    If mac os host:
+    Finds statoil.net as AD hostname in certificates or
+    Finds statoil.net, client.statoil.net or equinor.com in dns search domains
+    If Linux host:
+    Checks whether statoil.no is dns search domains
+
+    Returns:
+        bool: True if Equinor
+    """
+
+    hostname = socket.gethostname()
+
+    if is_windows():
+        if hostname.lower().startswith("eqdev") or hostname.lower().startswith("eqpc"):
+            return True
+        with winreg.OpenKey(  # type: ignore
+            winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\ControlSet001\Services\Tcpip\Parameters"  # type: ignore
+        ) as key:
+            domain = winreg.QueryValueEx(key, "Domain")  # type: ignore
+            if "statoil" in domain[0] or "equinor" in domain[0]:
+                return True
+    elif is_linux() or is_mac():
+        with open("/etc/resolv.conf", "r") as f:
+            if any(
+                domain in f.read()
+                for domain in ["client.statoil.net", "statoil.net", "equinor.com"]
+            ):
+                return True
+
+        if is_mac():
+
+            def get_mac_dns_search_list():
+                """
+                Retrieves the DNS search list configured on macOS.
+                """
+                try:
+                    # Execute the scutil command to get DNS configuration
+                    result = subprocess.run(
+                        ["scutil", "--dns"], capture_output=True, text=True, check=True
+                    )
+                    output = result.stdout
+
+                    search_list = []
+                    # Parse the output to find the search domains
+                    for line in output.splitlines():
+                        if "search domain" in line:
+                            # Extract the domain from the line
+                            parts = line.split(":")
+                            if len(parts) > 1:
+                                domain = parts[1].strip()
+                                # Remove any leading/trailing quotes if present
+                                domain = domain.strip('"')
+                                search_list.append(domain)
+                    return search_list
+                except subprocess.CalledProcessError as e:
+                    print(f"Error executing scutil: {e}")
+                    return []
+                except Exception as e:
+                    print(f"An unexpected error occurred: {e}")
+                    return []
+
+            if any(
+                domain in get_mac_dns_search_list()
+                for domain in ["client.statoil.net", "statoil.net", "equinor.com"]
+            ):
+                return True
+
+            s = subprocess.run(
+                ["security", "find-certificate", "-a", "-c" "client.statoil.net"],
+                stdout=subprocess.PIPE,
+            ).stdout
+
+            # deepcode ignore IdenticalBranches: Not an error. First test is just more precise.
+            if hostname + ".client.statoil.net" in str(s):
+                return True
+            elif "client.statoil.net" in hostname and hostname in str(s):
+                return True
+    else:
+        raise OSError(
+            f"Unsupported system: {platform.system()}. Please report this as an issue."
+        )
+    return False
+
+
+class IMSType(str, Enum):
+    PIWEBAPI = "piwebapi"
+    ASPENONE = "aspenone"
+    PI = "pi"
+    ASPEN = "aspen"
+    IP21 = "ip21"

{tagreader-6.0.3 → tagreader-6.1.0}/tagreader/web_handlers.py

@@ -19,7 +19,7 @@ from urllib3.exceptions import InsecureRequestWarning
 
 from tagreader.cache import BucketCache, SmartCache
 from tagreader.logger import logger
-from tagreader.utils import IMSType, ReaderType, is_mac, is_windows, urljoin
+from tagreader.utils import ReaderType, is_mac, is_windows, urljoin
 
 
 class MD4:
@@ -42,12 +42,7 @@ def patched_hashlib_new(name, data=b"", usedforsecurity=True):
     if name.lower() == "md4":
         return MD4(data)
     else:
-        # Try / Catch easier than detecting python version
-        try:
-            return hashlib_new_method(name, data=data, usedforsecurity=usedforsecurity)
-        except TypeError:
-            # Required for python 3.8
-            return hashlib_new_method(name, data=data)
+        return hashlib_new_method(name, data=data, usedforsecurity=usedforsecurity)
 
 
 # Monkey-patch md4 in hashlib.new due to missing support for md4 in later releases of Python:
@@ -95,7 +90,7 @@ def get_url_aspen(use_internal: bool = True) -> str:
 def list_aspenone_sources(
     url: Optional[str] = None,
     auth: Optional[Any] = None,
-    verify_ssl: Optional[bool] = True,
+    verify_ssl: Optional[Union[bool, str]] = True,
 ) -> List[str]:
     if url is None:
         url = get_url_aspen()
@@ -124,7 +119,7 @@ def list_aspenone_sources(
 def list_piwebapi_sources(
     url: Optional[str] = None,
     auth: Optional[Any] = None,
-    verify_ssl: Optional[bool] = True,
+    verify_ssl: Optional[Union[bool, str]] = True,
 ) -> List[str]:
     if url is None:
         url = get_url_pi()
@@ -152,7 +147,7 @@ def list_piwebapi_sources(
 def get_piwebapi_source_to_webid_dict(
     url: Optional[str] = None,
     auth: Optional[Any] = None,
-    verify_ssl: Optional[bool] = True,
+    verify_ssl: Optional[Union[bool, str]] = True,
 ) -> List[str]:
     if url is None:
         url = get_url_pi()
@@ -182,7 +177,7 @@ class BaseHandlerWeb(ABC):
         datasource: Optional[str],
         url: Optional[str],
         auth: Optional[Any],
-        verify_ssl: Optional[bool],
+        verify_ssl: Optional[Union[bool, str]],
     ):
         self.datasource = datasource
         self.base_url = url
@@ -213,6 +208,10 @@ class BaseHandlerWeb(ABC):
             logger.warning(f"No data found for {url} {params}")
             return {}
 
+        if res.text.startswith("XML Error"):
+            logger.warning(f"Invalid data returned for {url} {params}")
+            return {}
+
         try:
             return res.json()
         except JSONDecodeError:
@@ -241,7 +240,7 @@ class AspenHandlerWeb(BaseHandlerWeb):
         datasource: Optional[str],
         url: Optional[str] = None,
         auth: Optional[Any] = None,
-        verify_ssl: Optional[bool] = True,
+        verify_ssl: Optional[Union[bool, str]] = True,
         options: Dict[str, Any] = dict(),
     ):
         if url is None:
@@ -258,11 +257,6 @@ class AspenHandlerWeb(BaseHandlerWeb):
         self._max_rows = options.get("max_rows", 100000)
         self._connection_string = ""  # Used for raw SQL queries
 
-    @staticmethod
-    def generate_connection_string(host, *_, **__):
-        # todo: is obsolete after removing ODBC
-        raise NotImplementedError
-
     @staticmethod
     def generate_search_query(
         tag: Optional[str],
@@ -670,27 +664,6 @@ class AspenHandlerWeb(BaseHandlerWeb):
         connection_string += f"<![CDATA[{query}]]></SQL>"
         return connection_string
 
-    def initialize_connection_string(
-        # todo: is obsolete after removing ODBC
-        self,
-        host: Optional[str] = None,
-        port: int = 10014,
-        connection_string: Optional[str] = None,
-    ):
-        # todo: is obsolete after removing ODBC
-        if connection_string:
-            self._connection_string = connection_string
-        else:
-            if host is None:
-                from tagreader.clients import get_server_address_aspen
-
-                host = get_server_address_aspen(self.datasource)
-            self._connection_string = (
-                f"DRIVER=AspenTech SQLPlus;HOST={host};"
-                f"PORT={port};CHARINT=N;CHARFLOAT=N;"
-                "CHARTIME=N;CONVERTERRORS=N"
-            )
-
     def query_sql(self, query: str, parse: bool = True) -> Union[str, pd.DataFrame]:
         url = urljoin(self.base_url, "SQL")
         if self._connection_string is None:
@@ -736,7 +709,7 @@ class PIHandlerWeb(BaseHandlerWeb):
         url: Optional[str],
         datasource: Optional[str],
         auth: Optional[Any],
-        verify_ssl: bool,
+        verify_ssl: Optional[Union[bool, str]],
         options: Dict[str, Union[int, float, str]],
         cache: Optional[Union[SmartCache, BucketCache]],
     ):
@@ -758,11 +731,6 @@ class PIHandlerWeb(BaseHandlerWeb):
     def _time_to_UTC_string(time: datetime) -> str:
         return time.astimezone(pytz.UTC).strftime("%d-%b-%y %H:%M:%S")
 
-    @staticmethod
-    def generate_connection_string(host, *_, **__):
-        # todo: is obsolete after removing ODBC
-        raise NotImplementedError
-
     @staticmethod
     def escape(s: str) -> str:
         # https://techsupport.osisoft.com/Documentation/PI-Web-API/help/topics/search-queries.html

tagreader-6.0.3/tagreader/utils.py

@@ -1,270 +0,0 @@
-import enum
-import logging
-import platform
-import warnings
-from datetime import datetime, tzinfo
-from enum import Enum
-from typing import Union
-
-import pandas as pd
-import pytz
-
-from tagreader.logger import logger
-
-
-def is_windows() -> bool:
-    return platform.system() == "Windows"
-
-
-def is_mac() -> bool:
-    return platform.system() == "Darwin"
-
-
-def is_linux() -> bool:
-    return platform.system() == "Linux"
-
-
-if is_windows():
-    import winreg
-
-
-if is_mac():
-    import socket
-    import subprocess
-
-
-def find_registry_key(base_key, search_key_name: str):
-    search_key_name = search_key_name.lower()
-    if base_key is not None:
-        num_keys, _, _ = winreg.QueryInfoKey(base_key)
-        for i in range(0, num_keys):
-            key_name = winreg.EnumKey(base_key, i)
-            if key_name.lower() == search_key_name:
-                return winreg.OpenKey(base_key, key_name)
-            else:
-                key = find_registry_key(
-                    winreg.OpenKey(base_key, key_name), search_key_name
-                )
-            if key is not None:
-                return key
-    return None
-
-
-def find_registry_key_from_name(base_key, search_key_name: str):
-    search_key_name = search_key_name.lower()
-    num_keys, _, _ = winreg.QueryInfoKey(base_key)
-    key = key_string = None
-    for i in range(0, num_keys):
-        try:
-            key_string = winreg.EnumKey(base_key, i)
-            key = winreg.OpenKey(base_key, key_string)
-            _, num_vals, _ = winreg.QueryInfoKey(key)
-            if num_vals > 0:
-                (_, key_name, _) = winreg.EnumValue(key, 0)
-                if str(key_name).lower() == search_key_name:
-                    break
-        except Exception as err:
-            logging.error("{}: {}".format(i, err))
-    return key, key_string
-
-
-def convert_to_pydatetime(date_stamp: Union[datetime, str, pd.Timestamp]) -> datetime:
-    if isinstance(date_stamp, datetime):
-        return date_stamp
-    elif isinstance(date_stamp, pd.Timestamp):
-        return date_stamp.to_pydatetime()
-    else:
-        try:
-            return pd.to_datetime(str(date_stamp), format="ISO8601").to_pydatetime()
-        except ValueError:
-            return pd.to_datetime(str(date_stamp), dayfirst=True).to_pydatetime()
-
-
-def ensure_datetime_with_tz(
-    date_stamp: Union[datetime, str, pd.Timestamp],
-    tz: tzinfo = pytz.timezone("Europe/Oslo"),
-) -> datetime:
-    date_stamp = convert_to_pydatetime(date_stamp)
-
-    if not date_stamp.tzinfo:
-        date_stamp = tz.localize(date_stamp)
-
-    return date_stamp
-
-
-def urljoin(*args) -> str:
-    """
-    Joins components of URL. Ensures slashes are inserted or removed where
-    needed, and does not strip trailing slash of last element.
-
-    Returns:
-        str -- Generated URL
-    """
-    trailing_slash = "/" if args[-1].endswith("/") else ""
-    return "/".join(map(lambda x: str(x).strip("/"), args)) + trailing_slash
-
-
-class ReaderType(enum.IntEnum):
-    """Enumerates available types of data to read.
-
-    For members with more than one name per value, the first member (the
-    original) needs to be untouched since it may be used as back-reference
-    (specifically for cache hierarchies).
-    """
-
-    RAW = SAMPLED = ACTUAL = enum.auto()  # Raw sampled data
-    SHAPEPRESERVING = BESTFIT = enum.auto()  # Minimum data points for preserving shape
-    INT = INTERPOLATE = INTERPOLATED = enum.auto()  # Interpolated data
-    MIN = MINIMUM = enum.auto()  # Min value
-    MAX = MAXIMUM = enum.auto()  # Max value
-    AVG = AVERAGE = AVERAGED = enum.auto()  # Averaged data
-    VAR = VARIANCE = enum.auto()  # Variance of data
-    STD = STDDEV = enum.auto()  # Standard deviation of data
-    RNG = RANGE = enum.auto()  # Range of data
-    COUNT = enum.auto()  # Number of data points
-    GOOD = enum.auto()  # Number of good data points
-    BAD = NOTGOOD = enum.auto()  # Number of not good data points
-    TOTAL = enum.auto()  # Number of total data
-    SUM = enum.auto()  # Sum of data
-    SNAPSHOT = FINAL = LAST = enum.auto()  # Last sampled value
-
-
-def add_equinor_root_certificate() -> bool:
-    """
-    This is a utility function for Equinor employees on Equinor managed machines.
-
-    The function searches for the Equinor Root certificate in the
-    cert store and imports it to the cacert bundle. Does nothing if not
-    running on Equinor host.
-
-    NB! This needs to be repeated after updating the cacert module.
-    NB! Will not download certificates.
-
-    Returns:
-        bool: True if function completes successfully
-    """
-    import hashlib
-    import ssl
-
-    import certifi
-
-    EQUINOR_root_PEM_HASH = "5A206332CE73CED1D44C8A99C4C43B7CEE03DF5F"
-    ca_search = "Equinor Root CA"
-
-    found = False
-    der = None
-
-    if is_linux():
-        return True
-    elif is_windows():
-        logger.debug("Scanning CA certificate in Windows cert store", end="")
-        for cert in ssl.enum_certificates("CA"):
-            der = cert[0]
-            # deepcode ignore InsecureHash: <Only hashes to compare with known hash>
-            if hashlib.sha1(der).hexdigest().upper() == EQUINOR_root_PEM_HASH:
-                found = True
-                logger.debug("CA certificate found!")
-                break
-    elif is_mac():
-        macos_ca_certs = subprocess.run(
-            ["security", "find-certificate", "-a", "-c", ca_search, "-Z"],
-            stdout=subprocess.PIPE,
-        ).stdout
-
-        if EQUINOR_root_PEM_HASH in str(macos_ca_certs).upper():
-            c = get_macos_equinor_certificates()
-            for cert in c:
-                # deepcode ignore InsecureHash: <Only hashes to compare with known hash>
-                if hashlib.sha1(cert).hexdigest().upper() == EQUINOR_root_PEM_HASH:
-                    der = cert
-                    found = True
-                    break
-
-    if found and der:
-        pem = ssl.DER_cert_to_PEM_cert(der)
-        if pem in certifi.contents():
-            logger.debug(
-                "CA Certificate already exists in certifi store. Nothing to do."
-            )
-        else:
-            logger.debug("Writing certificate to certifi store.")
-            ca_file = certifi.where()
-            with open(ca_file, "ab") as f:
-                f.write(bytes(pem, "ascii"))
-            logger.debug("Completed")
-    else:
-        warnings.warn("Unable to locate root certificate on this host.")
-
-    return found
-
-
-def get_macos_equinor_certificates():
-    import ssl
-    import tempfile
-
-    ca_search = "Equinor Root CA"
-
-    ctx = ssl.create_default_context()
-    macos_ca_certs = subprocess.run(
-        ["security", "find-certificate", "-a", "-c", ca_search, "-p"],
-        stdout=subprocess.PIPE,
-    ).stdout
-    with tempfile.NamedTemporaryFile("w+b", delete=False) as tmp_file:
-        tmp_file.write(macos_ca_certs)
-
-    ctx.load_verify_locations(tmp_file.name)
-
-    return ctx.get_ca_certs(binary_form=True)
-
-
-def is_equinor() -> bool:
-    """Determines whether code is running on an Equinor host
-
-    If Windows host:
-    Finds host's domain in Windows Registry at
-    HKLM\\SYSTEM\\ControlSet001\\Services\\Tcpip\\Parameters\\Domain
-    If mac os host:
-    Finds statoil.net as AD hostname in certificates
-    If Linux host:
-    Checks whether statoil.no is search domain
-
-    Returns:
-        bool: True if Equinor
-    """
-    if is_windows():
-        with winreg.OpenKey(
-            winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\ControlSet001\Services\Tcpip\Parameters"
-        ) as key:
-            domain = winreg.QueryValueEx(key, "Domain")
-        if "statoil" in domain[0]:
-            return True
-    elif is_mac():
-        s = subprocess.run(
-            ["security", "find-certificate", "-a", "-c" "client.statoil.net"],
-            stdout=subprocess.PIPE,
-        ).stdout
-
-        host = socket.gethostname()
-
-        # deepcode ignore IdenticalBranches: Not an error. First test is just more precise.
-        if host + ".client.statoil.net" in str(s):
-            return True
-        elif "client.statoil.net" in host and host in str(s):
-            return True
-    elif is_linux():
-        with open("/etc/resolv.conf", "r") as f:
-            if "statoil.no" in f.read():
-                return True
-    else:
-        raise OSError(
-            f"Unsupported system: {platform.system()}. Please report this as an issue."
-        )
-    return False
-
-
-class IMSType(str, Enum):
-    PIWEBAPI = "piwebapi"
-    ASPENONE = "aspenone"
-    PI = "pi"
-    ASPEN = "aspen"
-    IP21 = "ip21"