iplotDataAccess 1.3.1.post1__tar.gz

iplotdataaccess-1.3.1.post1/MANIFEST.in

@@ -0,0 +1 @@
+include iplotDataAccess/tests/csv/ITER/MCTB-TEST/data_111.csv

iplotdataaccess-1.3.1.post1/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: iplotDataAccess
+Version: 1.3.1.post1
+Summary: Data access for applications using IDSs or CBS
+Author-email: ITER Organization <jscabanilla@minsait.com>
+Maintainer-email: Jhon Steeven Cabanilla Alvarado <jscabanilla@minsait.com>, Pablo Martin Villares <pmartin@minsait.com>, Mario Gomez Ballesteros <mgomezb@minsait.com>
+License: MIT
+Project-URL: Homepage, https://github.com/iplot-viz/iplotdataaccess
+Project-URL: Repository, https://github.com/iplot-viz/iplotdataaccess.git
+Project-URL: Issues, https://github.com/iplot-viz/iplotdataaccess/issues
+Keywords: IDS,CBS,Interface Data Structures,Control Breakdown Structure
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: cachetools>=4.2.0
+Requires-Dist: iplotLogging>=1.2.0
+Requires-Dist: requests>=2.25.1
+Requires-Dist: sseclient-py>=1.7
+Requires-Dist: pyarrow>=16.1.0
+Requires-Dist: h5py>=3.11.0
+Requires-Dist: pandas>=1.2.0
+Provides-Extra: imaspy
+Requires-Dist: PyYAML>=6.0.1; extra == "imaspy"
+Provides-Extra: test
+Requires-Dist: pytest>=6.0; extra == "test"
+
+# Data Access Library
+
+Data access for applications represented using Interface Data Structures (IDSs) or Control Breakdown Structure (CBS).
+
+## Requirements
+
+1. **python <= 3.11**
+2. **Dependencies**: Managed
+   via [pyproject.toml](https://github.com/iplot-viz/iplotdataaccess/blob/develop/pyproject.toml).
+
+## Installation
+
+Install the package from PyPi:
+
+  ```bash
+  pip install iplotDataAccess
+  ```
+
+### Installation with optional dependencies
+
+You can also install optional dependencies (e.g. imaspy and test):
+
+  ```bash
+  pip install "iplotDataAccess[imaspy, test]"
+  ```
+
+#### List of optional dependencies
+
+- ```imaspy```:  installs required packages.
+- ```test```: installs required packages to run the tests with pytest.
+
+### Usage Example
+
+  ```bash
+  from iplotDataAccess.dataAccess import DataAccess
+
+   # Create a DataAccess object
+   da = DataAccess()
+   
+   # Example: load data from CODAC UDA source
+   data_obj = da.get_data("codacuda", varname="IC_ICH1_FAFB_MEAS/phase", tsS="2018-11-21T09:30:00", tsE="2018-11-21T09:45:00", nbp=-1)
+   print(data_obj.xdata, data_obj.ydata)
+  ```
+
+### Contributing
+
+1. Fork it!
+2. Create your feature branch: ```git checkout -b my-new-feature ```
+3. Commit your changes: ```git commit -am 'Add some feature' ```
+4. Push to the branch:```git push origin my-new-feature ```
+5. Submit a pull request

iplotdataaccess-1.3.1.post1/README.md

@@ -0,0 +1,51 @@
+# Data Access Library
+
+Data access for applications represented using Interface Data Structures (IDSs) or Control Breakdown Structure (CBS).
+
+## Requirements
+
+1. **python <= 3.11**
+2. **Dependencies**: Managed
+   via [pyproject.toml](https://github.com/iplot-viz/iplotdataaccess/blob/develop/pyproject.toml).
+
+## Installation
+
+Install the package from PyPi:
+
+  ```bash
+  pip install iplotDataAccess
+  ```
+
+### Installation with optional dependencies
+
+You can also install optional dependencies (e.g. imaspy and test):
+
+  ```bash
+  pip install "iplotDataAccess[imaspy, test]"
+  ```
+
+#### List of optional dependencies
+
+- ```imaspy```:  installs required packages.
+- ```test```: installs required packages to run the tests with pytest.
+
+### Usage Example
+
+  ```bash
+  from iplotDataAccess.dataAccess import DataAccess
+
+   # Create a DataAccess object
+   da = DataAccess()
+   
+   # Example: load data from CODAC UDA source
+   data_obj = da.get_data("codacuda", varname="IC_ICH1_FAFB_MEAS/phase", tsS="2018-11-21T09:30:00", tsE="2018-11-21T09:45:00", nbp=-1)
+   print(data_obj.xdata, data_obj.ydata)
+  ```
+
+### Contributing
+
+1. Fork it!
+2. Create your feature branch: ```git checkout -b my-new-feature ```
+3. Commit your changes: ```git commit -am 'Add some feature' ```
+4. Push to the branch:```git push origin my-new-feature ```
+5. Submit a pull request

iplotdataaccess-1.3.1.post1/iplotDataAccess/__init__.py

@@ -0,0 +1,3 @@
+from . import _version
+
+__version__ = _version.get_versions()['version']

iplotdataaccess-1.3.1.post1/iplotDataAccess/_version.py

@@ -0,0 +1,21 @@
+
+# This file was generated by 'versioneer.py' (0.29) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+import json
+
+version_json = '''
+{
+ "date": "2025-12-19T09:53:38+0000",
+ "dirty": false,
+ "error": null,
+ "full-revisionid": "afb231aec5bbd77563b4857880ccc3c9e6ef3a33",
+ "version": "1.3.1.post1"
+}
+'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)

iplotdataaccess-1.3.1.post1/iplotDataAccess/appDataAccess.py

@@ -0,0 +1,26 @@
+from iplotDataAccess.dataAccess import DataAccess
+from iplotLogging import setupLogger
+
+logger = setupLogger.get_logger(__name__)
+
+
+class AppDataAccess:
+    da: DataAccess = None
+    configured: bool = False
+
+    # ---------------
+
+    @staticmethod
+    def initialize(config_file=None):
+        if AppDataAccess.da is None:
+            AppDataAccess.da = DataAccess()
+        AppDataAccess.configured = AppDataAccess.da.load_config(config_file)
+        return AppDataAccess.configured
+
+    @staticmethod
+    def get_data_access():
+        return AppDataAccess.da
+
+    @staticmethod
+    def is_configured():
+        return AppDataAccess.configured

iplotdataaccess-1.3.1.post1/iplotDataAccess/csvAccess.py

@@ -0,0 +1,191 @@
+import os
+import re
+import pandas as pd
+from pandas import DataFrame
+
+from iplotDataAccess import dataCommon
+from iplotDataAccess.dataSource import DataSource
+import iplotLogging.setupLogger as setupLog
+
+logger = setupLog.get_logger(__name__)
+
+
+class CsvAccess(DataSource):
+    source_type = "CSV"
+
+    def __init__(self, name: str, config: dict):
+        super().__init__(name, config)
+
+        self.folder_path = config.get("path", "")  # Store the folder path for accessing CSV files
+        self.def_pulse_location = self.folder_path.split('/')[-1]
+
+    def connect(self) -> bool:
+        self.connected = os.path.isdir(self.folder_path)
+        return self.connected
+
+    # Method to get data from a CSV file based on the 'pulse' and 'varname' arguments in kwargs
+    def get_data(self, **kwargs):
+
+        # Create a DataObj instance for storing the data
+        data_obj = dataCommon.DataObj()
+
+        # Transform the 'pulse' argument to a valid file path
+        path = self.transform_pulse_to_file_path(kwargs.get("pulse"))
+
+        # Read the CSV file into a pandas DataFrame
+        data = pd.read_csv(path)
+
+        if kwargs.get("tsS") is not None:
+            data = data[data['Time'] >= kwargs.get("tsS")]
+        if kwargs.get("tsE") is not None:
+            data = data[data['Time'] <= kwargs.get("tsE")]
+
+        sub_data = data.filter(like=kwargs.get("varname"))
+
+        # If multiple columns match 'varname', return an error
+        if len(sub_data.columns) > 1:
+            data_obj.set_err(-1, "Multiple columns with same variable name")
+            return data_obj
+
+        if len(sub_data.columns) == 0:
+            data_obj.set_err(-1, f"No columns found matching variable name '{kwargs.get('varname')}'")
+            return data_obj
+
+        # Set the data for the variable in the DataObj
+        data_obj.set_data(sub_data.iloc[:, 0].values, 2)
+        logger.debug(" found data_obj %s ", data_obj)
+
+        # Extract the unit from the variable's column name (if present)
+        yunit = re.findall(r' \((.*?)\)', sub_data.columns[0])
+        if yunit:
+            data_obj.yunit = yunit[0]
+
+        # Set the 'Time' column data in the DataObj and define xunit as "seconds"
+        data_obj.set_data(data["Time"].values, 1)
+        data_obj.xunit = "s"
+
+        return data_obj  # Return the populated DataObj
+
+    # Method to get all pulses (files) in the folder matching a pattern as a list
+    def get_pulses_df(self, pattern='.*') -> DataFrame:
+        all_pulses = []
+        pstatus = "completed"
+
+        # Walk through all files and folders in the directory
+        for folder, _, files in os.walk(self.folder_path):
+            relative_folder = os.path.relpath(folder, self.folder_path).replace(os.sep, "/")
+            # For each file, create a key with the folder and cleaned file name
+            for file in files:
+                if not file.endswith(".csv"):
+                    continue
+                value = f"{self.def_pulse_location}:{relative_folder}/{file.replace('.csv', '').replace('data_', '')};{pstatus}"
+                logger.debug(" found value and file %s %s", value, file)
+                if re.match(pattern, value):
+                    all_pulses.append(value)
+
+        # Return the pulses
+        logger.debug(" pulses  %s", all_pulses)
+        return pd.DataFrame([value.split(";") for value in all_pulses], columns=["pulseId", "Status"])
+
+    def get_cbs_list(self, pattern=".*"):
+        var_list = self.get_var_list(pattern)
+        cbs = set()
+
+        for v in var_list:
+            cbs.add(v + "?V")
+
+        logger.debug(" cbs list %s", cbs)
+        return cbs
+
+    def get_cbs_dictX(self, pattern='.*') -> dict:
+        cbs_list = self.get_cbs_list(pattern)
+        cbs_dict = dict()
+        for line in cbs_list:
+            cur_dict = cbs_dict
+            cbs = line.split(":")
+            for counter, s in enumerate(cbs):
+                data = '-'.join(cbs[0:counter])
+                if data not in cur_dict:
+                    cur_dict = cur_dict.setdefault(data, {})
+            cur_dict = cur_dict.setdefault(line.replace('?V'), '')
+        logger.debug(" cbs list %s", cbs_dict)
+        return cbs_dict
+
+    def get_cbs_dict(self, pattern='.*') -> dict:
+        cbs_list = self.get_cbs_list(pattern)
+        cbs_dict = dict()
+        for line in cbs_list:
+            cur_dict = cbs_dict
+            list_line = line.split('-')
+            for var in list_line:
+                if var.endswith('?V'):
+                    cur_dict = cur_dict.setdefault('-'.join(list_line).replace('?V', ''), '')
+                else:
+                    cur_dict = cur_dict.setdefault(var, {})
+
+        return cbs_dict
+
+    def get_pulse_info(self, pulse, run):
+        all_pulses = self.get_pulses_df()
+        return None
+
+    def get_var_dict(self, pattern='.*'):
+        return self.get_cbs_dict(pattern)
+
+    def is_connected(self):
+        return self.connected is not None
+
+    def search_pulses_df(self, text) -> DataFrame:
+        return self.get_pulses_df(pattern=text)
+
+    # Method to get a list of all unique variables from CSV files in the folder
+    def get_var_list(self, patt='.*'):
+        all_variables = set()  # Use a set to store unique variables
+
+        # Walk through all files and folders in the directory
+        for folder, _, files in os.walk(self.folder_path):
+            for file in files:
+                if not file.endswith(".csv"):
+                    continue
+                file_path = os.path.join(folder, file)  # Get the full file path
+                try:
+                    # Open each file and read the header line to extract variables
+                    with open(file_path, 'r') as f:
+                        x = f.readline().split(",")[1:]  # Skip the first column (Time)
+                        all_variables = all_variables.union([i.split(" ")[0] for i in x])
+
+                except Exception as e:
+                    logger.error("Could not open file %s for %s ", file_path, e)  # Handle file reading errors
+        logger.debug(" all variable %s", all_variables)
+
+        # Filter variables matching the given pattern
+        filtered_vars = []
+        for variable in all_variables:
+            s = re.match(patt, variable.strip())
+
+            if s is not None:
+                filtered_vars.append(s[0])
+
+        logger.debug(" variable  %s  and pattern %s ", filtered_vars, patt)
+
+        return filtered_vars  # Return the filtered list of variables
+
+    # Method to transform a pulse string into a file path
+    # Example: input 'ITER/COMM:111' into
+    # '{user_path}\\iplotdataaccess\\iplotDataAccess\\ITER\\COMM\\data_111.csv'
+    def transform_pulse_to_file_path(self, pulse):
+        # Split the pulse into folder and file
+        folders, file = pulse.split(":", 1)
+        f1, f2 = file.split("/", 1)
+        print(f" variable {pulse} ")
+        # Replace slashes with the OS-specific separator
+        folders = folders.replace("/", os.sep)
+        # Join all parts
+        result = f"{self.folder_path}{os.sep}{f1}{os.sep}data_{f2}.csv"
+        return result  # Return the constructed file path
+
+    def clear_cache(self):
+        pass
+
+    def get_envelope(self):
+        pass

iplotdataaccess-1.3.1.post1/iplotDataAccess/dataAccess.py

@@ -0,0 +1,195 @@
+import json
+import os
+import importlib.util
+import importlib.resources as pkg_resources
+from typing import Dict, List, Union, Type
+
+from iplotDataAccess.dataSource import DataSource
+from iplotLogging import setupLogger
+from iplotDataAccess.dataCommon import DataObj, DataEnvelope
+
+logger = setupLogger.get_logger(__name__)
+
+
+# class to interface with data source - here UDA
+class DataAccess:
+    DEFAULT_DATA_SOURCES_CFG_FILE: str = 'mydatasources.cfg'
+
+    def __init__(self):
+        self.proto: Dict[str, Type[DataSource]] = self.get_supported_data_source()
+        self.ds_list: Dict[str, DataSource] = {}
+        self.default_ds: Union[DataSource, None] = None
+
+    @staticmethod
+    def get_supported_data_source() -> Dict[str, Type[DataSource]]:
+        supported_data_sources = {}
+
+        try:
+            file_path = str(pkg_resources.files('iplotDataAccess').joinpath('data_sources.cfg'))
+            with open(file_path, 'r') as file:
+
+                data_sources = json.load(file)
+                for key, value in data_sources.items():
+                    try:
+                        module = importlib.import_module(value['pymodule'])
+                        imported_class = getattr(module, value['class'])
+                        supported_data_sources[imported_class.source_type] = imported_class
+                    except Exception as e:
+                        logger.error(f"Error loading DataSource {key} -> {e}")
+
+        except Exception as e:
+            logger.error(f"Error loading DataSources config file ->{e}")
+
+        return supported_data_sources
+
+    def get_default_ds_name(self):
+        if self.default_ds is None:
+            return None
+        else:
+            return self.default_ds.name
+
+    def load_config(self, conf_file=None) -> bool:
+        conf_files = [conf_file, os.environ.get('IPLOT_SOURCES_CONFIG'), self.DEFAULT_DATA_SOURCES_CFG_FILE]
+        # Remove None values
+        conf_files = list(filter(None, conf_files))
+        for ix, file in enumerate(conf_files):
+            try:
+                return self.load_config_file(file)
+            except (OSError, IOError, FileNotFoundError) as _:
+                if ix < len(conf_files) - 1:
+                    logger.warning(f"Error loading {file} data source file, fallback to {conf_files[ix + 1]}")
+        return False
+
+    def load_config_file(self, dspath: str) -> bool:
+        with open(dspath) as f:
+            try:
+                config = json.load(f)
+            except Exception as e:
+                logger.warning(f"Wrong json format in {dspath} -> {e}")
+                return False
+            for ds_name, ds_config in config.items():
+                ds_type = ds_config.get("type")
+                ds_class = self.proto.get(ds_type)
+                if not ds_class:
+                    logger.warning(f"DataSource '{ds_name}' has an unsupported data source type-> {ds_type}")
+                    continue
+                try:
+                    data_source = ds_class(ds_name, ds_config)
+                    if data_source.connect():
+                        self.ds_list[ds_name] = data_source
+                except Exception as e:
+                    logger.warning(f"Error importing class {ds_class} with error {e}")
+
+        if self.ds_list:
+            # Check which data source to set by default
+            # Set first DataSource that has default=true if no one has it, set the first one
+            self.default_ds = next(
+                (ds for ds in self.ds_list.values() if ds.default),
+                list(self.ds_list.values())[0])
+            return True
+
+        return False
+
+    def get_data_source(self, data_s_name):
+        logger.debug("entering getDataSource  %s", data_s_name)
+        if data_s_name is None:
+            if self.default_ds is not None:
+                logger.info("default source used ")
+                return self.default_ds
+            else:
+                logger.error("DataSourceName is None and not default data source name has been defined")
+                return None
+        if data_s_name not in self.ds_list.keys():
+            logger.warning(" Data source %s not found", data_s_name)
+            return None
+        else:
+            ds = self.ds_list[data_s_name]
+            if ds is None:
+                logger.debug("Invalid data source pointer for ds name  %s", data_s_name)
+            return ds
+
+    def get_data(self, data_s_name, **kwargs):
+        # we can use the var prefix to get the data source while we introduce
+        logger.debug("entering getdata  %s", data_s_name)
+        if data_s_name is not None and data_s_name in self.ds_list.keys():
+            if self.ds_list[data_s_name] is None:
+                dobj = DataObj()
+
+                dobj.set_empty("Invalid data source pointer for ds name " + data_s_name)
+                logger.debug("Invalid data source pointer for ds name  %s", data_s_name)
+                return dobj
+            else:
+
+                dobj = self.ds_list[data_s_name].get_data(**kwargs)
+                return dobj
+        else:
+            if data_s_name not in self.ds_list.keys():
+                logger.warning(" Invalid data source found %s ", data_s_name)
+                dobj = DataObj()
+                dobj.set_empty(f"Invalid data source name {data_s_name}")
+
+                return dobj
+            if self.default_ds is not None:
+                logger.info(" default source used ")
+                return self.default_ds.get_data(**kwargs)
+
+        return None
+
+    def start_subscription(self, data_s_name, **kwargs):
+        if data_s_name is not None and data_s_name in self.ds_list.keys():
+            self.ds_list[data_s_name].start_subscription(**kwargs)
+
+    def stop_subscription(self, data_s_name):
+        if data_s_name is not None and data_s_name in self.ds_list.keys():
+            logger.debug("stopSubscription A ")
+            self.ds_list[data_s_name].stop_subscription()
+
+    def get_next_data(self, data_s_name, vname):
+        if data_s_name is not None and data_s_name in self.ds_list.keys():
+            return self.ds_list[data_s_name].get_next_data(vname)
+        else:
+            dobj = DataObj()
+            dobj.set_empty(f"Invalid data source name {data_s_name}")
+            return dobj
+
+    def get_envelope(self, data_s_name, **kwargs):
+        if data_s_name is not None and data_s_name in self.ds_list.keys():
+            if self.ds_list[data_s_name] is None:
+                denv = DataEnvelope()
+                denv.set_empty(f"Invalid data source pointer for ds name {data_s_name}")
+
+                return denv
+            else:
+                return self.ds_list[data_s_name].get_envelope(**kwargs)
+        else:
+            if data_s_name not in self.ds_list.keys():
+                logger.warning(f"Invalid data source found {data_s_name}")
+                denv = DataEnvelope()
+                denv.set_empty(f"Invalid data source name {data_s_name}")
+
+                return denv
+            if self.default_ds is not None:
+                logger.info("default source used ")
+                return self.default_ds.get_envelope(**kwargs)
+
+        return None
+
+    def get_connected_data_source_names(self) -> List[str]:
+        data_sources = [self.get_default_ds_name()]
+        for ds_name, ds in self.ds_list.items():
+            if ds_name not in data_sources and ds.is_connected():
+                data_sources.append(ds_name)
+        return data_sources
+
+    def get_connected_data_sources(self) -> List[DataSource]:
+        data_sources = []
+        for ds_name, ds in self.ds_list.items():
+            if ds.is_connected():
+                data_sources.append(ds)
+        return data_sources
+
+    # Clear cache of all the dataSources
+    def clear_cache(self):
+        for ds in self.ds_list.values():
+            if ds.is_connected():
+                ds.clear_cache()