pipeline-eds 0.2.4__py3-none-any.whl

pipeline/api/rjn.py

@@ -0,0 +1,157 @@
+import requests
+import logging
+from typing import Union
+
+from src.pipeline.calls import make_request, call_ping
+from src.pipeline.env import find_urls
+from src.pipeline.decorators import log_function_call
+from src.pipeline import helpers
+from src.pipeline.time_manager import TimeManager
+
+logger = logging.getLogger(__name__)
+
+class RjnClient:
+    def __init__(self,config):
+        self.config = config
+    
+    def send_point(self, payload: dict):
+        request_url = f"{self.api_url}/data/point"  # Adjust this if needed
+        response = make_request(url=request_url, headers=self.headers, data=payload, method="POST")
+        if response.status_code == 200:
+            print(f"Successfully posted point {payload.get('rjn_name')}")
+        else:
+            print(f"Failed to post point {payload.get('rjn_name')}: {response.status_code}")
+
+    @staticmethod
+    def login_to_session(api_url, client_id, password):
+        logger.info("RjnClient.login_to_session()")
+        session = requests.Session()
+
+        data = {'client_id': client_id, 'password': password, 'type': 'script'}
+        
+        try:
+            response = session.post(f'{api_url}/auth', json=data, verify=True)
+            response.raise_for_status() # catch 4xx/5xx html status
+            token = response.json().get('token')
+            session.headers['Authorization'] = f'Bearer {token}'
+            print("Status code:", response.status_code)
+            print("Response text:", response.text)
+            return session
+        except requests.exceptions.SSLError as ssl_err:
+            logging.warning("SSL verification failed. Will retry on next scheduled cycle.")
+            logging.debug(f"SSL error details: {ssl_err}")
+            return None
+
+        except requests.exceptions.ConnectionError as conn_err:
+            logging.warning("Connection error during authentication. Will retry next hour.")
+            logging.debug(f"Connection error details: {conn_err}")
+            return None
+
+        except Exception as general_err:
+            logging.error("Unexpected error during login.", exc_info=True)
+            return None
+
+    @staticmethod
+    #def send_data_to_rjn(session, base_url:str, project_id:str, entity_id:int, timestamps: list[Union[int, float, str]], values: list[float]):
+    def send_data_to_rjn(session, base_url, project_id, entity_id, timestamps, values):
+        if timestamps is None:
+            raise ValueError("timestamps cannot be None")
+        if values is None:
+            raise ValueError("values cannot be None")
+        if not isinstance(timestamps, list):
+            raise ValueError("timestamps must be a list. If you have a single timestamp, use: [timestamp] ")
+        if not isinstance(values, list):
+            raise ValueError("values must be a list. If you have a single value, use: [value] ")
+        # Check for matching lengths of timestamps and values
+        if len(timestamps) != len(values):
+            raise ValueError(f"timestamps and values must have the same length: {len(timestamps)} vs {len(values)}")
+
+        timestamps_str = [TimeManager(ts).as_formatted_date_time() for ts in timestamps]
+
+        url = f"{base_url}/projects/{project_id}/entities/{entity_id}/data"
+        params = {
+            "interval": 300,    
+            "import_mode": "OverwriteExistingData",
+            "incoming_time": "DST"#, # DST seemed to fail and offset by an hour into the future. UTC with central time seemed to fail and offset the data 5 hours into the past. 
+            #"local_timezone": "CST_CentralStandardTime"
+        }
+        body = {
+        "comments": "Imported from EDS.",
+        "data": dict(zip(timestamps_str, values))  # Works for single or multiple entries
+        }
+        
+
+        response = None
+        try:
+            response = session.post(url=url, json= body, params = params)
+
+            print("Status code:", response.status_code)
+            print("Response text:", response.text)
+            if response is None:
+                print("Response = None, job cancelled")
+            else:
+                response.raise_for_status()
+                print(f"Sent timestamps and values to entity {entity_id} (HTTP {response.status_code})")
+                return True
+        except requests.exceptions.ConnectionError as e:
+            print("Skipping RjnClient.send_data_to_rjn() due to connection error")
+            print(e)
+            return False
+        except requests.exceptions.RequestException as e:
+            print(f"Error sending data to RJN: {e}")
+            if response is not None:# and response.status_code != 500:
+                logging.debug(f"Response content: {response.text}")  # Print error response
+                
+            return False
+                
+    @staticmethod
+    def ping():
+        from src.pipeline.env import SecretConfig
+        from src.pipeline.workspace_manager import WorkspaceManager
+        workspace_name = WorkspaceManager.identify_default_workspace_name()
+        workspace_manager = WorkspaceManager(workspace_name)
+        secrets_dict = SecretConfig.load_config(secrets_file_path = workspace_manager.get_secrets_file_path())
+        
+        secrets_dict = SecretConfig.load_config(secrets_file_path = workspace_manager.get_secrets_file_path())
+        sessions = {}
+
+        url_set = find_urls(secrets_dict)
+        for url in url_set:
+            if "rjn" in url.lower():
+                print(f"ping url: {url}")
+                call_ping(url)
+
+@log_function_call(level=logging.DEBUG)
+def demo_rjn_ping():
+    from src.pipeline.calls import call_ping
+    from src.pipeline.env import SecretConfig
+    from src.pipeline.workspace_manager import WorkspaceManager
+
+    from src.pipeline.env import SecretConfig
+    from src.pipeline.workspace_manager import WorkspaceManager
+    workspace_name = WorkspaceManager.identify_default_workspace_name()
+    workspace_manager = WorkspaceManager(workspace_name)
+
+    secrets_dict = SecretConfig.load_config(secrets_file_path = workspace_manager.get_secrets_file_path())
+    
+    base_url = secrets_dict.get("contractor_apis", {}).get("RJN", {}).get("url").rstrip("/")
+    session = RjnClient.login_to_session(api_url = base_url,
+                                    client_id = secrets_dict.get("contractor_apis", {}).get("RJN", {}).get("client_id"),
+                                    password = secrets_dict.get("contractor_apis", {}).get("RJN", {}).get("password"))
+    if session is None:
+        logger.warning("RJN session not established. Skipping RJN-related data transmission.\n")
+        return
+    else:
+        logger.info("RJN session established successfully.")
+        session.base_url = base_url
+        response = call_ping(session.base_url)
+
+if __name__ == "__main__":
+    import sys
+    cmd = sys.argv[1] if len(sys.argv) > 1 else "default"
+
+    if cmd == "ping":
+        demo_rjn_ping()
+    else:
+        print("Usage options: \n"
+        "poetry run python -m pipeline.api.rjn ping")

pipeline/api/status_api.py

@@ -0,0 +1,9 @@
+# src/api/status_api.py
+from fastapi import FastAPI
+from src.pipeline.daemon.status import get_latest_status
+
+app = FastAPI()
+
+@app.get("/status")
+def read_status():
+    return get_latest_status()

pipeline/calls.py

@@ -0,0 +1,108 @@
+import requests
+import certifi
+import platform
+import subprocess
+import sys
+import time
+import logging
+from urllib.parse import urlparse
+from urllib3.exceptions import NewConnectionError
+
+logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s")
+
+def test_connection_to_internet():
+    try:
+        # call Cloudflare's CDN test site, because it is lite.
+        response = requests.get("http://1.1.1.1", timeout = 5)
+        print("You are connected to the internet.")
+    except:
+        print(f"It appears you are not connected to the internet.")
+        sys.exit()
+
+def make_request(url, data=None, params = None, method="POST", headers=None, retries=3, delay=2, timeout=10, verify_ssl=True):
+    """Now defunct, converted to a requests.Session() paradigm."""
+    default_headers = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+    }
+
+    merged_headers = {**default_headers, **(headers or {})}
+    #print(f"merged_headers = {merged_headers}")
+
+    verify = certifi.where() if verify_ssl else False
+
+    request_func = {
+        "POST": requests.post,
+        "GET": requests.get,
+        "PUT": requests.put,
+        "DELETE": requests.delete,
+        "PATCH": requests.patch,
+    }.get(method.upper())
+
+    if not request_func:
+        logging.error(f"Unsupported HTTP method: {method}")
+        return None
+        #raise ValueError(f"Unsupported HTTP method: {method}")
+    try:
+        response = request_func(
+            url,
+            json=data,
+            params=params,
+            headers=merged_headers,
+            timeout=timeout,
+            verify=verify
+        )
+        response.raise_for_status()
+        return response
+    except requests.exceptions.SSLError as e:
+        #raise ConnectionError(f"SSL Error: {e}")
+        logging.error(f"SSL Error: {e}")
+        return None
+    except requests.exceptions.HTTPError as e:
+        if response.status_code == 500:
+            logging.error(f"HTTP 500 Error - Response content: {response.text}")
+        elif response.status_code == 503 and retries > 0:
+            logging.warning(f"Service unavailable (503). Retrying in {delay} seconds...")
+            time.sleep(delay)
+            #return make_request(url, data, retries - 1, delay * 2)  # Exponential backoff
+            return make_request(url, data, params, method, headers, retries - 1, delay * 2, timeout, verify_ssl)
+        elif response.status_code == 403:
+            #raise PermissionError("Access denied (403). The server rejected your credentials or IP.")
+            logging.error("Access denied (403). The server rejected your credentials or IP.")
+        else:
+            #raise RuntimeError(f"HTTP error: {response.status_code} {response.text}")
+            logging.error(f"HTTP error: {response.status_code} {response.text}")
+        return None
+    except requests.exceptions.RequestException as e:
+        logging.warning(f"Request failed: {e}")
+        return None  # Ensures calling functions properly handle failure
+    except NewConnectionError as e:
+        logging.warning("Request failed due to connection issues.")
+        logging.debug(f"Detailed error: {e}", exc_info=False)  # Only logs full traceback if DEBUG level is set
+
+def call_ping(url):
+    parsed = urlparse(url)
+    param = "-n" if platform.system().lower() == "windows" else "-c"
+    command = ["ping", param, "1", parsed.hostname]
+    return subprocess.call(command) == 0  # True if ping succeeds
+
+def find_urls(config_dict):
+    url_set = set()
+
+    def recursive_search(d):
+        if isinstance(d, dict):
+            for k, v in d.items():
+                if isinstance(v, str) and v.startswith("http"):
+                    url_set.add(v)
+                elif isinstance(v, dict):
+                    recursive_search(v)
+                elif isinstance(v, list):
+                    for item in v:
+                        recursive_search(item)
+
+    recursive_search(config_dict)
+    return url_set
+
+if __name__ == "__main__":
+    from src.pipeline.helpers import function_view
+    function_view()

pipeline/cli.py

@@ -0,0 +1,282 @@
+'''
+import typer
+from pathlib import Path
+
+app = typer.Typer()
+
+@app.command()
+def list_workspaces(workspaces_dir: Path = Path("workspaces")):
+    """List valid mulch workspaces in the given directory."""
+    if not workspaces_dir.exists():
+        typer.echo(f"Directory not found: {workspaces_dir}")
+        raise typer.Exit(code=1)
+    for path in workspaces_dir.iterdir():
+        if path.is_dir() and (path / ".mulch").is_dir():
+            typer.echo(f"🪴 {path.name}")
+
+@app.command()
+def list_mulch_folders(start: Path = Path(".")):
+    """Recursively find folders containing a .mulch/ directory."""
+    for path in start.rglob(".mulch"):
+        typer.echo(f"📁 {path.parent}")
+
+@app.command()
+def inspect(workspace: Path):
+    """Show scaffold or metadata info from a workspace."""
+    metadata = workspace / ".mulch" / "mulch-scaffold.json"
+    if metadata.exists():
+        typer.echo(f"🔍 {workspace.name}: {metadata}")
+        typer.echo(metadata.read_text())
+    else:
+        typer.echo(f"No scaffold found in {workspace}")
+'''
+# src/pipeline/cli.py
+
+import typer
+import importlib
+from pathlib import Path
+
+from src.pipeline.env import SecretConfig
+#from src.pipeline.helpers import setup_logging
+from src.pipeline.workspace_manager import WorkspaceManager
+
+app = typer.Typer(help="CLI for running pipeline workspaces.")
+
+
+@app.callback(invoke_without_command=True)
+def main(ctx: typer.Context):
+    """
+    Pipeline CLI – run workspaces built on the pipeline framework.
+    """
+    if ctx.invoked_subcommand is None:
+        typer.echo(ctx.get_help())
+        raise typer.Exit()
+
+@app.command()
+def run(
+    workspace: str = typer.Option(None, help="Workspace to run"),
+):
+    """
+    Import and run a workspace's main() function.
+    """
+    # Determine workspace name
+    if workspace is None:
+        workspace = WorkspaceManager.identify_default_workspace_name()
+    wm = WorkspaceManager(workspace)
+
+    workspace_dir = wm.get_workspace_dir()
+    module_path = f"workspaces.{workspace}.main"
+
+    typer.echo(f"🚀 Running {module_path} from {workspace_dir}")
+
+    try:
+        mod = importlib.import_module(module_path)
+        if not hasattr(mod, "main"):
+            typer.echo("❌ This workspace does not have a 'main()' function.")
+            raise typer.Exit(1)
+        mod.main()
+    except Exception as e:
+        typer.echo(f"💥 Error while running {workspace}: {e}")
+        raise typer.Exit(1)
+
+@app.command()
+def typical(zd: str):
+    """
+    Print the typical idcs list for an EDS zd.
+    """
+    pass
+
+@app.command()
+def trend(
+    idcs: list[str] = typer.Argument(..., help="Provide known idcs values that match the given zd."), # , "--idcs", "-i"
+    starttime: str = typer.Option(None, "--start", "-s", help="Index from 'mulch order' to choose scaffold source."),
+    endtime: str = typer.Option(None, "--end", "-end", help="Reference a known template for workspace organization."),
+    zd: str = typer.Option('Maxson', "--zd", "-z", help = "Define the EDS ZD from your secrets file. This must correlate with your idcs point selection(s)."),
+    workspace: str = typer.Option(WorkspaceManager.identify_default_workspace_name(),"--workspace","-w", help = "Provide the name of the workspace you want to use, for the secrets.yaml credentials and for the timezone config. If a start time is not provided, the workspace queries can checked for the most recent successful timestamp. ")
+    ):
+    """
+    Show a curve for a sensor over time.
+    """
+    #from dateutil import parser
+    import pendulum
+    from pipeline.api.eds import EdsClient, load_historic_data
+    from pipeline import helpers
+    from pipeline.queriesmanager import QueriesManager
+    from src.pipeline.plotbuffer import PlotBuffer
+    from src.pipeline import gui_fastapi_plotly_live
+    from src.pipeline import environment
+
+    if zd.lower() == "stiles":
+        zd = "WWTF"
+
+    if zd == "Maxson":
+        idcs_to_iess_suffix = ".UNIT0@NET0"
+    elif zd == "WWTF":
+        idcs_to_iess_suffix = ".UNIT1@NET1"
+    else:
+        # assumption
+        idcs_to_iess_suffix = ".UNIT0@NET0"
+    iess_list = [x+idcs_to_iess_suffix for x in idcs]
+
+
+    wm = WorkspaceManager(workspace)
+    secrets_dict = SecretConfig.load_config(secrets_file_path = wm.get_secrets_file_path())
+
+    base_url = secrets_dict.get("eds_apis", {}).get(zd, {}).get("url").rstrip("/")
+    session = EdsClient.login_to_session(api_url = base_url,
+                                                username = secrets_dict.get("eds_apis", {}).get(zd, {}).get("username"),
+                                                password = secrets_dict.get("eds_apis", {}).get(zd, {}).get("password"))
+    session.base_url = base_url
+    session.zd = secrets_dict.get("eds_apis", {}).get(zd, {}).get("zd")
+    queries_manager = QueriesManager(wm)
+
+    if starttime is None:
+        # back_to_last_success = True
+        dt_start = queries_manager.get_most_recent_successful_timestamp(api_id=zd)
+    else:
+        dt_start = pendulum.parse(starttime, strict=False)
+    if endtime is None:
+        dt_finish = helpers.get_now_time_rounded(wm)
+    else:
+        dt_finish = pendulum.parse(endtime, strict=False)
+
+    # Should automatically choose time step granularity based on time length; map 
+    
+    results = load_historic_data(queries_manager, wm, session, iess_list, dt_start, dt_finish) 
+    
+    data_buffer = PlotBuffer()
+    for idx, rows in enumerate(results):
+        for row in rows:
+            label = f"{row.get('rjn_entityid')} ({row.get('units')})"
+            ts = helpers.iso(row.get("ts"))
+            av = row.get("value")
+            data_buffer.append(label, ts, av)
+    
+    if not environment.matplotlib_enabled():
+        gui_fastapi_plotly_live.run_gui(data_buffer)
+    else:
+        from pipeline import gui_mpl_live
+        gui_mpl_live.run_gui(data_buffer)
+
+
+
+@app.command()
+def list_workspaces():
+    """
+    List all available workspaces detected in the workspaces folder.
+    """
+    # Determine workspace name
+    
+    workspace = WorkspaceManager.identify_default_workspace_name()
+    wm = WorkspaceManager(workspace)
+    workspaces = wm.get_all_workspaces_names()
+    typer.echo("📦 Available workspaces:")
+    for name in workspaces:
+        typer.echo(f" - {name}")
+
+@app.command()
+def demo_rjn_ping():
+    """
+    Demo function to ping RJN service.
+    """
+    from src.pipeline.api.rjn import RjnClient
+    from src.pipeline.calls import call_ping
+    from src.pipeline.env import SecretConfig
+    from src.pipeline.workspace_manager import WorkspaceManager
+    from src.pipeline import helpers
+    import logging
+
+    logger = logging.getLogger(__name__)
+    workspace_name = WorkspaceManager.identify_default_workspace_name()
+    workspace_manager = WorkspaceManager(workspace_name)
+
+    secrets_dict = SecretConfig.load_config(secrets_file_path = workspace_manager.get_secrets_file_path())    
+    base_url = secrets_dict.get("contractor_apis", {}).get("RJN", {}).get("url").rstrip("/")
+    session = RjnClient.login_to_session(api_url = base_url,
+                                    client_id = secrets_dict.get("contractor_apis", {}).get("RJN", {}).get("client_id"),
+                                    password = secrets_dict.get("contractor_apis", {}).get("RJN", {}).get("password"))
+    if session is None:
+        logger.warning("RJN session not established. Skipping RJN-related data transmission.\n")
+        return
+    else:
+        logger.info("RJN session established successfully.")
+        session.base_url = base_url
+        response = call_ping(session.base_url)
+
+@app.command()
+def ping_rjn_services():
+    """
+    Ping all RJN services found in the secrets configuration.
+    """
+    from src.pipeline.calls import find_urls, call_ping
+    from src.pipeline.env import SecretConfig
+    from src.pipeline.workspace_manager import WorkspaceManager
+    import logging
+
+    logger = logging.getLogger(__name__)
+    workspace_name = WorkspaceManager.identify_default_workspace_name()
+    workspace_manager = WorkspaceManager(workspace_name)
+
+    secrets_dict = SecretConfig.load_config(secrets_file_path = workspace_manager.get_secrets_file_path())
+    
+    sessions = {}
+
+    url_set = find_urls(secrets_dict)
+    for url in url_set:
+        if "rjn" in url.lower():
+            print(f"ping url: {url}")
+            call_ping(url)
+
+@app.command()
+def ping_eds_services():
+    """
+    Ping all EDS services found in the secrets configuration.
+    """
+    from src.pipeline.calls import find_urls, call_ping
+    from src.pipeline.env import SecretConfig
+    from src.pipeline.workspace_manager import WorkspaceManager
+    import logging
+
+    logger = logging.getLogger(__name__)
+    workspace_name = WorkspaceManager.identify_default_workspace_name()
+    workspace_manager = WorkspaceManager(workspace_name)
+
+    secrets_dict = SecretConfig.load_config(secrets_file_path = workspace_manager.get_secrets_file_path())
+    
+    sessions = {}
+
+    url_set = find_urls(secrets_dict)
+    typer.echo(f"Found {len(url_set)} URLs in secrets configuration.")
+    logger.info(f"url_set: {url_set}")
+    for url in url_set:
+        if "172.19.4" in url.lower():
+            print(f"ping url: {url}")
+            call_ping(url)
+
+@app.command()
+def daemon_runner_main():
+    """
+    Run the daemon_runner script from the eds_to_rjn workspace.
+    """
+    import workspaces.eds_to_rjn.scripts.daemon_runner as dr
+
+    dr.main()
+
+@app.command()
+def daemon_runner_once():
+    """
+    Run the daemon_runner script from the eds_to_rjn workspace.
+    """
+    import workspaces.eds_to_rjn.scripts.daemon_runner as dr
+
+    dr.run_hourly_tabular_trend_eds_to_rjn()
+
+@app.command()
+def help():
+    """
+    Show help information.
+    """
+    typer.echo(app.get_help())
+
+if __name__ == "__main__":
+    app()

pipeline/configrationmanager.py

@@ -0,0 +1,22 @@
+'''
+Title: configurationmanager.py
+Author: George Clayton Bennett
+Created : 05 June 2025
+
+Purpose: Modularized file-based configuration via a Singlteon class. In this case, configuration is just credentials.
+
+Attributes:
+    - Load default config values from a TOML file
+    - No fallbacks for secret.yaml files. 
+    - Tracking history of changes to allow undo functionality if values are changed from the default during p(unexpected in this case).
+'''
+
+
+import toml
+# import os
+# from colletion import defaultdict
+
+class ConfigurationManager:
+    
+    def __init__(self):
+        self._instance = None

pipeline/decorators.py

@@ -0,0 +1,13 @@
+import logging
+import inspect
+from functools import wraps
+
+def log_function_call(level=logging.DEBUG):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            func_name = func.__qualname__
+            logging.log(level, f"Start: {func_name}() args={args} kwargs={kwargs}")
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator

pipeline/env.py

@@ -0,0 +1,61 @@
+#env.__main__.py
+ 
+import yaml
+from src.pipeline.workspace_manager import WorkspaceManager 
+
+'''
+migrate this to ConfigurationManager
+'''
+
+class SecretConfig:
+    def __init__(self, config):
+        self.config = config
+
+    @staticmethod
+    def load_config(secrets_file_path): 
+        with open(secrets_file_path, 'r') as f:
+            return yaml.safe_load(f)
+        
+    def print_config(self):
+        # Print the values
+        for section, values in self.config.items():
+            print(f"[{section}]")
+            for key, val in values.items():
+                print(f"{key} = {val}")
+
+
+def find_urls(config, url_set=None):
+    '''determine all values with the key "url" in a config file.'''
+    if url_set is None:
+        url_set = set()
+
+    if isinstance(config, dict):
+        for key, value in config.items():
+            if key == "url":
+                url_set.add(value)
+            else:
+                find_urls(value, url_set)
+    elif isinstance(config, list):
+        for item in config:
+            find_urls(item, url_set)
+
+    return url_set
+
+def demo_secrets():
+    """
+    The defaut SecretConfig.load_config() call 
+    should load fromthe default-workspace 
+    as defined by the configuration file in the workspaces directorys,
+    caed defaut_workspace.toml - Clayton Bennett 26 April 2025.
+    However this call can also be made if another project is made the active project.
+    """
+    workspace_name = WorkspaceManager.identify_default_workspace_name()
+    workspace_manager = WorkspaceManager(workspace_name)
+    config = SecretConfig.load_config(secrets_file_path = workspace_manager.get_secrets_file_path())
+    secrets = SecretConfig(config)
+    return secrets
+
+if __name__ == "__main__":
+    # call from the root directory using: poetry run python -m pipeline.env
+    secrets=demo_secrets()
+    secrets.print_config()