PyPI - pipeline-eds - Versions diffs - 0.2.4__py3-none-any.whl - Mend

pipeline-eds 0.2.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (62) hide show

pipeline/__init__.py +4 -0
pipeline/__main__.py +1 -0
pipeline/api/__init__.py +0 -0
pipeline/api/eds.py +980 -0
pipeline/api/rjn.py +157 -0
pipeline/api/status_api.py +9 -0
pipeline/calls.py +108 -0
pipeline/cli.py +282 -0
pipeline/configrationmanager.py +22 -0
pipeline/decorators.py +13 -0
pipeline/env.py +61 -0
pipeline/environment.py +59 -0
pipeline/gui_fastapi_plotly_live.py +78 -0
pipeline/gui_mpl_live.py +113 -0
pipeline/helpers.py +125 -0
pipeline/logging_setup.py +45 -0
pipeline/pastehelpers.py +10 -0
pipeline/philosophy.py +62 -0
pipeline/plotbuffer.py +21 -0
pipeline/points_loader.py +19 -0
pipeline/queriesmanager.py +122 -0
pipeline/time_manager.py +211 -0
pipeline/workspace_manager.py +253 -0
pipeline_eds-0.2.4.dist-info/LICENSE +14 -0
pipeline_eds-0.2.4.dist-info/METADATA +238 -0
pipeline_eds-0.2.4.dist-info/RECORD +62 -0
pipeline_eds-0.2.4.dist-info/WHEEL +4 -0
pipeline_eds-0.2.4.dist-info/entry_points.txt +6 -0
workspaces/default-workspace.toml +3 -0
workspaces/eds_to_rjn/__init__.py +0 -0
workspaces/eds_to_rjn/code/__init__.py +0 -0
workspaces/eds_to_rjn/code/aggregator.py +84 -0
workspaces/eds_to_rjn/code/collector.py +60 -0
workspaces/eds_to_rjn/code/sanitizer.py +40 -0
workspaces/eds_to_rjn/code/storage.py +16 -0
workspaces/eds_to_rjn/configurations/config_time.toml +11 -0
workspaces/eds_to_rjn/configurations/configuration.toml +2 -0
workspaces/eds_to_rjn/exports/README.md +7 -0
workspaces/eds_to_rjn/exports/aggregate/README.md +7 -0
workspaces/eds_to_rjn/exports/aggregate/live_data - Copy.csv +355 -0
workspaces/eds_to_rjn/exports/aggregate/live_data_EFF.csv +17521 -0
workspaces/eds_to_rjn/exports/aggregate/live_data_INF.csv +17521 -0
workspaces/eds_to_rjn/exports/export_eds_points_neo.txt +11015 -0
workspaces/eds_to_rjn/exports/manual_data_load_to_postman_wetwell.csv +8759 -0
workspaces/eds_to_rjn/exports/manual_data_load_to_postman_wetwell.xlsx +0 -0
workspaces/eds_to_rjn/exports/manual_effluent.csv +8759 -0
workspaces/eds_to_rjn/exports/manual_influent.csv +8759 -0
workspaces/eds_to_rjn/exports/manual_wetwell.csv +8761 -0
workspaces/eds_to_rjn/history/time_sample.txt +0 -0
workspaces/eds_to_rjn/imports/zdMaxson_idcsD321E_sid11003.toml +14 -0
workspaces/eds_to_rjn/imports/zdMaxson_idcsFI8001_sid8528.toml +14 -0
workspaces/eds_to_rjn/imports/zdMaxson_idcsM100FI_sid2308.toml +14 -0
workspaces/eds_to_rjn/imports/zdMaxson_idcsM310LI_sid2382.toml +14 -0
workspaces/eds_to_rjn/queries/default-queries.toml +4 -0
workspaces/eds_to_rjn/queries/points-maxson.csv +4 -0
workspaces/eds_to_rjn/queries/points-stiles.csv +4 -0
workspaces/eds_to_rjn/queries/timestamps_success.json +20 -0
workspaces/eds_to_rjn/scripts/__init__.py +0 -0
workspaces/eds_to_rjn/scripts/daemon_runner.py +212 -0
workspaces/eds_to_rjn/secrets/README.md +24 -0
workspaces/eds_to_rjn/secrets/secrets-example.yaml +15 -0
workspaces/eds_to_termux/..txt +0 -0

pipeline/environment.py ADDED Viewed

@@ -0,0 +1,59 @@
+'''
+Title: environment.py
+Author: Clayton Bennett
+Created: 23 July 2024
+'''
+import platform
+import sys
+def vercel():
+    #return not(windows()) # conflated, when using any linux that is not a webserver
+    # the important questions is actually "are we running on a webserver?"
+    return False # hard code this
+def matplotlib_enabled():
+    if is_termux():
+        return False
+    else:
+        try:
+            import matplotlib
+        except ImportError:
+            return False
+def fbx_enabled():
+    if is_termux():
+        return False
+    else:
+        return True
+def is_termux():
+    # There might be other android versions that can work with the rise od Debian on android in 2025, but for now, assume all android is termux.
+    # I wonder how things would go on pydroid3
+    return is_android()
+def is_android():
+    return "android" in platform.platform().lower()
+def windows():
+    if 'win' in platform.platform().lower():
+        windows=True
+    else:
+        windows=False
+    return windows
+def pyinstaller():
+    if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
+        pyinstaller = True
+    else:
+        pyinstaller = False
+    return pyinstaller
+def frozen():
+    if getattr(sys, 'frozen', True):
+        frozen = True
+    else:
+        frozen = False
+    return frozen
+def operatingsystem():
+    return platform.system() #determine OS

pipeline/gui_fastapi_plotly_live.py ADDED Viewed

@@ -0,0 +1,78 @@
+# src/pipeline/gui_fastapi_plotly_live.py
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse, JSONResponse
+from threading import Lock
+import uvicorn
+import time
+import threading
+import webbrowser
+app = FastAPI()
+plot_buffer = None  # Set externally
+buffer_lock = Lock()
+HTML_TEMPLATE = """
+<!doctype html>
+<html>
+<head>
+    <title>Live Plot</title>
+    <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+</head>
+<body>
+    <h2>Live EDS Data Plot</h2>
+    <div id="live-plot" style="width:90%;height:80vh;"></div>
+    <script>
+        async function fetchData() {
+            const res = await fetch("/data");
+            return await res.json();
+        }
+        async function updatePlot() {
+            const data = await fetchData();
+            const traces = [];
+            for (const [label, series] of Object.entries(data)) {
+                traces.push({
+                    x: series.x,
+                    y: series.y,
+                    name: label,
+                    mode: 'lines+markers',
+                    type: 'scatter'
+                });
+            }
+            Plotly.newPlot('live-plot', traces, { margin: { t: 30 } });
+        }
+        setInterval(updatePlot, 2000);  // Refresh every 2 seconds
+        updatePlot();  // Initial load
+    </script>
+</body>
+</html>
+"""
+@app.get("/", response_class=HTMLResponse)
+async def index():
+    return HTML_TEMPLATE
+@app.get("/data", response_class=JSONResponse)
+async def get_data():
+    with buffer_lock:
+        data = plot_buffer.get_all()  # Should return { label: {"x": [...], "y": [...]}, ... }
+    return data
+def open_browser(port):
+    time.sleep(1)  # Give server a moment to start
+    ## Open in a new Chrome window (if installed)
+    ##chrome_path = webbrowser.get(using='windows-default')
+    ##chrome_path.open_new(f"http://127.0.0.1:{port}")
+    webbrowser.open(f"http://127.0.0.1:{port}")
+def run_gui(buffer, port=8000):
+    global plot_buffer
+    plot_buffer = buffer
+    threading.Thread(target=open_browser, args=(port,), daemon=True).start()
+    uvicorn.run("src.pipeline.gui_fastapi_plotly_live:app", host="127.0.0.1", port=port, log_level="info", reload=False)

pipeline/gui_mpl_live.py ADDED Viewed

@@ -0,0 +1,113 @@
+import numpy as np
+import time
+import logging
+import matplotlib.pyplot as plt
+import matplotlib.animation as animation
+from src.pipeline import helpers
+from src.pipeline.plotbuffer import PlotBuffer  # Adjust import path as needed
+from src.pipeline.time_manager import TimeManager
+logger = logging.getLogger(__name__)
+PADDING_RATIO = 0.25
+def compute_padded_bounds(data):
+    all_x_vals = []
+    all_y_vals = []
+    for series in data.values():
+        all_x_vals.extend(series["x"])
+        all_y_vals.extend(series["y"])
+    if not all_x_vals or not all_y_vals:
+        return (0, 1), (0, 1)
+    x_min, x_max = min(all_x_vals), max(all_x_vals)
+    y_min, y_max = min(all_y_vals), max(all_y_vals)
+    x_pad = max((x_max - x_min) * PADDING_RATIO, 1.0)
+    y_pad = max((y_max - y_min) * PADDING_RATIO, 1.0)
+    padded_x = (x_min - x_pad, x_max + x_pad)
+    padded_y = (y_min - y_pad, y_max + y_pad)
+    return padded_x, padded_y
+def run_gui(buffer: PlotBuffer, update_interval_ms=1000):
+    """
+    Runs a matplotlib live updating plot based on the PlotBuffer content.
+    `update_interval_ms` controls how often the plot refreshes (default 1000ms = 1s).
+    """
+    # plt.style.use('seaborn-darkgrid')
+    plt.style.use('ggplot')  # matplotlib built-in style as a lightweight alternative
+    fig, ax = plt.subplots(figsize=(10, 6))
+    ax.set_title("Live Pipeline Data")
+    ax.set_xlabel("Time")
+    ax.set_ylabel("Value")
+    lines = {}
+    legend_labels = []
+    def init():
+        ax.clear()
+        ax.set_title("Live Pipeline Data")
+        ax.set_xlabel("Time")
+        ax.set_ylabel("Value")
+        return []
+    def update(frame):
+        data = buffer.get_all()
+        if not data:
+            return []
+        # Add/update lines for each series
+        for label, series in data.items():
+            x_vals = series["x"]
+            y_vals = series["y"]
+            # Decide how many ticks you want (e.g., max 6)
+            num_ticks = min(6, len(x_vals))
+            # Choose evenly spaced indices
+            indices = np.linspace(0, len(x_vals) - 1, num_ticks, dtype=int)
+            if label not in lines:
+                # Create new line
+                line, = ax.plot(x_vals, y_vals, label=label)
+                lines[label] = line
+                legend_labels.append(label)
+                ax.legend()
+            else:
+                lines[label].set_data(x_vals, y_vals)
+        # Adjust axes limits with padding
+        padded_x, padded_y = compute_padded_bounds(data)
+        ax.set_xlim(padded_x)
+        ax.set_ylim(padded_y)
+        # Format x-axis ticks as human readable time strings
+        # Tick positions are x values at those indices
+        #tick_positions = x_vals[indices]
+        tick_positions = np.array(x_vals)[indices]
+        tick_labels = [TimeManager(ts).as_formatted_time() for ts in tick_positions]
+        # Convert UNIX timestamps to formatted strings on x-axis
+        #xticks = ax.get_xticks()
+        #xtick_labels = [TimeManager(x).as_formatted_time() for x in xticks]
+        ax.set_xticks(tick_positions)
+        #ax.set_xticklabels(xtick_labels, rotation=45, ha='right')
+        ax.set_xticklabels(tick_labels, rotation=45, ha='right')
+        return list(lines.values())
+    ani = animation.FuncAnimation(
+        fig,
+        update,
+        init_func=init,
+        interval=update_interval_ms,
+        blit=False  # blit=True can be tricky with multiple lines and dynamic axes
+    )
+    plt.tight_layout()
+    plt.show()

pipeline/helpers.py ADDED Viewed

@@ -0,0 +1,125 @@
+import json
+import toml
+from datetime import datetime
+import inspect
+import types
+import os
+import logging
+import socket
+from src.pipeline.time_manager import TimeManager
+logger = logging.getLogger(__name__)
+def load_json(filepath):
+    if not os.path.exists(filepath):
+        logger.warning(f"[load_json] File not found: {filepath}")
+        return {}
+    if os.path.getsize(filepath) == 0:
+        logger.warning(f"[load_json] File is empty: {filepath}")
+        return {}
+    try:
+        with open(filepath, 'r') as file:
+            return json.load(file)
+    except json.JSONDecodeError as e:
+        logger.error(f"[load_json] Failed to decode JSON in {filepath}: {e}")
+        return {}
+def load_toml(filepath):
+    # Load TOML data from the file
+    with open(filepath, 'r') as f:
+        dic_toml = toml.load(f)
+    return dic_toml
+#def round_datetime_to_nearest_past_five_minutes(dt: datetime) -> datetime:
+def round_datetime_to_nearest_past_five_minutes(dt):
+    #print(f"dt = {dt}")
+    allowed_minutes = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
+    # Find the largest allowed minute <= current minute
+    rounded_minute = max(m for m in allowed_minutes if m <= dt.minute)
+    return dt.replace(minute=rounded_minute, second=0, microsecond=0)
+def get_now_time_rounded(workspace_manager):# -> int:
+    '''
+    workspace_manager is included here so that references can be made to the configured timezone
+    '''
+    logger.debug(f"helpers.get_now_time_rounded(workspace_manager)")
+    nowtime = round_datetime_to_nearest_past_five_minutes(datetime.now())
+    logger.debug(f"rounded nowtime = {nowtime}")
+    nowtime_local =  int(nowtime.timestamp())+300
+    nowtime_local = TimeManager(nowtime_local).as_datetime()
+    if False:
+        try:
+            config = load_toml(workspace_manager.get_configuration_file_path())
+            timezone_config = config["settings"]["timezone"]
+        except:
+            timezone_config = "America/Chicago"
+        nowtime_utc = TimeManager.from_local(nowtime_local, zone_name = timezone_config).as_unix()
+        logger.debug(f"return nowtime_utc")
+        return nowtime_utc
+    else:
+        logger.debug(f"return nowtime_local")
+        return TimeManager(nowtime_local).as_unix() # nowtime_utc
+def function_view(globals_passed=None):
+    # Use the calling frame to get info about the *caller* module
+    caller_frame = inspect.stack()[1].frame
+    if globals_passed is None:
+        globals_passed = caller_frame.f_globals
+    # Get filename → basename only (e.g., 'calls.py')
+    filename = os.path.basename(caller_frame.f_code.co_filename)
+    print(f"Functions defined in {filename}:")
+    for name, obj in list(globals_passed.items()):
+        if isinstance(obj, types.FunctionType):
+            if getattr(obj, "__module__", None) == globals_passed.get('__name__', ''):
+                print(f"  {name}")
+    print("\n")
+#def get_nested_config(dct: dict, keys: list[str]):
+def get_nested_config(dct, keys):
+    """Retrieve nested dict value by keys list; raise KeyError with full path if missing."""
+    current = dct
+    for key in keys:
+        try:
+            current = current[key]
+        except KeyError as e:
+            full_path = " -> ".join(keys)
+            raise KeyError(f"Missing required configuration at path: {full_path}") from e
+    return current
+def human_readable(ts):
+    return datetime.fromtimestamp(ts).strftime("%H:%M:%S")
+def iso(ts):
+    return datetime.fromtimestamp(ts).isoformat()
+def get_lan_ip_address_of_current_machine():
+    """Get the LAN IP address of the current machine."""
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        # Doesn't need to be reachable; just picks the active interface
+        s.connect(("8.8.8.8", 80))
+        return s.getsockname()[0]
+    finally:
+        s.close()
+def nice_step(delta_sec: int) -> int:
+    """
+    Return a "nice" step in seconds (1,2,5,10,15,30,60,120,...)
+    """
+    nice_numbers = [1, 2, 5, 10, 15, 30, 60, 120, 300, 600, 900, 1800, 3600, 7200, 14400, 28800, 86400]
+    target_step = delta_sec // 400  # aim for ~400 points
+    # find the smallest nice_number >= target_step
+    for n in nice_numbers:
+        if n >= target_step:
+            return n
+    return nice_numbers[-1]
+if __name__ == "__main__":
+    function_view()

pipeline/logging_setup.py ADDED Viewed

@@ -0,0 +1,45 @@
+import os
+import json
+import logging.config
+def setup_logging(config_file="config/logging.json"): # relative to root
+    if not os.path.exists("logs"):
+        os.makedirs("logs")
+    with open(config_file, 'r') as f:
+        config = json.load(f)
+        logging.config.dictConfig(config)
+# Dynamically set logging level, per handler, like when using a GUI
+#def set_handler_level(handler_name: str, level_name: str):
+def set_handler_level(handler_name, level_name):
+    logger = logging.getLogger()  # root logger
+    level = getattr(logging, level_name.upper(), None)
+    if level is None:
+        raise ValueError(f"Invalid log level: {level_name}")
+    for handler in logger.handlers:
+        if handler.get_name() == handler_name:
+            handler.setLevel(level)
+            logging.info(f"Set {handler_name} handler level to {level_name}")
+            break
+    else:
+        raise ValueError(f"Handler '{handler_name}' not found")
+# Custom JSON formatter
+class JSONFormatter(logging.Formatter):
+    def format(self, record):
+        log_record = {
+            "time": self.formatTime(record, "%Y-%m-%d %H:%M:%S"),
+            "level": record.levelname,
+            "message": record.getMessage(),
+        }
+        return json.dumps(log_record, indent=4)
+class PrettyJSONFormatter(logging.Formatter):
+    def format(self, record):
+        if isinstance(record.msg, (dict, list)):
+            try:
+                record.msg = json.dumps(record.msg, indent=2, ensure_ascii=False)
+            except Exception:
+                pass  # fallback to default formatting
+        return super().format(record)

pipeline/pastehelpers.py ADDED Viewed

@@ -0,0 +1,10 @@
+'''
+title: pastehelpers.py
+author: Clayton Bennett
+created: 30 July 2025
+why: These functions will not be useful if imported. You'll need to paste the meat.
+'''
+import inspect
+def current_function_name():
+    return inspect.currentframe().f_code.co_name

pipeline/philosophy.py ADDED Viewed

@@ -0,0 +1,62 @@
+class Philosophy:
+    """
+    A playful class embodying the philosophy of Python's special (dunder) methods.
+    Use this to reflect on Python design or teach how dunders work.
+    """
+    def __new__(cls, *args, **kwargs):
+        print("Creating a new philosophical instance...")
+        return super().__new__(cls)
+    def __init__(self):
+        self.truth = 42
+        self.meaning = "Pythonic reflection"
+    def __repr__(self):
+        return '"__str__ is for users. __repr__ is for developers."'
+    def __str__(self):
+        return "Zen of Python: Beautiful is better than ugly."
+    def __bytes__(self):
+        return b"In the face of ambiguity, refuse the temptation to guess."
+    def __bool__(self):
+        return True  # Philosophy is always truthy
+    def __len__(self):
+        return len(self.meaning)
+    def __getitem__(self, key):
+        return f"Indexing into philosophy returns insight[{key}]"
+    def __setitem__(self, key, value):
+        print(f"Attempting to assign {value!r} to insight[{key}]... but wisdom is immutable.")
+    def __call__(self, question):
+        return f"You asked: {question!r}. Python answers with clarity."
+    def __eq__(self, other):
+        return isinstance(other, Philosophy)
+    def __add__(self, other):
+        return "Philosophy enriched with " + str(other)
+    def __contains__(self, item):
+        return item.lower() in self.meaning.lower()
+    def __iter__(self):
+        yield from self.meaning.split()
+    def __enter__(self):
+        print("Entering a context of enlightenment.")
+        return self
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        print("Exiting the context. Wisdom retained.")
+    def __del__(self):
+        print("A philosophical idea vanishes... but not forgotten.")
+    def __hash__(self):
+        return hash((self.truth, self.meaning))

pipeline/plotbuffer.py ADDED Viewed

@@ -0,0 +1,21 @@
+# src/pipeline/plotbuffer.py
+from collections import defaultdict
+KEEP_ALL_LIVE_POINTS = True
+class PlotBuffer:
+    def __init__(self, max_points=100):
+        self.data = defaultdict(lambda: {"x": [], "y": []})
+        self.max_points = max_points
+    def append(self, label, x, y):
+        self.data[label]["x"].append(x)
+        self.data[label]["y"].append(y)
+        if len(self.data[label]["x"]) > self.max_points:
+            if not KEEP_ALL_LIVE_POINTS:
+                self.data[label]["x"].pop(0)
+                self.data[label]["y"].pop(0)
+    def get_all(self):
+        return self.data

pipeline/points_loader.py ADDED Viewed

@@ -0,0 +1,19 @@
+# src/points_loader.py
+import csv
+class PointsCsvLoader:
+    """
+    loader = PointsCsvLoader(csv_path)
+    points_list = loader.load_points()
+    """
+    def __init__(self, csv_file_path: str):
+        self.csv_file_path = csv_file_path
+        self.points = []
+    def load_points(self):
+        with open(self.csv_file_path, mode='r', newline='') as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                self.points.append(row)
+        return self.points

pipeline/queriesmanager.py ADDED Viewed

@@ -0,0 +1,122 @@
+import os
+import toml
+from datetime import datetime
+import json
+import csv
+from collections import defaultdict
+import logging
+from src.pipeline import helpers
+from src.pipeline.time_manager import TimeManager
+logger = logging.getLogger(__name__)
+'''
+Goal:
+Set up to use the most recent query:
+use-most-recently-edited-query-file = true # while true, this will ignore the files variable list and instead use a single list of the most recent files
+'''
+class QueriesManager:
+    #def __init__(self, workspace_manager: object):
+    def __init__(self, workspace_manager):
+        self.workspace_manager = workspace_manager
+        logger.info(f"QueriesManager using project: {self.workspace_manager.workspace_name}")
+        if not workspace_manager:
+            raise ValueError("workspace_manager must be provided and not None.")
+        self.workspace_manager = workspace_manager
+    def load_tracking(self):
+        file_path = self.workspace_manager.get_timestamp_success_file_path()
+        try:
+            #logger.info({"Trying to load tracking file at": file_path})
+            logger.debug({
+                "event": "Loading tracking file",
+                "path": str(file_path)
+            })
+            data = helpers.load_json(file_path)
+            #logger.info({"Tracking data loaded": data})
+            logger.debug({
+                "event": "Tracking data loaded",
+                "data": data
+            })
+            return data
+        except FileNotFoundError:
+            return {}
+    def save_tracking(self,data):
+        file_path = self.workspace_manager.get_timestamp_success_file_path()
+        with open(file_path, 'w') as f:
+            json.dump(data, f, indent=2)
+    def get_most_recent_successful_timestamp(self, api_id):# -> int:
+        print("QueriesManager.get_most_recent_successful_timestamp()")
+        from src.pipeline.helpers import load_toml
+        try:
+            config = load_toml(self.workspace_manager.get_configuration_file_path())
+            timezone_config = config["settings"]["timezone"]
+        except:
+            timezone_config = "America/Chicago"
+        data = self.load_tracking()
+        if not data:
+            # No stored value found — go back one hour from now, rounded down to nearest 5 minutes
+            one_hour_ago_local = TimeManager.now().as_unix() - 3600  # now - 1 hour in unix seconds
+            one_hour_ago_local = TimeManager(one_hour_ago_local).as_datetime()
+            #one_hour_ago_utc = TimeManager.from_local(one_hour_ago_local, zone_name = timezone_config)
+            tm = TimeManager(one_hour_ago_local).round_down_to_nearest_five()
+        else:
+            # Stored value found — parse ISO timestamp and round down to nearest 5 minutes
+            last_success_iso = TimeManager(data[api_id]["timestamps"]["last_success"]).as_datetime()
+            #last_success_utc = TimeManager.from_local(last_success_iso, zone_name = timezone_config).as_datetime()
+            tm = TimeManager(last_success_iso).round_down_to_nearest_five()
+        return tm
+    def update_success(self,api_id,success_time=None):
+        # This should be called when data is definitely transmitted to the target API.
+        # A confirmation algorithm might be in order, like calling back the data and checking it against the original.
+        data = self.load_tracking()
+        if api_id not in data:
+            data[api_id] = {"timestamps": {}}
+        #now = success_time or datetime.now().isoformat()
+        now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        data[api_id]["timestamps"]["last_success"] = now
+        data[api_id]["timestamps"]["last_attempt"] = now
+        self.save_tracking(data)
+    def update_attempt(self, api_id):
+        data = self.load_tracking()
+        if api_id not in data:
+            logger.info(f"Creating new tracking entry for {api_id}")
+            data[api_id] = {"timestamps": {}}
+        #now = datetime.now().isoformat()
+        now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        data[api_id]["timestamps"]["last_attempt"] = now
+        self.save_tracking(data)
+        logger.info(f"Updated last_attempt for {api_id}: {now}")
+def load_query_rows_from_csv_files(csv_paths_list):
+    queries_dictlist_unfiltered = []
+    for csv_path in csv_paths_list:
+        with open(csv_path, newline='') as csvfile:
+            reader = csv.DictReader(csvfile)
+            for row in reader:
+                queries_dictlist_unfiltered.append(row)
+    return queries_dictlist_unfiltered
+def group_queries_by_col(queries_array,grouping_var_str='zd'):
+    queries_array_grouped = defaultdict(list)
+    for row in queries_array:
+        row_filter = row[grouping_var_str]
+        queries_array_grouped[row_filter].append(row)
+    return queries_array_grouped
+if __name__ ==  "__main__":
+    pass