ltc-client 0.2__tar.gz

ltc_client-0.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Tin Arm Engineering Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ltc_client-0.2/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.1
+Name: ltc-client
+Version: 0.2
+Summary: "Client and Worker module for Tin Arm Engineering LTC service"
+Home-page: https://github.com/TinArmEngineering/ltc_client/
+License: MIT
+Author: Martin West
+Author-email: martin@tinarmengineering.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: Pint (>=0.24,<0.25)
+Requires-Dist: numpy (>=2.0.0,<3.0.0)
+Requires-Dist: pika (>=1.3.2,<2.0.0)
+Requires-Dist: python-logging-rabbitmq (>=2.3.0,<3.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: tqdm (>=4.66.4,<5.0.0)
+Requires-Dist: webstompy (>=0.1.3,<0.2.0)
+Project-URL: Repository, https://github.com/TinArmEngineering/ltc_client/
+Description-Content-Type: text/markdown
+
+# ltc_client
+Node creation tool for TAE workers
+

ltc_client-0.2/README.md

@@ -0,0 +1,2 @@
+# ltc_client
+Node creation tool for TAE workers

ltc_client-0.2/ltc_client/__init__.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+# from ltc_client.worker import StandardWorker, DefaultIdLogFilter, HostnameFilter
+from ltc_client.api import Api, NameQuantityPair, Quantity, Unit
+from ltc_client.helpers import Machine, Job, decode
+
+__title__ = "TINARM - Node creation tool for TAE workers"
+__version__ = "0.1"
+__author__ = "Martin West, Chris Wallis"
+__license__ = "MIT License"
+__copyright__ = "Copyright 2023 Tin Arm Engineering Ltd."

ltc_client-0.2/ltc_client/api.py

@@ -0,0 +1,355 @@
+import logging
+import time
+import requests
+from math import prod
+import pint
+
+LOGGING_LEVEL = logging.INFO
+
+JOB_STATUS = {
+    "New": 0,
+    "QueuedForMeshing": 10,
+    "WaitingForMesh": 20,
+    "QueuedForSimSetup": 21,
+    "SimSetup": 22,
+    "QueuedForMeshConversion": 25,
+    "MeshConversion": 26,
+    "QueuedForSolving": 30,
+    "Solving": 40,
+    "QueuedForPostProcess": 50,
+    "PostProcess": 60,
+    "Complete": 70,
+    "Quarantined": 80,
+}
+
+STATUS_JOB = {value: key for key, value in JOB_STATUS.items()}
+
+### Configure Logging
+logger = logging.getLogger()
+logger.setLevel(LOGGING_LEVEL)
+
+
+class Unit:
+    def __init__(self, name: str, exponent: int):
+        self.name = name
+        self.exponent = exponent
+
+    def to_dict(self):
+        return {
+            "name": self.name,
+            "exponent": self.exponent,
+        }
+
+
+class Quantity:
+    """
+    Represents a quantity with magnitude, units, and shape.
+
+    Args:
+        magnitude: The magnitude of the quantity. It can be a single value, a list-like object, or a numpy array.
+        or even a pint.Quantity
+        units (list[Unit]): A list of Unit objects representing the units of the quantity.
+        shape (Optional): The shape of the quantity. If not provided, it will be inferred from the magnitude.
+
+    Attributes:
+        magnitude: The magnitude of the quantity.
+        shape: The shape of the quantity.
+        units (list[Unit]): A list of Unit objects representing the units of the quantity.
+    """
+
+    def __init__(self, magnitude, units=[None], shape=None):
+        if isinstance(magnitude, pint.Quantity):
+            magnitude, units = magnitude.to_tuple()
+        if hasattr(magnitude, "shape"):
+            if shape is None:
+                self.shape = list(magnitude.shape)
+                self.magnitude = magnitude.flatten().tolist()
+            elif prod(shape) == magnitude.size:
+                self.magnitude = magnitude.tolist()
+                self.shape = shape
+            else:
+                raise ValueError(
+                    f"Shape {shape} does not match magnitude size {magnitude.size}"
+                )
+
+        elif not hasattr(magnitude, "__len__"):
+            self.magnitude = [magnitude]
+            self.shape = [1]
+        elif shape is None:
+            self.shape = [len(magnitude)]
+            self.magnitude = magnitude
+        elif prod(shape) != len(magnitude):
+            raise ValueError(
+                f"Shape {shape} does not match magnitude size {len(magnitude)}"
+            )
+        else:
+            self.magnitude = magnitude
+            self.shape = shape
+
+        self.units = [Unit(*u) if type(u) != Unit else u for u in units]
+
+    def to_dict(self):
+        """
+        Converts the Quantity object to a dictionary.
+
+        Returns:
+            dict: A dictionary representation of the Quantity object.
+        """
+        return {
+            "magnitude": self.magnitude,
+            "shape": self.shape,
+            "units": [u.to_dict() for u in self.units],
+        }
+
+
+class NameQuantityPair:
+    def __init__(self, section, name, value: Quantity):
+        self.section = section
+        self.name = name
+        self.value = value
+
+    def to_dict(self):
+        return {
+            "section": self.section,
+            "name": self.name,
+            "value": self.value.to_dict(),
+        }
+
+
+class Api:
+    """
+    The TAE API
+    """
+
+    def __init__(self, root_url, api_key, org_id=None, node_id=None):
+        """
+        Initialize the API
+        """
+
+        self._root_url = root_url
+        self._api_key = api_key
+        self._org_id = org_id
+        self._node_id = node_id
+
+        logger.info(f"root_url: {self._root_url}")
+
+    def get_job(self, job_id):
+        """
+        Get a job from the TAE API
+        """
+        response = requests.get(
+            url=f"{self._root_url}/jobs/{job_id}?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def create_job(self, job):
+        """
+        Create a job for the TAE API
+        """
+        response = requests.post(
+            url=f"{self._root_url}/jobs/?apikey={self._api_key}&org_id={self._org_id}",
+            json=job.to_api(),
+        )
+        response.raise_for_status()
+        if response.status_code == 200:
+            job.id = response.json()["id"]
+        return response.json()
+
+    def update_job_status(self, job_id, status, percentage_complete=None):
+        """
+        Update a job status
+        """
+        url = f"{self._root_url}/jobs/{job_id}/status/{status}?node_id={self._node_id}&apikey={self._api_key}&percentage_complete={percentage_complete}"
+        logger.info(f"Updating job status: {url}")
+
+        response = requests.put(url=url)
+        response.raise_for_status()
+        return response.json()
+
+    def get_job_artifact(self, job_id, artifact_id):
+        """
+        Get job artifact
+        """
+        job = self.get_job(job_id)
+        for artifact in job["artifacts"]:
+            if artifact["id"] == artifact_id:
+                return artifact
+
+        raise Exception(f"Artifact {artifact_id} not found on job {job_id}")
+
+    def get_promoted_job_artifact(self, job_id, artifact_id):
+        # Get the artifact
+        artifact = self.get_job_artifact(job_id, artifact_id)
+
+        # If the url starts with https, it's already promoted
+        if artifact["url"].startswith("https"):
+            return artifact
+
+        for i in range(0, 10):
+            time.sleep(5)
+            artifact = self.get_job_artifact(job_id, artifact_id)
+            if artifact["url"].startswith("https"):
+                return artifact
+
+        raise Exception(
+            f"Artifact {artifact_id} on job {job_id} could not be promoted in a reasonable time"
+        )
+
+    def create_job_artifact(self, job_id, type, url, promote=False):
+        """
+        Post an artifact to a job
+        """
+        response = requests.post(
+            url=f"{self._root_url}/jobs/{job_id}/artifacts?promote={promote}&apikey={self._api_key}",
+            json={
+                "created_on_node": self._node_id,
+                "type": type,
+                "url": url,
+            },
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def create_job_artifact_from_file(self, job_id, type, filename, promote=False):
+        """
+        Post an artifact to a job
+        """
+        return self.create_job_artifact(
+            job_id, type, f"file://{self._node_id}{filename}", promote
+        )
+
+    def update_job_artifact(self, job_id, artifact_id, artifact):
+        """
+        Update an artifact
+        """
+        response = requests.put(
+            url=f"{self._root_url}/jobs/{job_id}/artifacts/{artifact_id}?apikey={self._api_key}",
+            json=artifact,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def promote_job_artifact(self, job_id, artifact_id):
+        """
+        Promote an artifact to a job
+        """
+        response = requests.put(
+            url=f"{self._root_url}/jobs/{job_id}/artifacts/{artifact_id}/promote?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def delete_job(self, job_id):
+        """
+        Delete a job
+        """
+        response = requests.delete(
+            url=f"{self._root_url}/jobs/{job_id}?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+        return
+
+    def create_job_data(self, job_id: str, data: NameQuantityPair):
+        """
+        Create job data
+        """
+        response = requests.post(
+            url=f"{self._root_url}/jobs/{job_id}/data?apikey={self._api_key}",
+            json=data.to_dict(),
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def update_job_data(self, job_id: str, data_name: str, data: NameQuantityPair):
+        """
+        Update job data
+        """
+        response = requests.put(
+            url=f"{self._root_url}/jobs/{job_id}/data/{data_name}?apikey={self._api_key}",
+            json=data.to_dict(),
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def delete_job_data(self, job_id: str, data_name: str):
+        """
+        Delete job data
+        """
+        response = requests.delete(
+            url=f"{self._root_url}/jobs/{job_id}/data/{data_name}?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+
+    def get_reusable_artifact(self, hash):
+        """
+        Get a reusable artifact from the TAE API
+        """
+        response = requests.get(
+            url=f"{self._root_url}/reusable_artifacts/{hash}?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def update_reusable_artifact(self, hash, reusable_artifact):
+        """
+        Update a reusable_artifact
+        """
+        response = requests.put(
+            url=f"{self._root_url}/reusable_artifacts/{hash}?apikey={self._api_key}",
+            json=reusable_artifact,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def update_reusable_artifact_url(self, hash, url, mimetype=None):
+        """
+        Update an reusable_artifact's URL
+        """
+        response = requests.patch(
+            url=f"{self._root_url}/reusable_artifacts/{hash}/url?apikey={self._api_key}",
+            json={"url": url, "mimetype": mimetype},
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def create_reusable_artifact_data(self, hash, data: NameQuantityPair):
+        """
+        Create reusable_artifact data
+        """
+        response = requests.post(
+            url=f"{self._root_url}/reusable_artifacts/{hash}/data?apikey={self._api_key}",
+            json=data.to_dict(),
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def promote_reusable_artifact(self, hash):
+        """
+        Promote reusable artifact
+        """
+        response = requests.put(
+            url=f"{self._root_url}/reusable_artifacts/{hash}/promote?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def get_material(self, material_id):
+        """
+        Get a material from the TAE API
+        """
+        response = requests.get(
+            url=f"{self._root_url}/materials/{material_id}?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def get_jobs(self):
+        """
+        Get all jobs
+        """
+        response = requests.get(
+            url=f"{self._root_url}/jobs?apikey={self._api_key}",
+        )
+        response.raise_for_status()
+        return response.json()

ltc_client-0.2/ltc_client/helpers.py

@@ -0,0 +1,239 @@
+from . import Quantity, NameQuantityPair
+from .api import JOB_STATUS, STATUS_JOB
+import random
+import requests
+import pint
+from webstompy import StompListener
+from tqdm.auto import tqdm
+import numpy as np
+import logging
+import uuid
+import asyncio
+
+logger = logging.getLogger(__name__)
+
+q = pint.get_application_registry()
+
+
+def decode(enc):
+    """Decode a quantity encoded object"""
+    if len(enc["magnitude"]) != 1:
+        enc_tuple = tuple(
+            (
+                np.array(enc["magnitude"], dtype=np.float64).reshape(enc["shape"]),
+                tuple((e["name"], e["exponent"]) for e in enc.get("units", ())),
+            )
+        )
+    else:
+        enc_tuple = (
+            enc["magnitude"][0],
+            tuple((e["name"], e["exponent"]) for e in enc.get("units", ())),
+        )
+    try:
+        quant = q.Quantity.from_tuple(enc_tuple)
+        quant.ito_base_units()
+    except:
+        logger.error(
+            "Error decoding {0}".format(
+                (
+                    enc["magnitude"],
+                    ((e["name"], e["exponent"]) for e in enc.get("units", ())),
+                )
+            )
+        )
+        raise
+
+    logger.debug("convert {i} -> {o:~P}".format(o=quant, i=enc))
+    return quant
+
+
+class Machine(object):
+    def __init__(self, stator, rotor, winding, materials=None):
+
+        self.stator = stator
+        self.rotor = rotor
+        self.winding = winding
+        if materials is not None:
+            self.materials = materials
+        else:
+            self.materials = {
+                "rotor_lamination": "66018e5d1cd3bd0d3453646f",  # default M230-35A
+                "rotor_magnet": "66018e5b1cd3bd0d3453646c",  # default is N35UH
+                "rotor_air_L": "6602fb42c4a87c305481e8a6",
+                "rotor_air_R": "6602fb42c4a87c305481e8a6",
+                "rotor_banding": "6602fb42c4a87c305481e8a6",
+                "stator_lamination": "66018e5d1cd3bd0d3453646f",  # default M230-35A
+                "stator_slot_wedge": "6602fb7239bfdea291a25dd7",
+                "stator_slot_liner": "6602fb5166d3c6adaa8ebe8c",
+                "stator_slot_winding": "66018e5d1cd3bd0d34536470",
+                "stator_slot_potting": "6602fd41b8e866414fe983ec",
+            }
+
+    def __repr__(self) -> str:
+        return f"Machine({self.stator}, {self.rotor}, {self.winding})"
+
+    def to_api(self):
+        stator_api = [
+            NameQuantityPair("stator", k, Quantity(*self.stator[k].to_tuple()))
+            for k in self.stator
+        ]
+        rotor_api = [
+            NameQuantityPair("rotor", k, Quantity(*self.rotor[k].to_tuple()))
+            for k in self.rotor
+        ]
+        winding_api = [
+            NameQuantityPair("winding", k, Quantity(*self.winding[k].to_tuple()))
+            for k in self.winding
+        ]
+        data = []
+        data.extend(list(x.to_dict() for x in stator_api))
+        data.extend(list(x.to_dict() for x in rotor_api))
+        data.extend(list(x.to_dict() for x in winding_api))
+        return data
+
+
+class Job(object):
+    def __init__(self, machine: Machine, operating_point, simulation, title=None):
+        if title is None:
+            self.title = self.generate_title()
+        else:
+            self.title = title
+        self.type = "electromagnetic_spmbrl_fscwseg"
+        self.status = 0
+        self.machine = machine
+        self.operating_point = operating_point
+        self.simulation = simulation
+
+    def __repr__(self) -> str:
+        return f"Job({self.machine}, {self.operating_point}, {self.simulation})"
+
+    def generate_title(self):
+        "gets a random title from the wordlists"
+        random_offset = random.randint(500, 286797)
+        response = requests.get(
+            "https://github.com/taikuukaits/SimpleWordlists/raw/master/Wordlist-Adjectives-All.txt",
+            headers={
+                "Range": "bytes={1}-{0}".format(random_offset, random_offset - 500),
+                "accept-encoding": "identity",
+            },
+        )
+        adjective = random.choice(response.text.splitlines()[1:-1])
+        random_offset = random.randint(500, 871742)
+        response = requests.get(
+            "https://github.com/taikuukaits/SimpleWordlists/raw/master/Wordlist-Nouns-All.txt",
+            headers={
+                "Range": "bytes={1}-{0}".format(random_offset, random_offset - 500),
+                "accept-encoding": "identity",
+            },
+        )
+        noun = random.choice(response.text.splitlines()[1:-1])
+        title = f"{adjective}-{noun}"
+        return title
+
+    def to_api(self):
+        job = {
+            "status": 0,
+            "title": self.title,
+            "type": self.type,
+            "tasks": 11,
+            "data": [],
+            "materials": [],
+        }
+
+        operating_point_api = [
+            NameQuantityPair(
+                "operating_point", k, Quantity(*self.operating_point[k].to_tuple())
+            )
+            for k in self.operating_point
+        ]
+
+        simulation_api = [
+            NameQuantityPair("simulation", k, Quantity(*self.simulation[k].to_tuple()))
+            for k in self.simulation
+        ]
+
+        job["data"].extend(list(x.to_dict() for x in operating_point_api))
+        job["data"].extend(list(x.to_dict() for x in simulation_api))
+        job["data"].extend(self.machine.to_api())
+        job["materials"] = [
+            {"part": key, "material_id": value}
+            for key, value in self.machine.materials.items()
+        ]
+        return job
+
+    def run(self):
+        pass
+
+
+class TqdmUpTo(tqdm):
+    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
+
+    def update_to(self, b=1, bsize=1, tsize=None):
+        """
+        b  : int, optional
+            Number of blocks transferred so far [default: 1].
+        bsize  : int, optional
+            Size of each block (in tqdm units) [default: 1].
+        tsize  : int, optional
+            Total size (in tqdm units). If [default: None] remains unchanged.
+        """
+        if tsize is not None:
+            self.total = tsize
+        return self.update(b * bsize - self.n)  # also sets self.n = b * bsize
+
+
+class MyListener(StompListener):
+    def __init__(self, job, uid):
+        self.job_id = job.id
+        self.uid = uid
+        self.done = False
+        self._callback_fn = None  # Initialize the callback function
+
+    @property
+    def callback_fn(self):
+        return self._callback_fn
+
+    @callback_fn.setter
+    def callback_fn(self, fn):
+        self._callback_fn = fn
+
+        # Rest of the code...
+
+    def on_message(self, frame):
+        headers = {key.decode(): value.decode() for key, value in frame.header}
+        if headers["subscription"] == self.uid:
+            try:
+                time_str, level_str, mesg_str = frame.message.decode().split(" - ")
+            except ValueError:
+                logger.warning(frame)
+
+            else:
+                line = [part.strip() for part in mesg_str.strip().split(":")]
+                if line[1] == "Time":
+                    done, total = (int(part) for part in line[2].split("/"))
+                    self.callback_fn(done, tsize=total)
+                    if done == total:
+                        self.done = True
+                        return self.done
+        else:
+            return
+
+
+async def async_job_monitor(api, my_job, connection, position):
+    uid = str(uuid.uuid4())
+    listener = MyListener(my_job, uid)
+    connection.add_listener(listener)
+    connection.subscribe(destination=f"/topic/{my_job.id}.solver.*.progress", id=uid)
+    with TqdmUpTo(
+        total=my_job.simulation["timestep_intervals"],
+        desc=f"Job {my_job.title}",
+        position=position,
+        leave=False,
+    ) as pbar:
+        listener.callback_fn = pbar.update_to
+
+        j1_result = api.update_job_status(my_job.id, JOB_STATUS["QueuedForMeshing"])
+
+        while not listener.done:
+            await asyncio.sleep(1)  # sleep for a second
+        return STATUS_JOB[api.get_job(my_job.id)["status"]]

ltc_client-0.2/ltc_client/worker.py

@@ -0,0 +1,393 @@
+import functools
+import json
+import logging
+import os
+import pika
+from socket import gaierror
+import platform
+import ssl
+import sys
+import threading
+import time
+
+from pathlib import Path
+from python_logging_rabbitmq import RabbitMQHandler
+
+from ltc_client.api import Api
+
+RABBIT_DEFAULT_PRE_FETCH_COUNT = 1
+RABBIT_FIRST_WAIT_BEFORE_RERTY_SECS = 0.5
+RABBIT_MAX_WAIT_BEFORE_RERTY_SECS = 64
+LOGGING_LEVEL = logging.INFO
+
+
+### Configure Logging
+logger = logging.getLogger()  # get the root logger?
+logger.setLevel(LOGGING_LEVEL)
+tld = threading.local()
+tld.job_id = "NoJobId"
+
+
+class HostnameFilter(logging.Filter):
+    """Used for logging the hostname
+    https://stackoverflow.com/a/55584223/20882432
+    """
+
+    hostname = platform.node()
+
+    def filter(self, record):
+        record.hostname = HostnameFilter.hostname
+        return True
+
+
+class DefaultIdLogFilter(logging.Filter):
+    """Used for logging the job id"""
+
+    def filter(self, record):
+        if not hasattr(tld, "job_id"):
+            record.id = "NoJobId"
+        else:
+            record.id = tld.job_id
+        return True
+
+
+def addLoggingLevel(levelName: str, levelNum: int, methodName: str = None) -> None:
+    """
+    Comprehensively adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    `levelName` becomes an attribute of the `logging` module with the value
+    `levelNum`. `methodName` becomes a convenience method for both `logging`
+    itself and the class returned by `logging.getLoggerClass()` (usually just
+    `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is
+    used.
+
+    To avoid accidental clobberings of existing attributes, this method will
+    raise an `AttributeError` if the level name is already an attribute of the
+    `logging` module or if the method name is already present
+
+    Example
+    -------
+    >>> addLoggingLevel('TRACE', logging.DEBUG - 5)
+    >>> logging.getLogger(__name__).setLevel("TRACE")
+    >>> logging.getLogger(__name__).trace('that worked')
+    >>> logging.trace('so did this')
+    >>> logging.TRACE
+    5
+
+    """
+    if not methodName:
+        methodName = levelName.lower()
+
+    if hasattr(logging, levelName):
+        raise AttributeError("{} already defined in logging module".format(levelName))
+    if hasattr(logging, methodName):
+        raise AttributeError("{} already defined in logging module".format(methodName))
+    if hasattr(logging.getLoggerClass(), methodName):
+        raise AttributeError("{} already defined in logger class".format(methodName))
+
+    # raise a value error if the level number is not an integer
+    if not isinstance(levelNum, int):
+        raise ValueError("levelNum must be an integer")
+
+    # This method was inspired by the answers to Stack Overflow post
+    # http://stackoverflow.com/q/2183233/2988730, especially
+    # http://stackoverflow.com/a/13638084/2988730
+    def logForLevel(self, message: str, *args, **kwargs) -> None:
+        if self.isEnabledFor(levelNum):
+            self._log(levelNum, message, args, **kwargs)
+
+    def logToRoot(message: str, *args, **kwargs) -> None:
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
+
+
+addLoggingLevel("PROGRESS", logging.INFO + 2)
+
+stream_handler = logging.StreamHandler(stream=sys.stdout)
+stream_handler.addFilter(HostnameFilter())
+stream_handler.addFilter(DefaultIdLogFilter())
+stream_handler.setFormatter(
+    logging.Formatter(
+        "%(asctime)s - %(id)s - %(levelname)s - %(hostname)s - %(filename)s->%(funcName)s() - %(message)s"
+    )
+)
+
+logger.addHandler(stream_handler)
+
+
+class StandardWorker:
+    """
+    The standard TAE worker class
+    """
+
+    def __init__(
+        self,
+        node_id,
+        worker_name,
+        queue_host,
+        queue_port,
+        queue_user,
+        queue_password,
+        queue_use_ssl,
+        queue_exchange,
+        queue_prefetch_count=RABBIT_DEFAULT_PRE_FETCH_COUNT,
+        x_priority=0,
+        projects_path=os.getenv("PROJECTS_PATH"),
+    ):
+        self._threads = []
+        self._node_id = node_id
+        self._worker_name = worker_name
+        self._exchange = queue_exchange
+        self._x_priority = x_priority
+        self._projects_path = projects_path
+        self._send_log_as_artifact = True
+
+        if queue_use_ssl:
+            ssl_options = pika.SSLOptions(context=ssl.create_default_context())
+        else:
+            ssl_options = None
+
+        self._connection = _rabbitmq_connect(
+            node_id,
+            worker_name,
+            queue_host,
+            queue_port,
+            queue_user,
+            queue_password,
+            ssl_options,
+        )
+
+        self._channel = self._connection.channel()
+        self._channel.basic_qos(prefetch_count=queue_prefetch_count, global_qos=False)
+        self._channel.exchange_declare(
+            exchange=queue_exchange, exchange_type="topic", durable=True
+        )
+
+        rabbit_handler = RabbitMQHandler(
+            host=queue_host,
+            port=queue_port,
+            username=queue_user,
+            password=queue_password,
+            connection_params={"ssl_options": ssl_options},
+            exchange="amq.topic",
+            declare_exchange=True,
+            routing_key_formatter=lambda r: (
+                "{jobid}.{worker_name}.{type}.{level}".format(
+                    jobid=r.id,
+                    worker_name=worker_name,
+                    type="python",
+                    level=r.levelname.lower(),
+                )
+            ),
+        )
+
+        rabbit_handler.addFilter(HostnameFilter())
+        rabbit_handler.addFilter(DefaultIdLogFilter())
+        rabbit_handler.setFormatter(
+            logging.Formatter(
+                "%(asctime)s - %(levelname)s  - %(message)s", datefmt="%H:%M:%S"
+            )
+        )
+
+        logger.addHandler(rabbit_handler)
+
+    def bind(self, queue, routing_key, func):
+        ch = self._channel
+
+        ch.queue_declare(
+            queue=queue,
+            durable=True,
+            exclusive=False,
+        )
+        ch.queue_bind(exchange=self._exchange, queue=queue, routing_key=routing_key)
+
+        # If func was provided, register the callback
+        if func is not None:
+            ch.basic_consume(
+                queue=queue,
+                on_message_callback=functools.partial(
+                    self._threaded_callback,
+                    args=(func, self._connection, ch, self._threads),
+                ),
+                arguments={"x-priority": self._x_priority},
+            )
+
+        logger.info(f"Declare::Bind, Q::RK, {queue}::{routing_key}")
+
+    def start(self):
+        try:
+            logger.info("Starting to consume messages")
+            self._channel.start_consuming()
+        except KeyboardInterrupt:
+            logger.info("Stopping consuming ...")
+            self._channel.stop_consuming()
+            logger.info("Stopped consuming messages")
+
+        # Wait for all to complete
+        for thread in self._threads:
+            thread.join()
+
+        # Close connection
+        self._connection.close()
+
+    def queue_message(self, routing_key, body):
+        _rabbitmq_queue_message(self._channel, self._exchange, routing_key, body)
+
+    def _threaded_callback(self, ch, method_frame, _header_frame, body, args):
+        (func, conn, ch, thrds) = args
+        delivery_tag = method_frame.delivery_tag
+        t = threading.Thread(
+            target=self._do_threaded_callback,
+            args=(conn, ch, delivery_tag, func, body),
+        )
+        t.start()
+        thrds.append(t)
+        logger.info(
+            "Thread count: %i of which %i active", len(thrds), threading.active_count()
+        )
+
+    def _do_threaded_callback(self, conn, ch, delivery_tag, func, body):
+
+        thread_id = threading.get_ident()
+        payload = json.loads(body.decode())
+        tld.job_id = payload["id"]
+
+        api_root = os.getenv("API_ROOT_URL")
+        api_key = payload.get("apikey", None)
+
+        can_send_log_as_artifact = self._send_log_as_artifact and api_root and api_key
+
+        job_log_directory = f"{self._projects_path}/jobs/{tld.job_id}"
+        job_log_filename = f"{job_log_directory}/{self._worker_name}.log"
+
+        if can_send_log_as_artifact:
+
+            # Emsure the job directory exists
+            Path(job_log_directory).mkdir(parents=True, exist_ok=True)
+
+            # Set up the log file handler for this job
+            file_handler = logging.FileHandler(filename=job_log_filename, mode="a")
+            file_handler.addFilter(HostnameFilter())
+            file_handler.addFilter(DefaultIdLogFilter())
+            file_handler.setFormatter(
+                logging.Formatter(
+                    "%(asctime)s - %(id)s - %(levelname)s - %(hostname)s - %(filename)s->%(funcName)s() - %(message)s"
+                )
+            )
+
+            logger.addHandler(file_handler)
+
+        logger.info(
+            "Thread id: %s Delivery tag: %s Message body: %s Job id: %s",
+            thread_id,
+            delivery_tag,
+            body,
+            tld.job_id,
+        )
+
+        next_routing_key, new_body = func(body)
+        if new_body is not None:
+            body = new_body
+        if next_routing_key is not None:
+            logger.info(f"next routing key: {next_routing_key}")
+            cbq = functools.partial(self.queue_message, next_routing_key, body)
+            conn.add_callback_threadsafe(cbq)
+
+        cb = functools.partial(_rabbitmq_ack_message, ch, delivery_tag)
+        conn.add_callback_threadsafe(cb)
+
+        if can_send_log_as_artifact:
+            logger.removeHandler(file_handler)
+            try:
+                logger.info("Creating artifact from job log")
+                api = Api(root_url=api_root, api_key=api_key, node_id=self._node_id)
+                api.create_job_artifact_from_file(
+                    tld.job_id, f"{self._worker_name}_log", job_log_filename
+                )
+            except Exception as e:
+                logger.error(f"Failed to create artifact from job log: {e}")
+
+
+def _rabbitmq_connect(node_id, worker_name, host, port, user, password, ssl_options):
+    client_properties = {
+        "connection_name": f"{node_id}-{worker_name}-{platform.node()}"
+    }
+
+    connection_params = pika.ConnectionParameters(
+        host=host,
+        port=port,
+        credentials=pika.PlainCredentials(user, password),
+        client_properties=client_properties,
+        heartbeat=10,
+        ssl_options=ssl_options,
+    )
+
+    sleepTime = RABBIT_FIRST_WAIT_BEFORE_RERTY_SECS
+    connected = False
+
+    while not connected:
+        try:
+            logger.info("Trying to connect to the rabbitmq server")
+            connection = pika.BlockingConnection(connection_params)
+
+        except pika.exceptions.AMQPConnectionError as err:
+            sleepTime *= 2
+            if sleepTime >= RABBIT_MAX_WAIT_BEFORE_RERTY_SECS:
+                logger.error(f"Failed to connect to the rabbitmq after {sleepTime} s")
+                raise err
+            else:
+                logger.warning(
+                    f"Failed to connect to the rabbitmq, retry in {sleepTime} s"
+                )
+                time.sleep(sleepTime)
+        except gaierror as err:
+            sleepTime *= 2
+            if sleepTime >= RABBIT_MAX_WAIT_BEFORE_RERTY_SECS:
+                logger.error(f"Failed to connect to the rabbitmq after {sleepTime} s")
+                raise err
+            else:
+                logger.warning(
+                    f"Failed to connect to the rabbitmq, [{err}] retry in {sleepTime} s"
+                )
+                time.sleep(sleepTime)
+        except Exception as err:
+            logger.error(f"Failed to connect to the rabbitmq: {err}")
+            raise err
+
+        else:
+            connected = True
+
+    return connection
+
+
+def _rabbitmq_ack_message(ch, delivery_tag):
+    """Note that `ch` must be the same pika channel instance via which
+    the message being ACKed was retrieved (AMQP protocol constraint).
+    """
+    if ch.is_open:
+        logger.info("Acknowledging message %s", delivery_tag)
+        ch.basic_ack(delivery_tag)
+    else:
+        # Channel is already closed, so we can't ACK this message;
+        # log and/or do something that makes sense for your app in this case.
+        logger.error("Channel is closed, cannot ack message")
+
+
+def _rabbitmq_queue_message(ch, exchange, routing_key, body):
+    if ch.is_open:
+        logger.info(f"Sending {body} to {routing_key}")
+        ch.basic_publish(
+            exchange=exchange,
+            routing_key=routing_key,
+            body=body,
+            properties=pika.BasicProperties(
+                delivery_mode=2,  # make message persistent
+            ),
+        )
+    else:
+        logger.error("Channel is closed, cannot queue message")

ltc_client-0.2/pyproject.toml

@@ -0,0 +1,42 @@
+[project.urls]
+Homepage = "https://github.com/TinArmEngineering/ltc_client/"
+Issues = "https://github.com/TinArmEngineering/ltc_client/issues"
+
+[tool.poetry]
+name = "ltc-client"
+version = "0.2"
+description = "\"Client and Worker module for Tin Arm Engineering LTC service\""
+authors = ["Martin West <martin@tinarmengineering.com>"]
+license = "MIT"
+readme = "README.md"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+repository = "https://github.com/TinArmEngineering/ltc_client/"
+
+
+[tool.poetry.dependencies]
+python = "^3.10"
+pika = "^1.3.2"
+python-logging-rabbitmq = "^2.3.0"
+requests = "^2.32.3"
+Pint = "^0.24"
+numpy = "^2.0.0"
+webstompy = "^0.1.3"
+tqdm = "^4.66.4"
+
+[tool.poetry.group.dev.dependencies]
+black = "^24.4.2"
+pytest = "^8.2.2"
+
+[tool.poetry.group.test.dependencies]
+mock = "^5.1.0"
+teamcity-messages = "^1.32"
+pytest = "^8.2.2"
+coverage = "^7.5.3"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"