auto-trainer-core 2.0.1.dev349__py3-none-any.whl

autotrainer/core/configuration/system_configuration.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import json
+import shutil
+from dataclasses import dataclass, field, asdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Dict, Optional, Union, ClassVar, TextIO, Type, Any
+from typing_extensions import Self
+import yaml
+
+import humps
+
+from autotrainer.core.logging import get_verbose_logger
+from . import GenericSafeLoader, SystemConfigurationLoader, SystemConfigurationDumper, SystemConfigurationSafeLoader
+from .. import make_camelize_representer, make_decamelize_constructor
+from .behavior_configuration import BehaviorConfiguration, add_behavior_configuration_representers, \
+    add_behavior_configuration_constructors
+from .camera_configuration import CameraConfiguration, CameraId, camera_configuration_representer, \
+    camera_configuration_constructor
+from .hardware_configuration import HardwareConfiguration, hardware_configuration_representer, \
+    hardware_configuration_constructor
+from .inference_configuration import InferenceConfiguration, inference_configuration_representer, \
+    inference_configuration_constructor
+from .persistence_configuration import PersistenceConfiguration, persistence_configuration_representer, \
+    persistence_configuration_constructor
+from ..project.project_info import DATE_FORMAT, TIME_FORMAT
+
+logger = get_verbose_logger(__name__)
+
+#
+
+
+@dataclass
+class SystemConfiguration:
+    """
+    The user-visible and editable elements of the system configuration are evolving.  This class is largely intended
+    manage transitions and maintain a consistent interface to applications.
+    """
+
+    DEFAULT_CONFIG_DIR: ClassVar[Path] = Path("~/Autotrainer")  # caller/user must expanduser() on it
+    DEFAULT_NAME: ClassVar[str] = "system_configuration"
+
+    DEFAULT_PATH: ClassVar[Path] = DEFAULT_CONFIG_DIR.joinpath(f"{DEFAULT_NAME}.yaml")  # caller/user must expanduser() on it
+
+    version: int = 36
+
+    cameras: List[CameraConfiguration] = field(default_factory=list)
+    hardware: HardwareConfiguration = field(default_factory=HardwareConfiguration)
+    inference: InferenceConfiguration = field(default_factory=InferenceConfiguration)
+    behavior: BehaviorConfiguration = field(default_factory=BehaviorConfiguration)
+    persistence: PersistenceConfiguration = field(default_factory=PersistenceConfiguration)
+
+    _camera_map = None  # do not include in fields
+
+    def __post_init__(self):
+        self._camera_map = {}
+
+    @classmethod
+    def load_yaml(cls: Type[Self], data: TextIO, *, file_path: Optional[Path] = None) -> Self:
+        raw_content: Dict[str, Any] = yaml.load(data, GenericSafeLoader)
+        data.seek(0)
+        version = raw_content.get("version", 0)
+        if version == SystemConfiguration.version:
+            # easy case
+            configuration: Self = yaml.load(data, SystemConfigurationLoader)
+        elif version < SystemConfiguration.version:
+            content = humps.decamelize(raw_content)
+            configuration = cls()
+            if version == 0:
+                configuration._deserialize_version_zero(content)
+            elif version == 1:
+                configuration._deserialize_version_one(content)
+            else:
+                configuration: Self = yaml.load(data, SystemConfigurationSafeLoader)
+        else:
+            assert version > SystemConfiguration.version
+            logger.warning("Loading configuration version %s while SystemConfiguration.version == %s, "
+                           "only considering known config attributes/properties.",
+                           version, SystemConfiguration.version)
+            configuration: Self = yaml.load(data, SystemConfigurationSafeLoader)
+
+        if version != SystemConfiguration.version and file_path is not None:
+            now = datetime.now()
+            now_str = now.strftime(f"{DATE_FORMAT}_{TIME_FORMAT}")
+            new_p = file_path.parent.joinpath(
+                f"{file_path.stem}_v{version}_{now_str}{file_path.suffix}")
+            logger.notice("Detected config version change/mismatch, saving old config to %s,"
+                          " and replacing with new after.", new_p)
+            shutil.copy2(file_path, new_p)
+            configuration.version = SystemConfiguration.version
+            # and save new one over previous:
+            configuration.save_file(file_path.with_suffix(""), as_yaml=True)
+
+        return configuration
+
+    @classmethod
+    def load_yaml_file(cls: Type[Self], path: Union[Path, str], *, save_backup: bool = True) -> Self:
+        path: Path = Path(path)
+        logger.debug("loading configuration from %r", path)
+        with path.open() as fh:
+            return cls.load_yaml(fh, file_path=path if save_backup else None)
+
+    @classmethod
+    def make_default_yaml_config_path(cls, dir_path: Path) -> Path:
+        return dir_path.joinpath(f"{SystemConfiguration.DEFAULT_NAME}.yaml")
+
+    @classmethod
+    def load_default(cls: Type[Self], location: Union[str, Path]) -> Optional[Self]:
+        path = cls.make_default_yaml_config_path(Path(location))
+        if path.is_file():
+            return cls.load_yaml_file(path)
+        logger.debug("cannot load default from %s ; not a file", path)
+        return None
+
+    def save_default(self, dir_path: Union[Path, str]):
+        path = self.make_default_yaml_config_path(Path(dir_path))
+        save_path: Path = path.with_suffix("")  # noqa
+        self.save_file(save_path, as_yaml=True)
+
+    def dump_yaml(self) -> str:
+        return yaml.dump(self, Dumper=SystemConfigurationDumper,
+                         # we sort/iter by dataclasses.fields() order in our representer function
+                         sort_keys=False)
+
+    def save_file(self, path: Union[Path, str], as_yaml: bool = False, as_json: bool = False):
+        if not (as_json or as_yaml):
+            raise ValueError("Missing one of as_json or as_yaml")
+        path: Path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        if as_json:
+            p = path.with_suffix(".json")
+            logger.notice("Writing to %r as json", p.as_posix())
+            content = json.dumps(asdict(self))
+            # dump before writing to file, to prevent empty file on dump issue
+            with p.open("w") as file:
+                file.write(content)
+        if as_yaml:
+            p = path.with_suffix(".yaml")
+            logger.notice("Writing to %r as yaml", p.as_posix())
+            content = self.dump_yaml()
+            # dump before writing to file, to prevent empty file on dump issue
+            with p.open("w") as file:
+                file.write(content)
+
+    def get_camera(self, camera_id: CameraId) -> Optional[CameraConfiguration]:
+        if len(self._camera_map) == 0 or camera_id not in self._camera_map:
+            for camera in self.cameras:
+                self._camera_map[camera.id] = camera
+
+        return self._camera_map.get(camera_id, None)
+
+    def _deserialize_version_zero(self, content: Dict):
+        self.cameras.clear()
+
+        self._try_append_version_zero_camera("camera1", content)
+        self._try_append_version_zero_camera("camera2", content)
+        self._try_append_version_zero_camera("camera3", content)
+
+        self.hardware = HardwareConfiguration.from_version_zero(content)
+
+        # Typo from earlier version of the file.
+        self.inference = InferenceConfiguration.from_version_zero(content.get("pellet", {}))
+
+        self.behavior = BehaviorConfiguration.from_version_zero(content)
+        self.persistence = PersistenceConfiguration.from_version_zero(content)
+
+    def _try_append_version_zero_camera(self, entry: str, content: Dict) -> bool:
+        if entry in content:
+            camera = CameraConfiguration.from_version_zero(entry, content[entry])
+            if camera is not None:
+                self.cameras.append(camera)
+                return True
+
+        return False
+
+    def _deserialize_version_one(self, content: Dict):
+        self.cameras = [
+            CameraConfiguration(**kw)
+            for kw in content.get("cameras", [])
+        ]
+        self.hardware = HardwareConfiguration(**content.get("hardware", {}))
+        self.inference = InferenceConfiguration(**content.get("inference", {}))
+        self.behavior = BehaviorConfiguration.from_version_one(content.get("behavior", {}))
+        self.persistence = PersistenceConfiguration(**content.get("persistence", {}))
+
+
+#
+
+system_configuration_representer = make_camelize_representer("!SystemConfiguration")
+
+SystemConfigurationDumper.add_representer(SystemConfiguration, system_configuration_representer)
+SystemConfigurationDumper.add_representer(CameraConfiguration, camera_configuration_representer)
+SystemConfigurationDumper.add_representer(HardwareConfiguration, hardware_configuration_representer)
+SystemConfigurationDumper.add_representer(InferenceConfiguration, inference_configuration_representer)
+SystemConfigurationDumper.add_representer(PersistenceConfiguration, persistence_configuration_representer)
+add_behavior_configuration_representers(SystemConfigurationDumper)
+
+
+system_configuration_constructor = make_decamelize_constructor(SystemConfiguration)
+
+for cls in SystemConfigurationLoader, :
+    # SystemConfigurationSafeLoader:
+    # no need also add on SystemConfigurationSafeLoader given it subclass SystemConfigurationLoader
+
+    cls.add_constructor("!SystemConfiguration", system_configuration_constructor)
+    cls.add_constructor("!CameraConfiguration", camera_configuration_constructor)
+    cls.add_constructor("!HardwareConfiguration", hardware_configuration_constructor)
+    cls.add_constructor("!InferenceConfiguration", inference_configuration_constructor)
+    cls.add_constructor("!PersistenceConfiguration", persistence_configuration_constructor)
+    add_behavior_configuration_constructors(cls)

autotrainer/core/configuration/system_fault_config.py

@@ -0,0 +1,9 @@
+import dataclasses
+
+
+@dataclasses.dataclass
+class SystemFaultConfig:
+
+    use_free_disk_space: bool = True
+    auto_resume_on_free_disk_space: bool = True
+    free_disk_space_min_limit_mb: int = 500

autotrainer/core/configuration/system_maintenance_config.py

@@ -0,0 +1,17 @@
+import dataclasses
+
+
+@dataclasses.dataclass
+class SystemMaintenanceConfig:
+
+    use_max_pellet_loaded: bool = True
+    auto_resume_on_max_pellets_loaded: bool = True
+    max_pellets_loaded_count: int = 500
+
+    use_max_consecutive_failed_load: bool = True
+    auto_resume_on_max_consecutive_failed_load: bool = True
+    max_consecutive_failed_loaded: int = 10
+
+    use_free_disk_space: bool = True
+    auto_resume_on_free_disk_space: bool = True
+    free_disk_space_min_limit_mb: int = 500

autotrainer/core/diamond_triangle_config.py

@@ -0,0 +1,150 @@
+import dataclasses
+import itertools
+import math
+import os
+from pathlib import Path
+from typing import ClassVar, Optional
+
+import yaml
+from typing_extensions import Self
+
+from autotrainer.core import Offset3DTuple
+from autotrainer.core.logging import get_verbose_logger
+
+
+logger = get_verbose_logger(__name__)
+
+
+DEFAULT_DIAMOND_TRIANGLE_CONFIG_PATH = Path(
+    os.getenv("AUTOTRAINER_DIAMOND_TRIANGLE_CONFIG", "~/Autotrainer/diamond_triangle_offset.yaml")
+).expanduser()
+
+_offset_nans = Offset3DTuple(math.nan, math.nan, math.nan)
+
+
+@dataclasses.dataclass
+class DiamondTriangleOffsetConfig:
+
+    used_position: Offset3DTuple  # motor coordinate used
+    measured_offset: Offset3DTuple  # inference coordinate offset between diamond and triangle (diamond - triangle)
+
+    raw_diamond_coord: Offset3DTuple = _offset_nans  # output from triangulate_3d
+    diamond_coord: Offset3DTuple = _offset_nans  # inference diamond coordinate, output from reorient_and_center
+
+    version: Optional[int] = None
+    current_config_version: ClassVar[int] = 2  # NB: bump me when need to
+
+    DEFAULT_CONFIG_PATH: ClassVar = DEFAULT_DIAMOND_TRIANGLE_CONFIG_PATH
+
+    # 3 coordinate systems,
+    # but each has some axis direction difference (when one increases, the other decreases)
+    flips_inference_motor: ClassVar[Offset3DTuple] = Offset3DTuple(1, -1, -1)
+    flips_inference_diamond: ClassVar[Offset3DTuple] = Offset3DTuple(1, 1, 1)
+    # we also want inference diamond flips equals (1, 1, 1), given inference == diamond
+
+    # defining flips between motor and diamond with flips between motor and inference * flips between inference and diamond:
+    flips_motor_diamond: ClassVar[Offset3DTuple] = flips_inference_diamond * flips_inference_motor  # Offset3DTuple(1, -1, -1)
+    # assert flips_inference_diamond * flips_inference_motor == flips_motor_diamond
+
+    # custom init to ensure kwarg only :
+    def __init__(
+        self,
+        *,
+        used_position: Offset3DTuple,
+        measured_offset: Offset3DTuple,
+        diamond_coord: Offset3DTuple = _offset_nans,
+        raw_diamond_coord: Offset3DTuple = _offset_nans,
+        version: Optional[int] = None,
+    ):
+        super().__init__()
+        self.used_position = used_position
+        self.measured_offset = measured_offset
+        self.diamond_coord = diamond_coord
+        self.raw_diamond_coord = raw_diamond_coord
+        self.version = version
+
+    @property
+    def fully_valid(self):
+        return self.version == self.current_config_version and all(map(math.isfinite,
+            itertools.chain(
+                self.used_position,
+                self.measured_offset,
+                self.diamond_coord,
+                self.raw_diamond_coord,
+            )))
+
+    @classmethod
+    def load_config(cls, cfg_path: Optional[Path]) -> Optional[Self]:
+        if cfg_path is None:
+            logger.notice("No diamond-triangle offset config path provided")
+        else:
+            if not cfg_path.expanduser().is_file():
+                logger.warning("Diamond triangle config %r not a file", cfg_path.as_posix())
+            else:
+                logger.verbose("Loading diamond-triangle file %r", cfg_path.as_posix())
+                cfg = DiamondTriangleOffsetConfig.from_file(cfg_path)
+                if not all(math.isfinite(c) for c in cfg.diamond_coord):
+                    logger.warning("Diamond coordinate undefined or not finite in config %r",
+                                   cfg_path.as_posix())
+                return cfg
+        return None
+
+    @classmethod
+    def from_file(cls, path: Path) -> Self:
+        with path.expanduser().open() as fh:
+            dct = yaml.safe_load(fh)
+        if dct is None:
+            # in case of empty file
+            return cls(used_position=_offset_nans, measured_offset=_offset_nans, version=cls.current_config_version)
+        return cls(**dict((k, v if k == "version" else Offset3DTuple(v)) for k, v in dct.items()))
+
+    def to_file(self, path: Path):
+        d = dataclasses.asdict(self)
+        for k, v in d.items():
+            if k != "version":
+                d[k] = list(v)  # getting yaml type error with Offset3D not natively supported by yaml
+        with path.expanduser().open("w") as fh:
+            yaml.safe_dump(d, fh)
+
+    def inference_to_motor(self, inference_xyz: Offset3DTuple) -> Offset3DTuple:
+        """Transform an inference coordinate to motor corresponding coordinate,
+        relatively to the diamond-triangle known position & relative offset"""
+        # only used by tests
+        assert isinstance(inference_xyz, Offset3DTuple), inference_xyz
+        return self.flips_inference_motor * (
+            self.flips_inference_diamond * self.measured_offset - inference_xyz
+        ) + self.used_position
+
+    def motor_to_inference(self, motor_xyz: Offset3DTuple) -> Offset3DTuple:
+        # only used by tests
+        assert isinstance(motor_xyz, Offset3DTuple), motor_xyz
+        # return self.flips_inference_motor * (motor_xyz - self.used_position) - self.flips_inference_diamond * self.measured_offset
+        return (
+            self.flips_inference_diamond * self.measured_offset
+            - self.flips_inference_motor * (motor_xyz - self.used_position)
+        )
+
+    def motor_to_diamond(self, motor_xyz: Offset3DTuple) -> Offset3DTuple:
+        """Transform the motor coordinate to corresponding triangle coordinate in DCS"""
+        assert isinstance(motor_xyz, Offset3DTuple), motor_xyz
+        return (
+            self.flips_motor_diamond * (motor_xyz - self.used_position)
+            + self.measured_offset
+        )
+
+    def diamond_to_motor(self, diamond_xyz: Offset3DTuple) -> Offset3DTuple:
+        """Transform the triangle coordinate from DCS to corresponding motor coordinates"""
+        assert isinstance(diamond_xyz, Offset3DTuple), diamond_xyz
+        return self.used_position + self.flips_motor_diamond * (
+            diamond_xyz - self.measured_offset
+        )
+
+    def inference_to_diamond(self, inference_xyz: Offset3DTuple) -> Offset3DTuple:
+        assert isinstance(inference_xyz, Offset3DTuple), inference_xyz
+        return self.motor_to_diamond(self.inference_to_motor(inference_xyz))
+        # return self.flips_inference_diamond * inference_xyz
+
+    def diamond_to_inference(self, diamond_xyz: Offset3DTuple) -> Offset3DTuple:
+        assert isinstance(diamond_xyz, Offset3DTuple), diamond_xyz
+        return self.motor_to_inference(self.diamond_to_motor(diamond_xyz))
+        # return self.flips_inference_diamond * diamond_xyz

autotrainer/core/event/__init__.py

@@ -0,0 +1,54 @@
+
+from typing import Optional, Any
+
+from autotrainer.api.api_event_kind import ApiEventKind
+from autotrainer.api.api_options import create_default_api_options
+
+from autotrainer.core.logging import get_verbose_logger
+from autotrainer.core.event.api_event_plugin import ApiEventPlugin
+
+from .event_manager import EventManager
+
+
+logger = get_verbose_logger(__name__)
+
+
+__all__ = [
+    "ApiEventKind",
+    "try_register_api_event_plugin",
+    "post_api_event_content",
+    "post_api_detector_event_content",
+]
+
+
+def try_register_api_event_plugin() -> Optional[ApiEventPlugin]:
+    """
+    Attempt to register the Autotrainer External API plugin with the default instance of the event manager.
+
+    Returns:
+        bool: True if the plugin was successfully registered, False otherwise.
+    """
+    try:
+        # NB: keeping import here,
+        # given otherwise it gives app crash at start if it's imported before the main qt app instance is created
+
+        plugin = ApiEventPlugin(create_default_api_options())
+        plugin.set_enable(True)
+        EventManager.default().register_plugin(plugin)
+        return plugin
+    except Exception as err:
+        logger.exception("API plugin creation or registration failed: %s", err)
+
+    return None
+
+
+def post_api_event_content(kind: int, data: Any):
+    EventManager.default().post_event_content(kind, data=data)
+
+
+def post_api_detector_event_content(manager: EventManager, detector_id: int, active: bool, enabled: bool):
+    manager.post_event_content(ApiEventKind.detectorChanged, data=dict(
+        detector_id=detector_id,
+        is_active=active,
+        is_enabled=enabled,
+    ))

autotrainer/core/event/api_event_kind.py

@@ -0,0 +1,5 @@
+
+from autotrainer.api.api_event_kind import ApiEventKind
+
+
+__all__ = ["ApiEventKind"]

autotrainer/core/event/api_event_plugin.py

@@ -0,0 +1,57 @@
+from dataclasses import asdict
+from typing import Optional
+
+from autotrainer.api import ApiOptions, RpcService, create_api_service, ApiTopic
+
+from ..logging import get_verbose_logger
+from ..project import ProjectInfo
+from .event_info import EventInfo
+
+from .event_manager_plugin import EventManagerPlugin
+
+logger = get_verbose_logger(__name__)
+
+
+class ApiEventPlugin(EventManagerPlugin):
+    """
+    An EventManager plugin that sends events to the auto-trainer-api facilities.
+
+    This plugin requires the auto-trainer-api package to be installed.
+
+    See auto-trainer-api for additional details.
+    """
+
+    def __init__(self, options: ApiOptions):
+        self._options = options
+        self._service: Optional[RpcService] = None
+
+    @property
+    def service(self) -> Optional[RpcService]:
+        return self._service
+
+    def set_project(self, project: Optional[ProjectInfo]) -> None:
+        pass
+
+    def set_enable(self, enable: bool) -> None:
+        svc = self._service
+        if enable and svc is None:
+            svc = self._service = create_api_service(self._options)
+            if svc is not None:
+                svc.start()
+        elif not enable and svc is not None:
+            svc.stop()
+            self._service = None
+
+    def process_event(self, info: EventInfo, repeat_count: int) -> None:
+        svc = self._service
+        if svc is not None:
+            svc.send_dict(ApiTopic.EVENT, asdict(info))
+
+    def flush(self) -> None:
+        pass
+
+    def close(self) -> None:
+        svc = self._service
+        if svc is not None:
+            svc.stop()
+            self._service = None

autotrainer/core/event/event_info.py

@@ -0,0 +1,53 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+from typing_extensions import Self
+
+
+@dataclass(frozen=True)
+class EventInfo:
+    """
+    EventInfo is a structured definition for "events" in the system.  The primary objective is to provide a history of
+    actions and changes that are consequential to animal training in structured format that can be parsed in
+    conjunction with outer data files.  This is generally a subset of the information that is needed or useful for
+    general logging.
+    """
+    kind: int
+    """
+    The kind of event.  This is a unique identifier for the event type. Some coordination is required across modules 
+    to prevent id duplication, if desired.  For most event sinks, the `str()` value is also used as part of the 
+    output.  Using and `IntEnum` or similar that generates a descriptive string by default is encouraged.
+    """
+    when: datetime
+    """
+    The time the event occurred.  This is typically presented as a date/time string in the output.
+    """
+    index: int
+    """
+    An time-based index that may be more precise than standard date-time functions.  Typically set via 
+    `time.perf_counter_ns()`.  The same method of indexing must be used across all `EventInfo` providers if indexing 
+    is expected to be consistent across all event types.
+    """
+    context: Optional[object] = None
+    """
+    An optional object containing event specific information.  `EventInfo` collections are typically presented in a
+    structured, row-column format.  Complex hierarchical data structures may not be represented well in this format.  
+    The object must be serializable to JSON using the default JSON encoder and be pickleable to ensure support in all
+    event manager sinks.
+    """
+
+    def is_same(self, info: Self) -> bool:
+        """
+        Provide a deeper than ReferenceEquals comparison of two `EventInfo` objects to determine repeat events.  It is
+        less stringent than a raw shallow compare as fields like when and index are expected to be different for repeat
+        events.  There may be specialized `EventInfo` instances require a custom comparison of `context` to determine
+        sameness in which case this can be overridden in a subclass.
+
+        Args:
+            info: another `EventInfo` object to compare against.
+
+        Returns:
+            True if the two `EventInfo` objects are the same, False otherwise.
+        """
+        return info is not None and self.kind == info.kind and self.context == info.context