fmtr.tools 1.3.81__py3-none-any.whl → 1.4.37__py3-none-any.whl

fmtr/tools/networking_tools.py

@@ -0,0 +1,73 @@
+import socket
+
+import uuid
+from dataclasses import dataclass, field
+from functools import cached_property
+
+get_hostname = socket.gethostname
+get_fqdn = socket.getfqdn
+
+
+@dataclass(frozen=True)
+class MAC:
+    value: int = field(default_factory=uuid.getnode)
+
+    @classmethod
+    def from_string(cls, s: str):
+        """
+
+        Initialise from a string.
+
+        """
+        s = s.lower().replace(":", "").replace("-", "")
+        return cls(int(s, 16))
+
+    @cached_property
+    def int(self):
+        return self.value
+
+    @cached_property
+    def hex(self):
+        """
+
+        Plain 12-character hex string, no separators.
+
+        """
+        return f"{self.value:012x}"
+
+    @cached_property
+    def components(self):
+        """
+
+        Return list of six 2-digit hex strings.
+
+        """
+        h = self.hex
+        return [h[i:i + 2] for i in range(0, 12, 2)]
+
+    @cached_property
+    def hex_colon(self):
+        """
+
+        Colon-separated hex string (default standard).
+
+        """
+        return self.string(":")
+
+    @cached_property
+    def is_random(self) -> bool:
+        """
+
+        Detect if this MAC is likely randomly generated by uuid.getnode().
+        Checks the "multicast" bit (bit 1 of the first byte).
+
+        """
+        return bool(self.value & (1 << 40))
+
+    def string(self, sep: str = ":") -> str:
+        """
+
+        Return MAC as a string with separator `sep`.
+
+        """
+        return sep.join(self.components)

fmtr/tools/path_tools/__init__.py

@@ -1,5 +1,5 @@
 from fmtr.tools.import_tools import MissingExtraMockModule
-from fmtr.tools.path_tools.path_tools import Path, PackagePaths
+from fmtr.tools.path_tools.path_tools import Path, PackagePaths, root
 
 try:
     from fmtr.tools.path_tools.app_path_tools import AppPaths

fmtr/tools/path_tools/path_tools.py

@@ -1,8 +1,9 @@
+from pathlib import Path
+
 import re
 import subprocess
-from pathlib import Path
 from tempfile import gettempdir
-from typing import Union, Any
+from typing import Union, Any, Self
 
 from fmtr.tools.constants import Constants
 from fmtr.tools.platform_tools import is_wsl
@@ -204,11 +205,51 @@ class Path(type(Path())):
         return kind
 
     @property
-    def children(self):
+    def children(self) -> list[Self]:
+        """
+
+        Recursive children property
+
+        """
         if not self.is_dir():
             return None
         return sorted(self.iterdir(), key=lambda x: x.is_dir(), reverse=True)
 
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source, handler):
+        """
+
+        Support Pydantic de/serialization and validation
+
+        TODO: Ideally these would be a mixin in dm, but then we'd need Pydantic to use it. Split dm module into Pydantic depts and other utils and import from there.
+
+        """
+        from pydantic_core import core_schema
+        return core_schema.no_info_plain_validator_function(
+            cls.__deserialize_pydantic__,
+            serialization=core_schema.plain_serializer_function_ser_schema(cls.__serialize_pydantic__),
+        )
+
+    @classmethod
+    def __serialize_pydantic__(cls, self) -> str:
+        """
+
+        Serialize to string
+
+        """
+        return str(self)
+
+    @classmethod
+    def __deserialize_pydantic__(cls, data) -> Self:
+        """
+
+        Deserialize from string
+
+        """
+        if isinstance(data, cls):
+            return data
+        return cls(data)
+
 
 class FromCallerMixin:
     """
@@ -361,8 +402,6 @@ class PackagePaths(FromCallerMixin):
 
         return self.data / Constants.DIR_NAME_SOURCE
 
-
-
     @property
     def settings(self) -> Path:
         """
@@ -381,6 +420,24 @@ class PackagePaths(FromCallerMixin):
         """
         return self.artifact / Constants.DIR_NAME_HF
 
+    @property
+    def docs(self) -> Path:
+        """
+
+        Path of docs directory
+
+        """
+        return self.repo / Constants.DOCS_DIR
+
+    @property
+    def docs_config(self) -> Path:
+        """
+
+        Path of docs config file
+
+        """
+        return self.repo / Constants.DOCS_CONFIG_FILENAME
+
     def __repr__(self) -> str:
         """
 
@@ -390,7 +447,9 @@ class PackagePaths(FromCallerMixin):
         return f'{self.__class__.__name__}("{self.path}")'
 
 
+root = Path('/')
+
 if __name__ == "__main__":
     path = Path('/usr/bin/bash').absolute()
     path.type
-    path
+    path

fmtr/tools/pattern_tools.py

@@ -127,6 +127,23 @@ class Transformer:
     is_recursive: bool = False
 
     def __post_init__(self):
+        """
+
+        Compile on init
+
+        """
+        return self.compile(clear=False)
+
+    def compile(self, clear=True):
+        """
+
+        Re/compile regex pattern, invalidating existing caches if recompile.
+
+        """
+        if clear:
+            del self.pattern
+            del self.rx
+
         with logger.span(f'Compiling expression {len(self.items)=}'):
             rx = self.rx
         logger.debug(f'Compiled successfully {rx.groups=}')

fmtr/tools/settings_tools.py

@@ -2,6 +2,7 @@ from typing import ClassVar, Any
 
 from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, YamlConfigSettingsSource, EnvSettingsSource, CliSettingsSource
 
+from fmtr.tools import Constants
 from fmtr.tools.data_modelling_tools import CliRunMixin
 from fmtr.tools.path_tools import PackagePaths, Path
 
@@ -32,6 +33,7 @@ class Base(BaseSettings, CliRunMixin):
 
     """
 
+    ENV_NESTED_DELIMITER: ClassVar = Constants.ENV_NESTED_DELIMITER
     paths: ClassVar = PackagePaths()
 
     @classmethod
@@ -52,7 +54,7 @@ class Base(BaseSettings, CliRunMixin):
         sources = (
             init_settings,
             CliSettingsSource(settings_cls, cli_parse_args=True),
-            EnvSettingsSource(settings_cls, env_prefix=cls.get_env_prefix()),
+            EnvSettingsSource(settings_cls, env_prefix=cls.get_env_prefix(), env_nested_delimiter=cls.ENV_NESTED_DELIMITER),
             YamlScriptConfigSettingsSource(settings_cls, yaml_file=cls.paths.settings),
         )
 
@@ -70,7 +72,7 @@ class Base(BaseSettings, CliRunMixin):
         else:
             stem = f'{cls.paths.name}'
 
-        prefix = f'{stem}_'.upper()
+        prefix = f'{stem}{cls.ENV_NESTED_DELIMITER}'.upper()
         return prefix
 
     @property

fmtr/tools/setup_tools/setup_tools.py

@@ -63,7 +63,7 @@ class SetupPaths(FromCallerMixin):
         packages = [
             dir for dir in base.iterdir()
             if (dir / Constants.INIT_FILENAME).is_file()
-               and not any(fnmatch(dir.name, pattern) for pattern in Constants.PACKAGE_EXCLUDE_DIRS)
+               and not any(fnmatch(dir.name, pattern) for pattern in Constants.PACKAGE_EXCLUDE_DIRS)  # todo add scripts dir
         ]
 
         if len(packages) != 1:
@@ -106,6 +106,16 @@ class SetupPaths(FromCallerMixin):
         """
         return self.path / Constants.ENTRYPOINTS_DIR
 
+    @property
+    def scripts(self) -> Path:
+        """
+
+        Paths of shell scripts
+
+        """
+
+        return self.repo / Constants.SCRIPTS_DIR
+
     @property
     def is_namespace(self) -> bool:
         return bool(self.org)
@@ -159,6 +169,7 @@ class Setup(FromCallerMixin):
 
         if do_setup:
             self.setup()
+        self
 
     def get_requirements_extras(self) -> Optional[List[str]]:
         """
@@ -214,6 +225,28 @@ class Setup(FromCallerMixin):
 
         return console_scripts
 
+    @property
+    def scripts(self) -> List[str]:
+        """
+
+        Generate list of shell scripts.
+
+        """
+
+        paths = []
+
+        if not self.paths.scripts.exists():
+            return paths
+
+        for path in self.paths.scripts.iterdir():
+            if path.is_dir():
+                continue
+
+            path_rel = path.relative_to(self.paths.repo)
+            paths.append(str(path_rel))
+
+        return paths
+
     @cached_property
     def name_command(self) -> str:
         """
@@ -353,6 +386,7 @@ class Setup(FromCallerMixin):
             ),
             install_requires=self.dependencies.install,
             extras_require=self.dependencies.extras,
+            scripts=self.scripts,
         ) | self.kwargs
         return data
 

fmtr/tools/version

@@ -1 +1 @@
-1.3.81
+1.4.37

fmtr/tools/yaml_tools.py

@@ -3,8 +3,6 @@ from typing import Any
 from yaml import CDumper as Dumper
 from yaml import dump
 
-from fmtr.tools import environment_tools as env
-
 try:
     import yamlscript
 except ImportError:
@@ -29,7 +27,7 @@ def install():
 
 
 @lru_cache
-def get_module(is_auto=env.IS_DEV):
+def get_module(is_auto=True):
     """
 
     Get the YAML Script runtime module, installing the runtime if specified

fmtr/tools/youtube_tools.py

@@ -0,0 +1,128 @@
+from dataclasses import dataclass
+from functools import cached_property
+from pytubefix import YouTube, Stream, extract, request
+from pytubefix.exceptions import RegexMatchError
+from typing import AsyncIterator, Iterator
+from urllib.error import HTTPError
+
+from fmtr.tools.path_tools.path_tools import Path
+
+Stream = Stream
+
+
+class Video(YouTube):
+    """
+
+    Video stub
+
+    """
+
+
+class AudioStreamDownloadError(Exception):
+    """
+
+    Error downloading audio stream
+
+    """
+    pass
+
+
+@dataclass
+class AudioStreamData:
+    """
+
+    Audio stream download data and progress information
+
+    """
+    message: str | None = None
+    chunk: bytes | None = None
+    percentage: int | None = None
+
+
+class AudioStreamDownloader:
+    """
+
+    Download the highest-bitrate audio stream and write to temp directory.
+
+    """
+
+    def __init__(self, url_or_id: str):
+        """
+
+        Initialise with URL or video ID
+
+        """
+
+        try:
+            self.id = extract.video_id(url_or_id)
+        except RegexMatchError:
+            self.id = url_or_id
+
+        self.path = None
+
+    @cached_property
+    def url(self) -> str:
+        """
+
+        Get URL from ID
+
+        """
+        return f'https://youtube.com/watch?v={self.id}'
+
+    async def download(self) -> AsyncIterator[AudioStreamData]:
+        """
+
+        Download the audio stream and yield chunks and progress information
+
+        """
+
+        yield AudioStreamData(message='Fetching video metadata...')
+        video = Video(self.url)
+
+        yield AudioStreamData('Finding audio streams...')
+
+        audio_streams = video.streams.filter(only_audio=True).order_by('bitrate')
+        if not audio_streams:
+            raise AudioStreamDownloadError(f'Error downloading: no audio streams found in "{video.title}"')
+
+        stream = audio_streams.last()
+        yield AudioStreamData(f'Found highest-bitrate audio stream: {stream.audio_codec}/{stream.subtype}@{stream.abr}')
+
+        self.path = Path.temp() / stream.default_filename
+        if self.path.exists():
+            self.path.unlink()
+
+        if stream.filesize == 0:
+            raise AudioStreamDownloadError(f'Error downloading: empty audio stream found in "{video.title}"')
+
+        yield AudioStreamData('Downloading...')
+
+        with self.path.open('wb') as out_file:
+            for data in self.iter_data(stream):
+                out_file.write(data.chunk)
+                yield data
+
+    def iter_data(self, stream: Stream, chunk_size: int | None = None) -> Iterator[AudioStreamData]:
+        """
+
+        Iterate over chunks of the specified size
+
+        """
+        bytes_total = bytes_remaining = stream.filesize
+
+        if chunk_size:
+            request.default_range_size = chunk_size
+
+        try:
+            stream = request.stream(stream.url)
+        except HTTPError as e:
+            if e.code != 404:
+                raise
+            stream = request.seq_stream(stream.url)
+
+        for chunk in stream:
+            bytes_remaining -= len(chunk)
+            percentage = round(((bytes_total - bytes_remaining) / bytes_total) * 100)
+
+            data = AudioStreamData(chunk=chunk, percentage=percentage)
+            yield data

fmtr_tools-1.4.37.data/scripts/add-service

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+. parse-args "$@"
+
+loginctl enable-linger $USER
+
+NAME=$(basename "$FILE")
+
+mkdir -p ~/.config/systemd/user
+cp "$FILE" ~/.config/systemd/user/
+systemctl --user daemon-reload
+systemctl --user enable "$NAME"
+systemctl --user start "$NAME"
+systemctl --user status -l -n 50 "$NAME"

fmtr_tools-1.4.37.data/scripts/add-user-path

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+source $SCRIPT_DIR/parse-args "$@"
+
+NEW=${NEW:-/home/${USERNAME}/.local/bin/}
+
+PATH="${NEW}:${PATH}"

fmtr_tools-1.4.37.data/scripts/apt-headless

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+source $SCRIPT_DIR/parse-args "$@"
+
+export DEBIAN_FRONTEND=noninteractive
+apt update --yes
+
+# if --full was passed, run full-upgrade
+if [ "${FULL}" = "1" ]; then
+    apt --yes full-upgrade
+fi
+
+# if ARGS is not empty, install the packages
+if [ ${#ARGS[@]} -gt 0 ]; then
+    apt install --yes --no-install-recommends "${ARGS[@]}"
+fi
+
+apt autoremove --yes
+apt clean
+rm -rf /var/lib/apt/lists/*

fmtr_tools-1.4.37.data/scripts/compose-update

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+. parse-args "$@"
+
+PROJECT_DIR=/opt/dev/repo/infrastructure/${CONTEXT-gex}/docker/${SERVICE}
+CONTEXT=${CONTEXT-gex}
+
+docker --context "$CONTEXT" compose --project-directory "$PROJECT_DIR" pull
+docker --context "$CONTEXT" compose --project-directory "$PROJECT_DIR" up --detach
+docker --context "$CONTEXT" compose --project-directory "$PROJECT_DIR" logs --follow

fmtr_tools-1.4.37.data/scripts/docker-sandbox

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+. parse-args "$@"
+
+#Spin up the container in the background with sleep infinity
+CONTAINER_ID=$(docker run -d \
+  --hostname=sandbox \
+  --user=user \
+  --volume=/opt/dev/:/opt/dev/ \
+  --publish=${PORT-9000}:${PORT-9000} \
+  ${IMAGE-fmtr/python} \
+  bash -c "sleep infinity")
+
+
+# Capture Docker's default name
+DEFAULT_NAME=$(docker inspect --format '{{.Name}}' $CONTAINER_ID | cut -c2-)
+
+# Modify it dynamically (prefix or suffix)
+MODIFIED_NAME="sandbox_${DEFAULT_NAME}"
+docker rename $CONTAINER_ID $MODIFIED_NAME
+
+echo "Started container $MODIFIED_NAME."
+
+#Run the sandbox-init script as root (interactive)
+docker exec \
+  --interactive=true \
+  --tty=true \
+  --user=root \
+  $CONTAINER_ID \
+  /opt/dev/repo/fmtr.tools/scripts/docker-sandbox-init --tools
+
+#Start an interactive bash session as user
+docker exec \
+  --interactive=true \
+  --tty=true \
+  --user=user \
+  --env USER=user \
+  $CONTAINER_ID \
+  bash
+
+# Cleanup
+docker rm -f $MODIFIED_NAME
+echo "Container $MODIFIED_NAME removed."

fmtr_tools-1.4.37.data/scripts/docker-sandbox-init

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+SECURE_PATH=$PATH
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+source $SCRIPT_DIR/add-user-path --username=user --new=$SCRIPT_DIR
+
+. parse-args "$@"
+
+echo "Setting up sandbox..."
+
+set-password
+set-password --username=user
+
+echo "user ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/sandbox
+echo "Defaults secure_path=\"${SECURE_PATH}\"" >> /etc/sudoers.d/sandbox
+chmod 440 /etc/sudoers.d/sandbox
+
+if [ "$TOOLS" == "1" ]; then
+  sudo -u user -H pip install --user fmtr.tools --upgrade
+fi

fmtr_tools-1.4.37.data/scripts/docs-deploy

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+. parse-args "$@"
+
+mkdocs gh-deploy --remote-branch docs --force
+

fmtr_tools-1.4.37.data/scripts/docs-serve

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. parse-args "$@"
+
+mkdocs serve --dev-addr=${HOST-0.0.0.0}:${PORT-8180} --livereload

fmtr_tools-1.4.37.data/scripts/download

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+. parse-args "$@"
+
+OUT_OPT=${FILE:+-o "$FILE"}
+[ -z "$FILE" ] && OUT_OPT="-O"
+
+curl -L -C - --retry 5 --retry-delay 5 --retry-connrefused --progress-bar $OUT_OPT "$URL"
+

fmtr_tools-1.4.37.data/scripts/fmtr-test-script

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Test Script Ran"

fmtr_tools-1.4.37.data/scripts/ftu

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+pip install fmtr.tools --upgrade

fmtr_tools-1.4.37.data/scripts/ha-addon-launch

@@ -0,0 +1,16 @@
+#!/usr/bin/with-contenv bashio
+
+. parse-args "$@"
+
+bashio::log.info "${NAME} container started."
+
+export FMTR_DEV="$(bashio::config 'fmtr_dev')"
+
+if bashio::var.true "${FMTR_DEV}"; then
+    bashio::log.info "Starting ${NAME} SSH development server"
+    printenv > /addon.env
+    echo "root:password" | chpasswd
+    /usr/sbin/sshd -D -o Port=22 -o PermitRootLogin=yes -o PasswordAuthentication=yes -o AllowTcpForwarding=yes -o LogLevel=VERBOSE
+else
+    ${NAME}
+fi

fmtr_tools-1.4.37.data/scripts/install-browser

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+
+sudo -u user /opt/dev/venv/fmtr.tools/bin/python -m pip install fmtr.tools[browsers] --upgrade
+/opt/dev/venv/fmtr.tools/bin/python -m playwright install-deps
+sudo -u user /opt/dev/venv/fmtr.tools/bin/python -m playwright install chromium