mm-std 0.0.1__tar.gz

mm_std-0.0.1/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.1
+Name: mm-std
+Version: 0.0.1
+Requires-Python: >=3.11
+Requires-Dist: requests~=2.32.3
+Requires-Dist: PySocks~=1.7.1
+Requires-Dist: pydash~=8.0.0
+Requires-Dist: python-dateutil~=2.9.0
+Requires-Dist: pydantic~=2.8.2
+Requires-Dist: python-dotenv~=1.0.1
+Requires-Dist: PyYAML~=6.0.1
+Requires-Dist: cryptography~=43.0.0
+Requires-Dist: rich
+Provides-Extra: dev
+Requires-Dist: build~=1.2.1; extra == "dev"
+Requires-Dist: twine~=5.1.0; extra == "dev"
+Requires-Dist: pytest~=8.3.2; extra == "dev"
+Requires-Dist: pytest-xdist~=3.6.1; extra == "dev"
+Requires-Dist: pytest-httpserver~=1.0.8; extra == "dev"
+Requires-Dist: coverage~=7.6.0; extra == "dev"
+Requires-Dist: ruff~=0.5.2; extra == "dev"
+Requires-Dist: pip-audit~=2.7.0; extra == "dev"
+Requires-Dist: bandit~=1.7.7; extra == "dev"
+Requires-Dist: mypy~=1.11.0; extra == "dev"
+Requires-Dist: types-python-dateutil~=2.9.0; extra == "dev"
+Requires-Dist: types-requests~=2.32.0.20240523; extra == "dev"
+Requires-Dist: types-PyYAML~=6.0.12.12; extra == "dev"

mm_std-0.0.1/pyproject.toml

@@ -0,0 +1,78 @@
+[project]
+name = "mm-std"
+version = "0.0.1"
+description = ""
+requires-python = ">=3.11"
+dependencies = [
+    "requests~=2.32.3",
+    "PySocks~=1.7.1",
+    "pydash~=8.0.0",
+    "python-dateutil~=2.9.0",
+    "pydantic~=2.8.2",
+    "python-dotenv~=1.0.1",
+    "PyYAML~=6.0.1",
+    "cryptography~=43.0.0",
+    "rich",
+]
+[project.optional-dependencies]
+dev = [
+    "build~=1.2.1",
+    "twine~=5.1.0",
+    "pytest~=8.3.2",
+    "pytest-xdist~=3.6.1",
+    "pytest-httpserver~=1.0.8",
+    "coverage~=7.6.0",
+    "ruff~=0.5.2",
+    "pip-audit~=2.7.0",
+    "bandit~=1.7.7",
+    "mypy~=1.11.0",
+    "types-python-dateutil~=2.9.0",
+    "types-requests~=2.32.0.20240523",
+    "types-PyYAML~=6.0.12.12",
+]
+
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+
+[tool.mypy]
+python_version = "3.12"
+warn_no_return = false
+strict = true
+exclude = ["^tests/", "^tmp/"]
+
+
+[tool.ruff]
+line-length = 130
+target-version = "py312"
+lint.select = [
+    "F", # Pyflakes
+    "E", "W", # pycodestyle
+    "B", # flake8-bugbear
+    "A", # flake8-builtins
+    "COM", # flake8-commas
+    "C40", # flake8-comprehensions
+    "G", # flake8-logging-format
+    "PIE", # flake8-pie
+    "T20", # flake8-print
+    "RUF", # Ruff-specific rules
+]
+lint.ignore = [
+    "A003", # builtin-attribute-shadowing
+    "UP040", # non-pep695-type-alias
+    "COM812"
+]
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+
+[tool.bandit]
+exclude_dirs = ["tests"]
+skips = ["B311"]
+
+
+[tool.pytest.ini_options]
+markers = ["proxy: requires access proxies", "telegram: requires a telegram bot"]

mm_std-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

mm_std-0.0.1/src/mm_std/__init__.py

@@ -0,0 +1,46 @@
+from .command import CommandResult as CommandResult
+from .command import run_command as run_command
+from .command import run_ssh_command as run_ssh_command
+from .concurrency import ConcurrentTasks as ConcurrentTasks
+from .concurrency import Scheduler as Scheduler
+from .concurrency import synchronized as synchronized
+from .concurrency import synchronized_parameter as synchronized_parameter
+from .config import BaseConfig as BaseConfig
+from .crypto import fernet_decrypt as fernet_decrypt
+from .crypto import fernet_encrypt as fernet_encrypt
+from .crypto import fernet_generate_key as fernet_generate_key
+from .date import parse_date as parse_date
+from .date import utc_delta as utc_delta
+from .date import utc_now as utc_now
+from .date import utc_random as utc_random
+from .dict import replace_empty_values as replace_empty_values
+from .env import get_dotenv as get_dotenv
+from .json_ import CustomJSONEncoder as CustomJSONEncoder
+from .json_ import json_dumps as json_dumps
+from .log import init_logger as init_logger
+from .net import CHROME_USER_AGENT as CHROME_USER_AGENT
+from .net import FIREFOX_USER_AGENT as FIREFOX_USER_AGENT
+from .net import HResponse as HResponse
+from .net import add_query_params_to_url as add_query_params_to_url
+from .net import check_port as check_port
+from .net import hr as hr
+from .net import hrequest as hrequest
+from .print_ import PrintFormat as PrintFormat
+from .print_ import fatal as fatal
+from .print_ import print_console as print_console
+from .print_ import print_json as print_json
+from .print_ import print_plain as print_plain
+from .print_ import print_table as print_table
+from .random_ import random_choice as random_choice
+from .random_ import random_decimal as random_decimal
+from .random_ import random_str_choice as random_str_choice
+from .result import Err as Err
+from .result import Ok as Ok
+from .result import Result as Result
+from .result import try_ok as try_ok
+from .str import number_with_separator as number_with_separator
+from .str import str_ends_with_any as str_ends_with_any
+from .str import str_starts_with_any as str_starts_with_any
+from .str import str_to_list as str_to_list
+from .telegram import send_telegram_message as send_telegram_message
+from .zip import read_text_from_zip_archive as read_text_from_zip_archive

mm_std-0.0.1/src/mm_std/command.py

@@ -0,0 +1,35 @@
+import subprocess  # nosec
+from dataclasses import dataclass
+
+
+@dataclass
+class CommandResult:
+    stdout: str
+    stderr: str
+    code: int
+
+    @property
+    def out(self) -> str:
+        if self.stdout:
+            return self.stdout + "\n" + self.stderr
+        return self.stderr
+
+
+def run_command(cmd: str, timeout: int | None = 60, capture_output: bool = True, echo_cmd_console: bool = False) -> CommandResult:
+    if echo_cmd_console:
+        print(cmd)  # noqa: T201
+    try:
+        process = subprocess.run(cmd, timeout=timeout, capture_output=capture_output, shell=True, check=False)  # nosec
+        stdout = process.stdout.decode("utf-8") if capture_output else ""
+        stderr = process.stderr.decode("utf-8") if capture_output else ""
+        return CommandResult(stdout=stdout, stderr=stderr, code=process.returncode)
+    except subprocess.TimeoutExpired:
+        return CommandResult(stdout="", stderr="timeout", code=124)
+
+
+def run_ssh_command(host: str, cmd: str, ssh_key_path: str | None = None, timeout: int = 60) -> CommandResult:
+    ssh_cmd = "ssh -o 'StrictHostKeyChecking=no' -o 'LogLevel=ERROR'"
+    if ssh_key_path:
+        ssh_cmd += f" -i {ssh_key_path} "
+    ssh_cmd += f" {host} {cmd}"
+    return run_command(ssh_cmd, timeout=timeout)

mm_std-0.0.1/src/mm_std/concurrency.py

@@ -0,0 +1,157 @@
+import concurrent.futures
+import functools
+import time
+from collections import defaultdict
+from collections.abc import Callable
+from concurrent.futures.thread import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from datetime import datetime
+from logging import Logger
+from threading import Lock, Thread
+from typing import ParamSpec, TypeAlias, TypeVar
+
+from .date import is_too_old, utc_now
+
+Func: TypeAlias = Callable[..., object]
+Args: TypeAlias = tuple[object, ...]
+Kwargs: TypeAlias = dict[str, object]
+
+
+class ConcurrentTasks:
+    def __init__(self, max_workers: int = 5, timeout: int | None = None, thread_name_prefix: str = "concurrent_tasks") -> None:
+        self.max_workers = max_workers
+        self.timeout = timeout
+        self.thread_name_prefix = thread_name_prefix
+        self.tasks: list[ConcurrentTasks.Task] = []
+        self.exceptions: dict[str, Exception] = {}
+        self.error = False
+        self.timeout_error = False
+        self.result: dict[str, object] = {}
+
+    @dataclass
+    class Task:
+        key: str
+        func: Func
+        args: Args
+        kwargs: Kwargs
+
+    def add_task(self, key: str, func: Func, args: Args = (), kwargs: Kwargs | None = None) -> None:
+        if kwargs is None:
+            kwargs = {}
+        self.tasks.append(ConcurrentTasks.Task(key, func, args, kwargs))
+
+    def execute(self) -> None:
+        with ThreadPoolExecutor(self.max_workers, thread_name_prefix=self.thread_name_prefix) as executor:
+            future_to_key = {executor.submit(task.func, *task.args, **task.kwargs): task.key for task in self.tasks}
+            try:
+                result_map = concurrent.futures.as_completed(future_to_key, timeout=self.timeout)
+                for future in result_map:
+                    key = future_to_key[future]
+                    try:
+                        self.result[key] = future.result()
+                    except Exception as err:
+                        self.error = True
+                        self.exceptions[key] = err
+            except concurrent.futures.TimeoutError:
+                self.error = True
+                self.timeout_error = True
+
+
+T = TypeVar("T")
+P = ParamSpec("P")
+
+
+def synchronized_parameter(arg_index: int = 0, skip_if_locked: bool = False) -> Callable[..., Callable[P, T | None]]:
+    locks: dict[object, Lock] = defaultdict(Lock)
+
+    def outer(func: Callable[P, T]) -> Callable[P, T | None]:
+        @functools.wraps(func)
+        def wrapper(*args: P.args, **kwargs: P.kwargs) -> T | None:
+            if skip_if_locked and locks[args[arg_index]].locked():
+                return None
+            try:
+                with locks[args[arg_index]]:
+                    return func(*args, **kwargs)
+            finally:
+                locks.pop(args[arg_index], None)
+
+        wrapper.locks = locks  # type: ignore[attr-defined]
+        return wrapper
+
+    return outer
+
+
+def synchronized(fn: Callable[P, T]) -> Callable[P, T]:
+    lock = Lock()
+
+    @functools.wraps(fn)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+        with lock:
+            return fn(*args, **kwargs)
+
+    return wrapper
+
+
+class Scheduler:
+    def __init__(self, log: Logger, loop_delay: float = 0.5, debug: bool = False):
+        self.log = log
+        self.debug = debug
+        self.loop_delay = loop_delay
+        self.stopped = False
+        self.jobs: list[Scheduler.Job] = []
+        self.run_immediately_jobs: list[Scheduler.Job] = []
+        self._debug("init")
+
+    @dataclass
+    class Job:
+        func: Func
+        args: tuple[object, ...]
+        interval: int
+        is_running: bool = False
+        last_at: datetime = field(default_factory=utc_now)
+
+        def __str__(self) -> str:
+            return str(self.func)
+
+    def add_job(self, func: Func, interval: int, args: tuple[object, ...] = (), run_immediately: bool = False) -> None:
+        job = Scheduler.Job(func, args, interval)
+        self.jobs.append(job)
+        if run_immediately:
+            self.run_immediately_jobs.append(job)
+
+    def _run_job(self, job: Job) -> None:
+        self._debug(f"_run_job: {job}")
+        if self.stopped:
+            return
+        try:
+            job.func(*job.args)
+            self._debug(f"_run_job: {job} done")
+        except Exception as e:
+            self.log.exception("scheduler error: %s", str(e))
+            self._debug(f"_run_job: {job} error")
+        finally:
+            job.is_running = False
+
+    def _start(self) -> None:
+        self._debug(f"_start: jobs={len(self.jobs)}, run_immediately_jobs={len(self.run_immediately_jobs)}")
+        for j in self.run_immediately_jobs:
+            j.is_running = True
+            j.last_at = utc_now()
+            Thread(target=self._run_job, args=(j,)).start()
+        while not self.stopped:
+            for j in self.jobs:
+                if not j.is_running and is_too_old(j.last_at, j.interval):
+                    j.is_running = True
+                    j.last_at = utc_now()
+                    Thread(target=self._run_job, args=(j,)).start()
+            time.sleep(self.loop_delay)
+
+    def _debug(self, message: str) -> None:
+        if self.debug:
+            self.log.debug("Scheduler: %s", message)
+
+    def start(self) -> None:
+        Thread(target=self._start).start()
+
+    def stop(self) -> None:
+        self.stopped = True

mm_std-0.0.1/src/mm_std/config.py

@@ -0,0 +1,78 @@
+import io
+from pathlib import Path
+from typing import TypeVar
+
+import yaml
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from .print_ import PrintFormat, print_console, print_json, print_plain, print_table
+from .str import str_to_list
+from .zip import read_text_from_zip_archive
+
+T = TypeVar("T")
+
+
+class BaseConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    @classmethod
+    def to_list_str_validator(
+        cls,
+        v: str | list[str] | None,
+        *,
+        lower: bool = False,
+        unique: bool = False,
+        remove_comments: bool = False,
+        split_line: bool = False,
+    ) -> list[str]:
+        if v is None:
+            return []
+        if isinstance(v, str):
+            return str_to_list(v, unique=unique, remove_comments=remove_comments, split_line=split_line, lower=lower)
+        return v
+
+    @classmethod
+    def read_config(  # nosec
+        cls: type[T],
+        config_path: io.TextIOWrapper | str | Path,
+        error_print_type: PrintFormat = PrintFormat.PLAIN,
+        zip_password: str = "",
+    ) -> T:
+        try:
+            # is it zip archive?
+            if isinstance(config_path, str) and config_path.endswith(".zip"):
+                config_path = str(Path(config_path).expanduser())
+                return cls(**yaml.full_load(read_text_from_zip_archive(config_path, password=zip_password)))
+            if isinstance(config_path, io.TextIOWrapper) and config_path.name.endswith(".zip"):
+                config_path = str(Path(config_path.name).expanduser())
+                return cls(**yaml.full_load(read_text_from_zip_archive(config_path, password=zip_password)))
+            if isinstance(config_path, Path) and config_path.name.endswith(".zip"):
+                config_path = str(config_path.expanduser())
+                return cls(**yaml.full_load(read_text_from_zip_archive(config_path, password=zip_password)))
+
+            # plain yml file
+            if isinstance(config_path, str):
+                return cls(**yaml.full_load(Path(config_path).expanduser().read_text()))
+            elif isinstance(config_path, Path):
+                return cls(**yaml.full_load(config_path.expanduser().read_text()))
+            else:
+                return cls(**yaml.full_load(config_path))
+        except ValidationError as err:
+            print_plain("config validation errors", error_print_type)
+            json_errors = []
+            rows = []
+            for e in err.errors():
+                loc = e["loc"]
+                field = ".".join(str(lo) for lo in loc) if len(loc) > 0 else ""
+                print_plain(f"{field} {e['msg']}", error_print_type)
+                json_errors.append({field: e["msg"]})
+                rows.append([field, e["msg"]])
+            print_table("config validation errors", ["field", "message"], rows)
+            print_json({"errors": json_errors}, error_print_type)
+            exit(1)
+        except Exception as err:
+            if error_print_type == "json":
+                print_json({"exception": str(err)})
+            else:
+                print_console(f"config error: {err!s}")
+            exit(1)

mm_std-0.0.1/src/mm_std/crypto.py

@@ -0,0 +1,13 @@
+from cryptography.fernet import Fernet
+
+
+def fernet_generate_key() -> str:
+    return Fernet.generate_key().decode()
+
+
+def fernet_encrypt(*, data: str, key: str) -> str:
+    return Fernet(key).encrypt(data.encode()).decode()
+
+
+def fernet_decrypt(*, encoded_data: str, key: str) -> str:
+    return Fernet(key).decrypt(encoded_data).decode()

mm_std-0.0.1/src/mm_std/date.py

@@ -0,0 +1,48 @@
+import random
+from datetime import UTC, datetime, timedelta
+
+from dateutil import parser
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+def utc_delta(
+    *,
+    days: int | None = None,
+    hours: int | None = None,
+    minutes: int | None = None,
+    seconds: int | None = None,
+) -> datetime:
+    params = {}
+    if days:
+        params["days"] = days
+    if hours:
+        params["hours"] = hours
+    if minutes:
+        params["minutes"] = minutes
+    if seconds:
+        params["seconds"] = seconds
+    return datetime.now(UTC) + timedelta(**params)
+
+
+def parse_date(value: str, ignore_tz: bool = False) -> datetime:
+    return parser.parse(value, ignoretz=ignore_tz)
+
+
+def utc_random(
+    *,
+    from_time: datetime | None = None,
+    range_hours: int = 0,
+    range_minutes: int = 0,
+    range_seconds: int = 0,
+) -> datetime:
+    if from_time is None:
+        from_time = utc_now()
+    to_time = from_time + timedelta(hours=range_hours, minutes=range_minutes, seconds=range_seconds)
+    return from_time + (to_time - from_time) * random.random()
+
+
+def is_too_old(value: datetime | None, seconds: int) -> bool:
+    return value is None or value < utc_delta(seconds=-1 * seconds)

mm_std-0.0.1/src/mm_std/dict.py

@@ -0,0 +1,4 @@
+def replace_empty_values(data: dict[object, object], defaults: dict[object, object]) -> None:
+    for k, v in defaults.items():
+        if not data.get(k):
+            data[k] = v

mm_std-0.0.1/src/mm_std/env.py

@@ -0,0 +1,9 @@
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def get_dotenv(key: str) -> str | None:
+    return os.getenv(key)

mm_std-0.0.1/src/mm_std/fs.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+
+def read_text(path: str | Path) -> str:
+    if isinstance(path, str):
+        path = Path(path)
+    return path.read_text()
+
+
+def get_filename_without_extension(path: str | Path) -> str:
+    if isinstance(path, str):
+        path = Path(path)
+    return path.stem

mm_std-0.0.1/src/mm_std/json_.py

@@ -0,0 +1,36 @@
+import json
+from dataclasses import asdict, is_dataclass
+from datetime import date, datetime
+from decimal import Decimal
+from enum import Enum
+from json import JSONEncoder
+from typing import Any
+
+from pydantic import BaseModel
+
+from mm_std.result import Err, Ok
+
+
+class CustomJSONEncoder(JSONEncoder):
+    def default(self, o: Any) -> Any:
+        if isinstance(o, Ok):
+            return {"ok": o.ok}
+        if isinstance(o, Err):
+            return {"err": o.err}
+        if isinstance(o, Decimal):
+            return str(o)
+        if isinstance(o, datetime | date):
+            return o.isoformat()
+        if is_dataclass(o) and not isinstance(o, type):
+            return asdict(o)
+        if isinstance(o, Enum):
+            return o.value
+        if isinstance(o, BaseModel):
+            return o.model_dump()
+        if isinstance(o, Exception):
+            return str(o)
+        return JSONEncoder.default(self, o)
+
+
+def json_dumps(data: object) -> str:
+    return json.dumps(data, cls=CustomJSONEncoder)

mm_std-0.0.1/src/mm_std/log.py

@@ -0,0 +1,22 @@
+import logging
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+
+def init_logger(name: str, file_path: str | None = None, file_mkdir: bool = True, level: int = logging.DEBUG) -> logging.Logger:
+    log = logging.getLogger(name)
+    log.setLevel(level)
+    log.propagate = False
+    fmt = logging.Formatter(fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.DEBUG)
+    console_handler.setFormatter(fmt)
+    log.addHandler(console_handler)
+    if file_path:
+        if file_mkdir:
+            Path(file_path).parent.mkdir(exist_ok=True)
+        file_handler = RotatingFileHandler(file_path, maxBytes=10 * 1024 * 1024, backupCount=1)
+        file_handler.setLevel(logging.INFO)
+        file_handler.setFormatter(fmt)
+        log.addHandler(file_handler)
+    return log