tfmx 0.7__tar.gz

tfmx-0.7/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2025 Hansimov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tfmx-0.7/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: tfmx
+Version: 0.7
+Summary: Serve Transformers in one line
+Author: Hansimov
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Hansimov/tfmx
+Project-URL: Issues, https://github.com/Hansimov/tfmx/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: tclogger
+Requires-Dist: numpy
+Requires-Dist: requests
+Requires-Dist: fastapi
+Requires-Dist: uvicorn
+Requires-Dist: httpx
+Requires-Dist: webu
+Requires-Dist: orjson
+Dynamic: license-file
+
+# tfmx
+
+![](https://img.shields.io/pypi/v/tfmx?label=tfmx&color=blue&cacheSeconds=60)
+
+## Install
+
+```sh
+pip install tfmx --upgrade
+```
+
+## Commands
+
+Set GPU control state:
+
+```sh
+gpu_fan -cs a:1
+```
+
+Set GPU power limit:
+
+```sh
+# M-X GPU-0/1
+gpu_pow -pm a:1 && gpu_pow -pl "0:160;1:240"
+
+# M-A GPU-0/1
+gpu_pow -pm a:1 && gpu_pow -pl "0,1:160"
+```
+
+Set GPU fan speed:
+
+```sh
+gpu_fan -cs a:1 && gpu_fan -fs a:100
+```
+
+Set GPU monitor with curve:
+
+```sh
+# gpu_mon -c "a:30-50/50-65/60-80/75-100;3,7:25-100" -s
+```
+
+Run tei compose and machine:
+
+```sh
+tei_compose up && sleep 45 && tei_machine run --perf-track
+```
+
+Run tei benchmark:
+
+```sh
+tei_benchmark -E "http://localhost:28800" -n 1000000 run
+```

tfmx-0.7/README.md

@@ -0,0 +1,51 @@
+# tfmx
+
+![](https://img.shields.io/pypi/v/tfmx?label=tfmx&color=blue&cacheSeconds=60)
+
+## Install
+
+```sh
+pip install tfmx --upgrade
+```
+
+## Commands
+
+Set GPU control state:
+
+```sh
+gpu_fan -cs a:1
+```
+
+Set GPU power limit:
+
+```sh
+# M-X GPU-0/1
+gpu_pow -pm a:1 && gpu_pow -pl "0:160;1:240"
+
+# M-A GPU-0/1
+gpu_pow -pm a:1 && gpu_pow -pl "0,1:160"
+```
+
+Set GPU fan speed:
+
+```sh
+gpu_fan -cs a:1 && gpu_fan -fs a:100
+```
+
+Set GPU monitor with curve:
+
+```sh
+# gpu_mon -c "a:30-50/50-65/60-80/75-100;3,7:25-100" -s
+```
+
+Run tei compose and machine:
+
+```sh
+tei_compose up && sleep 45 && tei_machine run --perf-track
+```
+
+Run tei benchmark:
+
+```sh
+tei_benchmark -E "http://localhost:28800" -n 1000000 run
+```

tfmx-0.7/pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=62"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tfmx"
+version = "0.7"
+authors = [
+    { name="Hansimov" },
+]
+description = "Serve Transformers in one line"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.6"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = [ "tclogger", "numpy", "requests", "fastapi", "uvicorn", "httpx", "webu", "orjson" ]
+
+[project.urls]
+Homepage = "https://github.com/Hansimov/tfmx"
+Issues = "https://github.com/Hansimov/tfmx/issues"
+
+[project.scripts]
+gpu_fan = "tfmx.gpu_fan:main"
+gpu_mon = "tfmx.gpu_mon:main"
+gpu_pow = "tfmx.gpu_pow:main"
+tei_compose = "tfmx.tei_compose:main"
+tei_machine = "tfmx.tei_machine:main"
+tei_client = "tfmx.tei_client:main"
+tei_clients = "tfmx.tei_clients:main"
+tei_clients_stats = "tfmx.tei_clients_stats:main"
+tei_benchmark = "tfmx.tei_benchmark:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"tfmx" = [
+    "config_qwen3_embedding_06b.json",
+    "config_sentence_transformers.json",
+]

tfmx-0.7/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tfmx-0.7/src/tfmx/__init__.py

@@ -0,0 +1,51 @@
+from .llm import LLMConfigsType, LLMClient, LLMClientByConfig
+from .vector_utils import floats_to_bits, bits_to_hash, bits_dist, hash_dist
+from .vector_utils import bits_sim, hash_sim, dot_sim
+from .embed_client import EmbedClientConfigsType
+from .embed_client import EmbedClient, EmbedClientByConfig
+from .embed_server import TEIEmbedServerConfigsType
+from .embed_server import TEIEmbedServer, TEIEmbedServerByConfig
+from .embed_server import EmbedServerArgParser
+from .tei_compose import TEIComposer, TEIComposeArgParser
+from .tei_compose import GPUInfo, GPUDetector
+from .tei_compose import ModelConfigManager, DockerImageManager, ComposeFileGenerator
+from .tei_client import TEIClient, AsyncTEIClient, TEIClientArgParser
+from .tei_client import HealthResponse, InfoResponse, InstanceInfo, MachineStats
+from .tei_clients_core import (
+    MachineState,
+    MachineScheduler,
+    ClientsHealthResponse,
+    IteratorBuffer,
+)
+from .tei_clients import TEIClients
+from .tei_clients_stats import TEIClientsWithStats
+from .tei_clients_cli import TEIClientsArgParserBase, TEIClientsCLIBase
+from .tei_performance import (
+    ExplorationConfig,
+    PerformanceTracker,
+    PerformanceMetrics,
+    ExplorationState,
+)
+from .tei_scheduler import (
+    WorkerState,
+    IdleFillingScheduler,
+    DistributionResult,
+    distribute_with_scheduler,
+    distribute_with_pipeline,
+    distribute_with_adaptive_pipeline,
+    distribute_to_workers,
+    MAX_CLIENT_BATCH_SIZE,
+)
+from .perf_tracker import (
+    PerfTracker,
+    WorkerEvent,
+    TaskRecord,
+    RoundRecord,
+    WorkerStats,
+    RoundContext,
+    TaskContext,
+    get_global_tracker,
+    reset_global_tracker,
+)
+from .gpu_fan import NvidiaSettingsParser, GPUFanController, GPUFanArgParser
+from .gpu_pow import NvidiaSmiParser, GPUPowerController, GPUPowerArgParser

tfmx-0.7/src/tfmx/config_qwen3_embedding_06b.json

@@ -0,0 +1,30 @@
+{
+    "architectures": [
+        "Qwen3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "eos_token_id": 151643,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "model_type": "qwen3",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000,
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.51.3",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151669
+}

tfmx-0.7/src/tfmx/config_sentence_transformers.json

@@ -0,0 +1 @@
+{}

tfmx-0.7/src/tfmx/embed_client.py

@@ -0,0 +1,88 @@
+import numpy as np
+import requests
+
+from tclogger import StrsType, logger
+from typing import Literal, TypedDict, Optional, Union
+
+EmbedApiFormat = Literal["openai", "tei"]
+EmbedResFormat = Literal["ndarray", "list2d"]
+
+
+class EmbedClientConfigsType(TypedDict):
+    endpoint: str
+    api_key: Optional[str]
+    model: str
+    api_format: EmbedApiFormat = "tei"
+    res_format: EmbedResFormat = "list2d"
+    batch_size: int = 100
+
+
+class EmbedClient:
+    def __init__(
+        self,
+        endpoint: str,
+        model: str = None,
+        api_key: str = None,
+        api_format: EmbedApiFormat = "tei",
+        res_format: EmbedResFormat = "list2d",
+        batch_size: int = 100,
+        verbose: bool = False,
+    ):
+        self.endpoint = endpoint
+        self.model = model
+        self.api_key = api_key
+        self.api_format = api_format
+        self.res_format = res_format
+        self.batch_size = batch_size
+        self.verbose = verbose
+
+    def log_resp_status(self, resp: requests.Response):
+        err_mesg = f"× Embed error: {resp.status_code} {resp.text}"
+        if self.verbose:
+            logger.warn(err_mesg)
+        raise ValueError(err_mesg)
+
+    def log_embed_res(self, embeddings: list[list[float]]):
+        if self.verbose:
+            num = len(embeddings)
+            dim = len(embeddings[0]) if num > 0 else 0
+            val_type = type(embeddings[0][0]).__name__ if dim > 0 else "N/A"
+            logger.okay(f"✓ Embed success: num={num}, dim={dim}, type={val_type}")
+
+    def embed_batch(self, inputs: StrsType) -> list[list[float]]:
+        headers = {
+            "content-type": "application/json",
+        }
+        # https://huggingface.github.io/text-embeddings-inference/#/Text%20Embeddings%20Inference/embed
+        payload = {
+            "inputs": inputs,
+            "normalize": True,
+            "truncate": True,
+        }
+        resp = requests.post(self.endpoint, headers=headers, json=payload)
+        if resp.status_code != 200:
+            self.log_resp_status(resp)
+            return []
+        embeddings = resp.json()
+        self.log_embed_res(embeddings)
+        return embeddings
+
+    def embed(self, inputs: StrsType) -> Union[list[list[float]], np.ndarray]:
+        if isinstance(inputs, str):
+            inputs = [inputs]
+        if len(inputs) <= self.batch_size:
+            return self.embed_batch(inputs)
+        embs = []
+        for i in range(0, len(inputs), self.batch_size):
+            inputs_batch = inputs[i : i + self.batch_size]
+            embs_batch = self.embed_batch(inputs_batch)
+            embs.extend(embs_batch)
+        if self.res_format == "ndarray":
+            return np.array(embs)
+        else:
+            return embs
+
+
+class EmbedClientByConfig(EmbedClient):
+    def __init__(self, configs: EmbedClientConfigsType):
+        super().__init__(**configs)

tfmx-0.7/src/tfmx/embed_server.py

@@ -0,0 +1,134 @@
+import argparse
+import shlex
+
+from pathlib import Path
+from tclogger import logger, shell_cmd
+from typing import TypedDict
+
+
+class TEIEmbedServerConfigsType(TypedDict):
+    port: int
+    model_name: str
+    instance_id: str
+    verbose: bool
+
+
+class TEIEmbedServer:
+    def __init__(
+        self,
+        port: int = None,
+        model_name: str = None,
+        instance_id: str = None,
+        hf_token: str = None,
+        verbose: bool = False,
+    ):
+        self.port = port
+        self.model_name = model_name
+        self.instance_id = instance_id or self.default_instance_id()
+        self.hf_token = hf_token
+        self.verbose = verbose
+
+    def default_instance_id(self) -> str:
+        return "tei--" + self.model_name.replace("/", "--")
+
+    def run(self):
+        script_path = Path(__file__).resolve().parent / "run_tei.sh"
+        if not script_path.exists():
+            logger.warn(f"× Missing `run_tei.sh`: {script_path}")
+            return
+
+        run_parts = ["bash", str(script_path)]
+        if self.port:
+            run_parts.extend(["-p", str(self.port)])
+        if self.model_name:
+            run_parts.extend(["-m", self.model_name])
+        if self.instance_id:
+            run_parts.extend(["-id", self.instance_id])
+        if self.hf_token:
+            run_parts.extend(["-u", self.hf_token])
+        cmd_run = shlex.join(run_parts)
+        shell_cmd(cmd_run)
+
+        if self.verbose:
+            cmd_logs = f'docker logs -f --tail 0 "{self.instance_id}"'
+            shell_cmd(cmd_logs)
+
+    def kill(self):
+        if not self.instance_id:
+            logger.warn("× Missing arg: -id (--instance-id)")
+            return
+
+        cmd_kill = f'docker stop "{self.instance_id}"'
+        shell_cmd(cmd_kill)
+
+    def remove(self):
+        if not self.instance_id:
+            logger.warn("× Missing arg: -id (--instance-id)")
+            return
+
+        cmd_rm = f'docker rm -f "{self.instance_id}"'
+        shell_cmd(cmd_rm)
+
+
+class TEIEmbedServerByConfig(TEIEmbedServer):
+    def __init__(self, configs: TEIEmbedServerConfigsType):
+        super().__init__(**configs)
+
+
+class TEIEmbedServerArgParser(argparse.ArgumentParser):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.add_argument("-m", "--model-name", type=str, default=None)
+        self.add_argument("-id", "--instance-id", type=str, default=None)
+        self.add_argument("-u", "--hf-token", type=str, default=None)
+        self.add_argument("-p", "--port", type=int, default=28888)
+        self.add_argument("-b", "--verbose", action="store_true")
+        self.add_argument("-rm", "--remove", action="store_true")
+        self.add_argument("-k", "--kill", action="store_true")
+        self.args, _ = self.parse_known_args()
+
+
+class EmbedServerArgParser(argparse.ArgumentParser):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.add_argument("-t", "--type", type=str, choices=["tei"], default="tei")
+        self.args, _ = self.parse_known_args()
+
+
+def main():
+    main_args = EmbedServerArgParser().args
+    if main_args.type == "tei":
+        args = TEIEmbedServerArgParser().args
+        embed_server = TEIEmbedServer(
+            port=args.port,
+            model_name=args.model_name,
+            instance_id=args.instance_id,
+            hf_token=args.hf_token,
+            verbose=args.verbose,
+        )
+        if args.remove:
+            embed_server.remove()
+        elif args.kill:
+            embed_server.kill()
+        else:
+            embed_server.run()
+
+
+if __name__ == "__main__":
+    main()
+
+    # Case 1: gte-multilingual-base
+    # python -m tfmx.embed_server -t "tei" -m "Alibaba-NLP/gte-multilingual-base" -p 28888 -b
+    # python -m tfmx.embed_server -t "tei" -m "Alibaba-NLP/gte-multilingual-base" -k
+
+    # Case 2: bge-large-zh-v1.5
+    # python -m tfmx.embed_server -t "tei" -m "BAAI/bge-large-zh-v1.5" -p 28889 -b
+    # python -m tfmx.embed_server -t "tei" -m "BAAI/bge-large-zh-v1.5" -k
+
+    # Case 3: Qwen/Qwen3-Embedding-0.6B
+    # python -m tfmx.embed_server -t "tei" -m "Qwen/Qwen3-Embedding-0.6B" -p 28887 -b
+    # python -m tfmx.embed_server -t "tei" -m "Qwen/Qwen3-Embedding-0.6B" -k
+    # python -m tfmx.embed_server -t "tei" -m "Qwen/Qwen3-Embedding-0.6B" -rm
+
+    # Case 4: Use HF_TOKEN
+    # python -m tfmx.embed_server -t "tei" -m "Qwen/Qwen3-Embedding-0.6B" -p 28887 -b -u hf_****Y