lalamo 0.5.16__py3-none-any.whl → 0.5.17__py3-none-any.whl

lalamo/__init__.py

@@ -1,4 +1,14 @@
-from lalamo.main import collect_traces, convert, estimate_batchsize, train
+from lalamo.commands import (
+    CollectTracesCallbacks,
+    ConversionCallbacks,
+    EstimateBatchsizeCallbacks,
+    Precision,
+    TrainCallbacks,
+    collect_traces,
+    convert,
+    estimate_batchsize,
+    train,
+)
 from lalamo.message_processor import (
     AssistantMessage,
     ContentBlock,
@@ -9,27 +19,41 @@ from lalamo.message_processor import (
     UserMessage,
 )
 from lalamo.model_import import ModelSpec, import_model
+from lalamo.model_import.model_specs.common import ConfigMap, FileSpec, JSONFieldSpec, ModelType, UseCase, WeightsType
 from lalamo.models import ClassifierModel, LanguageModel
+from lalamo.quantization import QuantizationMode
 from lalamo.speculator import (
     CollectTracesEvent,
     SpeculatorTrainingEvent,
 )
 
-__version__ = "0.5.16"
+__version__ = "0.5.17"
 
 __all__ = [
     "AssistantMessage",
     "ClassifierModel",
+    "CollectTracesCallbacks",
     "CollectTracesEvent",
+    "ConfigMap",
     "ContentBlock",
+    "ConversionCallbacks",
+    "EstimateBatchsizeCallbacks",
+    "FileSpec",
     "Image",
+    "JSONFieldSpec",
     "LanguageModel",
     "Message",
     "ModelSpec",
+    "ModelType",
+    "Precision",
+    "QuantizationMode",
     "SpeculatorTrainingEvent",
     "SystemMessage",
     "ToolSchema",
+    "TrainCallbacks",
+    "UseCase",
     "UserMessage",
+    "WeightsType",
     "collect_traces",
     "convert",
     "estimate_batchsize",

lalamo/commands.py

@@ -0,0 +1,377 @@
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import Enum
+from itertools import chain
+from pathlib import Path
+
+from jaxtyping import DTypeLike
+
+from lalamo.common import flatten_parameters
+from lalamo.data import import_hf_parquet
+from lalamo.data.lalamo_completions import LalamoCompletion
+from lalamo.message_processor import UserMessage
+from lalamo.model_import import ModelMetadata, ModelSpec, import_model
+from lalamo.model_import.common import (
+    DownloadingFileEvent,
+    FileSpec,
+    FinishedDownloadingFileEvent,
+    FinishedInitializingModelEvent,
+    InitializingModelEvent,
+    StatusEvent,
+)
+from lalamo.models import LanguageModelConfig
+from lalamo.modules import config_converter
+from lalamo.safetensors import safe_write
+from lalamo.speculator.estimator import EstimateBatchsizeFromMemoryEvent, estimate_batchsize_from_memory
+from lalamo.speculator.inference import CollectTracesEvent, inference_collect_traces
+from lalamo.speculator.ngram import NGramSpeculator
+from lalamo.speculator.utils import SpeculatorTrainingEvent, train_speculator
+
+
+class Precision(Enum):
+    FLOAT32 = "float32"
+    FLOAT16 = "float16"
+    BFLOAT16 = "bfloat16"
+
+
+@dataclass
+class ConversionCallbacks:
+    model_spec: ModelSpec
+    output_dir: Path
+    precision: Precision | None
+    context_length: int | None
+    include_traces: bool
+    message_for_trace: str | None
+
+    def started(self) -> None:
+        pass
+
+    def output_dir_exists(self) -> None:
+        raise RuntimeError(f"{self.output_dir=} already exists, refusing to overwrite!")
+
+    def downloading(self, file_spec: FileSpec) -> None:
+        pass
+
+    def finished_downloading(self, file_spec: FileSpec) -> None:
+        pass
+
+    def initializing_model(self) -> None:
+        pass
+
+    def finished_initializing_model(self) -> None:
+        pass
+
+    def saving_model(self) -> None:
+        pass
+
+    def finished_saving_model(self) -> None:
+        pass
+
+
+def convert(
+    model_spec: ModelSpec,
+    output_dir: Path,
+    precision: Precision | None = None,
+    context_length: int | None = None,
+    include_traces: bool = False,
+    message_for_trace: str | None = None,
+    callbacks_type: Callable[
+        [
+            ModelSpec,
+            Path,
+            Precision | None,
+            int | None,
+            bool,
+            str | None,
+        ],
+        ConversionCallbacks,
+    ] = ConversionCallbacks,
+) -> None:
+    callbacks = callbacks_type(
+        model_spec,
+        output_dir,
+        precision,
+        context_length,
+        include_traces,
+        message_for_trace,
+    )
+
+    if precision is not None:
+        precision_dtype = config_converter.structure(precision.value, DTypeLike)  # type: ignore
+    else:
+        precision_dtype = None
+
+    if output_dir.exists():
+        callbacks.output_dir_exists()
+
+    callbacks.started()
+
+    def progress_callback(event: StatusEvent) -> None:
+        match event:
+            case DownloadingFileEvent(file_spec):
+                callbacks.downloading(file_spec)
+            case FinishedDownloadingFileEvent(file_spec):
+                callbacks.finished_downloading(file_spec)
+            case InitializingModelEvent():
+                callbacks.initializing_model()
+            case FinishedInitializingModelEvent():
+                callbacks.finished_initializing_model()
+
+    model, metadata = import_model(
+        model_spec,
+        precision=precision_dtype,
+        context_length=context_length,
+        progress_callback=progress_callback,
+    )
+    callbacks.saving_model()
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if include_traces:
+        message = None if message_for_trace is None else [UserMessage(content=message_for_trace)]
+        result = model.record_trace(message)
+        traces = flatten_parameters(result.export())
+        with Path(output_dir / "traces.safetensors").open("wb") as fd:
+            safe_write(fd, traces)
+
+    model.message_processor.tokenizer.save(str(output_dir / "tokenizer.json"))
+    weights = flatten_parameters(model.export_weights())
+    del model
+
+    with Path(output_dir / "model.safetensors").open("wb") as fd:
+        safe_write(fd, weights)
+
+    config_json = config_converter.unstructure(metadata, ModelMetadata)
+    with open(output_dir / "config.json", "w") as file:
+        json.dump(config_json, file, indent=4)
+
+    callbacks.finished_saving_model()
+
+
+@dataclass
+class EstimateBatchsizeCallbacks:
+    model_path: Path
+    max_input_length: int
+    max_output_length: int
+    num_logits_per_token: int
+    mem: int
+
+    def loading_model(self) -> None:
+        pass
+
+    def finished_loading_model(self) -> None:
+        pass
+
+    def estimating_batchsize(self, lo: int, hi: int | None) -> None:
+        pass
+
+    def finished_estimating_batchsize(self, batchsize: int) -> None:
+        pass
+
+
+def estimate_batchsize(
+    model_path: Path,
+    mem: int,
+    max_input_length: int = 1024,
+    max_output_length: int = 1024,
+    num_logits_per_token: int = 8,
+    callbacks_type: Callable[
+        [
+            Path,
+            int,
+            int,
+            int,
+            int,
+        ],
+        EstimateBatchsizeCallbacks,
+    ] = EstimateBatchsizeCallbacks,
+) -> int:
+    callbacks = callbacks_type(model_path, max_input_length, max_output_length, num_logits_per_token, mem)
+
+    callbacks.loading_model()
+    model = LanguageModelConfig.load_model(model_path)
+    callbacks.finished_loading_model()
+
+    def progress_callback(event: EstimateBatchsizeFromMemoryEvent) -> None:
+        callbacks.estimating_batchsize(event.lo, event.hi)
+
+    bs = estimate_batchsize_from_memory(
+        model,
+        max_input_length,
+        max_output_length,
+        num_logits_per_token,
+        mem,
+        progress_callback,
+    )
+
+    callbacks.finished_estimating_batchsize(bs)
+    return bs
+
+
+@dataclass
+class CollectTracesCallbacks:
+    model_path: Path
+    dataset_path: Path
+    output_path: Path
+    num_logits_per_token: int
+    max_input_length: int
+    max_output_length: int
+    batch_size: int
+    num_tokens_to_generate: int | None
+
+    def loading_model(self) -> None:
+        pass
+
+    def finished_loading_model(self) -> None:
+        pass
+
+    def loading_dataset(self) -> None:
+        pass
+
+    def finished_loading_dataset(self) -> None:
+        pass
+
+    def inference_progress(self, tokens_generated: int) -> None:
+        pass
+
+    def finished_inference(self) -> None:
+        pass
+
+
+def collect_traces(
+    model_path: Path,
+    dataset_path: Path,
+    output_path: Path,
+    num_logits_per_token: int = 8,
+    max_input_length: int = 1024,
+    max_output_length: int = 1024,
+    batch_size: int = 1,
+    num_tokens_to_generate: int | None = None,
+    callbacks_type: Callable[
+        [
+            Path,
+            Path,
+            Path,
+            int,
+            int,
+            int,
+            int,
+            int | None,
+        ],
+        CollectTracesCallbacks,
+    ] = CollectTracesCallbacks,
+) -> None:
+    callbacks = callbacks_type(
+        model_path,
+        dataset_path,
+        output_path,
+        num_logits_per_token,
+        max_input_length,
+        max_output_length,
+        batch_size,
+        num_tokens_to_generate,
+    )
+
+    callbacks.loading_model()
+    model = LanguageModelConfig.load_model(model_path)
+    callbacks.finished_loading_model()
+
+    callbacks.loading_dataset()
+    dataset = iter(import_hf_parquet(dataset_path))
+    dataset = chain([next(dataset)], dataset)  # iterator is lazy, force it to actually open the file
+    callbacks.finished_loading_dataset()
+
+    def progress_callback(event: CollectTracesEvent) -> None:
+        callbacks.inference_progress(event.tokens_generated)
+
+    traces = inference_collect_traces(
+        model,
+        dataset,
+        num_logits_per_token,
+        batch_size,
+        max_input_length,
+        max_output_length,
+        num_tokens_to_generate,
+        progress_callback,
+    )
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "wb") as output_fd:
+        for trace in traces:
+            blob = trace.serialize()
+            output_fd.write(blob)
+
+    callbacks.finished_inference()
+
+
+@dataclass
+class TrainCallbacks:
+    trace_path: Path
+    output_path: Path
+    hashtable_size: int
+    num_logits_per_token: int
+    ngram_size: int
+    subsample_size: int | None
+
+    def started(self) -> None:
+        pass
+
+    def training_progress(self, trained_tokens: int) -> None:
+        pass
+
+    def finished_training(self) -> None:
+        pass
+
+    def saving_speculator(self) -> None:
+        pass
+
+    def finished_saving_speculator(self) -> None:
+        pass
+
+
+def train(
+    trace_path: Path,
+    output_path: Path,
+    hashtable_size: int = 65536,
+    num_logits_per_token: int = 8,
+    ngram_size: int = 2,
+    subsample_size: int | None = None,
+    callbacks_type: Callable[
+        [
+            Path,
+            Path,
+            int,
+            int,
+            int,
+            int | None,
+        ],
+        TrainCallbacks,
+    ] = TrainCallbacks,
+) -> None:
+    callbacks = callbacks_type(
+        trace_path,
+        output_path,
+        hashtable_size,
+        num_logits_per_token,
+        ngram_size,
+        subsample_size,
+    )
+
+    callbacks.started()
+
+    with open(trace_path, "rb") as trace_fd:
+        traces = LalamoCompletion.deserialize_many(trace_fd)
+        speculator = NGramSpeculator.new(hashtable_size, num_logits_per_token, ngram_size)
+
+        def progress_callback(event: SpeculatorTrainingEvent) -> None:
+            callbacks.training_progress(event.trained_tokens)
+
+        train_speculator(speculator, traces, subsample_size, progress_callback)
+
+    callbacks.finished_training()
+
+    callbacks.saving_speculator()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(output_path, "wb") as fd:
+        fd.write(speculator.serialize())
+    callbacks.finished_saving_speculator()

lalamo/main.py

@@ -1,20 +1,19 @@
-import json
 import random
 import re
 import shutil
 import sys
-from enum import Enum
-from itertools import chain, islice
+from contextlib import ExitStack
+from dataclasses import dataclass, field
+from functools import partial
+from itertools import islice
 from pathlib import Path
 from typing import Annotated
 
-import jax
 import jax.profiler
 import thefuzz.process
 from click import Context as ClickContext
 from click import Parameter as ClickParameter
 from click import ParamType
-from jaxtyping import DTypeLike
 from rich import box
 from rich.console import Console
 from rich.live import Live
@@ -23,48 +22,40 @@ from rich.progress import (
     MofNCompleteColumn,
     Progress,
     SpinnerColumn,
+    TaskID,
     TextColumn,
     TimeElapsedColumn,
     TimeRemainingColumn,
 )
+from rich.prompt import Confirm
 from rich.table import Table
-from safetensors.flax import save_file
 from typer import Argument, Context, Exit, Option, Typer
 
-from lalamo.common import flatten_parameters
-from lalamo.data import import_hf_parquet
+from lalamo.commands import (
+    CollectTracesCallbacks,
+    ConversionCallbacks,
+    EstimateBatchsizeCallbacks,
+    Precision,
+    TrainCallbacks,
+)
+from lalamo.commands import collect_traces as _collect_traces
+from lalamo.commands import convert as _convert
+from lalamo.commands import estimate_batchsize as _estimate_batchsize
+from lalamo.commands import train as _train
 from lalamo.data.lalamo_completions import LalamoCompletion
 from lalamo.message_processor import UserMessage
-from lalamo.model_import import REPO_TO_MODEL, ModelMetadata, ModelSpec, import_model
-from lalamo.model_import.common import (
-    DownloadingFileEvent,
-    FinishedDownloadingFileEvent,
-    FinishedInitializingModelEvent,
-    InitializingModelEvent,
-    StatusEvent,
-)
+from lalamo.model_import import REPO_TO_MODEL, ModelSpec
+from lalamo.model_import.common import FileSpec
 from lalamo.models import ClassifierModelConfig, LanguageModelConfig
-from lalamo.modules import config_converter
-from lalamo.speculator.estimator import EstimateBatchsizeFromMemoryEvent, estimate_batchsize_from_memory
-from lalamo.speculator.inference import CollectTracesEvent, inference_collect_traces
+from lalamo.speculator.estimator import get_default_device_memory
 from lalamo.speculator.ngram import NGramSpeculator
-from lalamo.speculator.utils import (
-    SpeculatorTrainingEvent,
-    test_speculator,
-    train_speculator,
-)
+from lalamo.speculator.utils import test_speculator
 
 SCRIPT_NAME = Path(sys.argv[0]).name
 
 DEFAULT_OUTPUT_DIR = Path("models")
 
 
-class Precision(Enum):
-    FLOAT32 = "float32"
-    FLOAT16 = "float16"
-    BFLOAT16 = "bfloat16"
-
-
 console = Console()
 err_console = Console(stderr=True)
 app = Typer(
@@ -182,6 +173,68 @@ def classify(
         console.print()
 
 
+@dataclass
+class CliConversionCallbacks(ConversionCallbacks):
+    overwrite: bool = False
+
+    stack: ExitStack = field(default_factory=ExitStack)
+    downloading_tasks: dict[FileSpec, TaskID] = field(default_factory=dict)
+    initializing_task: TaskID | None = None
+    saving_task: TaskID | None = None
+
+    def started(self) -> None:
+        conversion_strs = [
+            f"🚀 Converting [cyan]{self.model_spec.name}[/cyan] by [cyan]{self.model_spec.vendor}[/cyan]",
+        ]
+        if self.precision is not None:
+            conversion_strs.append(
+                f" and converting floating-point weights into [cyan]{self.precision.name.lower()}[/cyan] precision",
+            )
+        conversion_strs.append(".")
+        console.print("".join(conversion_strs))
+
+        self.progress = self.stack.enter_context(
+            Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                transient=True,
+            ),
+        )
+
+    def output_dir_exists(self) -> None:
+        if not self.overwrite and not Confirm().ask(
+            rf"⚠️ Output directory [cyan]{self.output_dir}[/cyan] already exists."
+            r" Do you want to overwrite it?",
+        ):
+            raise Exit
+
+        shutil.rmtree(self.output_dir)
+
+    def downloading(self, file_spec: FileSpec) -> None:
+        self.downloading_tasks[file_spec] = self.progress.add_task(f"Retrieving {file_spec.filename}...")
+
+    def finished_downloading(self, file_spec: FileSpec) -> None:
+        self.progress.remove_task(self.downloading_tasks[file_spec])
+
+    def initializing_model(self) -> None:
+        self.initializing_task = self.progress.add_task("Initializing model...")
+
+    def finished_initializing_model(self) -> None:
+        assert self.initializing_task is not None
+
+        self.progress.remove_task(self.initializing_task)
+
+    def saving_model(self) -> None:
+        self.saving_task = self.progress.add_task(f"💾 Saving the model to {self.output_dir}")
+
+    def finished_saving_model(self) -> None:
+        assert self.saving_task is not None
+
+        self.progress.remove_task(self.saving_task)
+        self.stack.close()
+        console.print(f"🧑‍🍳 Model successfully cooked and saved to [cyan]`{self.output_dir}`[/cyan]!")
+
+
 @app.command(help="Convert the model for use with the Uzu inference engine.")
 def convert(
     model_repo: Annotated[
@@ -238,85 +291,18 @@ def convert(
         ),
     ] = None,
 ) -> None:
-    if precision is not None:
-        precision_dtype = config_converter.structure(precision.value, DTypeLike)  # type: ignore
-    else:
-        precision_dtype = None
-
     if output_dir is None:
         output_dir = DEFAULT_OUTPUT_DIR / model_repo.name
 
-    conversion_strs = [f"🚀 Converting [cyan]{model_repo.name}[/cyan] by [cyan]{model_repo.vendor}[/cyan]"]
-    if precision is not None:
-        conversion_strs.append(
-            f" and converting floating-point weights into [cyan]{precision.name.lower()}[/cyan] precision",
-        )
-    conversion_strs.append(".")
-    console.print("".join(conversion_strs))
-
-    if output_dir.exists() and not overwrite:
-        answer = console.input(
-            rf"⚠️ Output directory [cyan]{output_dir}[/cyan] already exists."
-            r" Do you want to overwrite it? [cyan]\[y/n][/cyan]: ",
-        )
-        while answer.lower() not in ["y", "n", "yes", "no"]:
-            answer = console.input("Please enter 'y' or 'n': ")
-        if answer.lower() in ["y", "yes"]:
-            shutil.rmtree(output_dir)
-        else:
-            console.print("Exiting...")
-            raise Exit
-
-    message = None if message_for_trace is None else [UserMessage(content=message_for_trace)]
-
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        transient=True,
-    ) as progress:
-        event_to_task = {}
-
-        def progress_callback(event: StatusEvent) -> None:
-            match event:
-                case DownloadingFileEvent(file_spec):
-                    event_to_task[event] = progress.add_task(f"Retrieving {file_spec.filename}...")
-                case FinishedDownloadingFileEvent(file_spec):
-                    progress.remove_task(event_to_task[event])
-                case InitializingModelEvent():
-                    event_to_task[event] = progress.add_task("Initializing model...")
-                case FinishedInitializingModelEvent():
-                    progress.remove_task(event_to_task[event])
-
-        main_task = progress.add_task("👨‍🍳 Cooking...")
-        model, metadata = import_model(
-            model_repo,
-            precision=precision_dtype,
-            context_length=context_length,
-            progress_callback=progress_callback,
-        )
-        save_task = progress.add_task(f"💾 Saving the model to {output_dir}")
-        output_dir.mkdir(parents=True, exist_ok=True)
-
-        if include_traces:
-            trace_task = progress.add_task("🚁 Generating traces...")
-            result = model.record_trace(message)
-            traces = flatten_parameters(result.export())
-            save_file(traces, output_dir / "traces.safetensors")
-            progress.remove_task(trace_task)
-        progress.remove_task(main_task)
-
-        model.message_processor.tokenizer.save(str(output_dir / "tokenizer.json"))
-        weights = flatten_parameters(model.export_weights())
-        del model
-
-        save_file(weights, output_dir / "model.safetensors")
-
-        config_json = config_converter.unstructure(metadata, ModelMetadata)
-        with open(output_dir / "config.json", "w") as file:
-            json.dump(config_json, file, indent=4)
-        progress.remove_task(save_task)
-
-    console.print(f"🧑‍🍳 Model successfully cooked and saved to [cyan]`{output_dir}`[/cyan]!")
+    _convert(
+        model_repo,
+        output_dir,
+        precision,
+        context_length,
+        include_traces,
+        message_for_trace,
+        partial(CliConversionCallbacks, overwrite=overwrite),
+    )
 
 
 def _model_size_string_to_int(
@@ -385,6 +371,41 @@ speculator_app = Typer()
 app.add_typer(speculator_app, name="speculator", help="Train a speculator for a model.")
 
 
+@dataclass
+class CliEstimateBatchsizeCallbacks(EstimateBatchsizeCallbacks):
+    stack: ExitStack = field(default_factory=ExitStack)
+    loading_task: TaskID | None = None
+    estimating_task: TaskID | None = None
+
+    def loading_model(self) -> None:
+        self.progress = self.stack.enter_context(
+            Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                transient=True,
+            ),
+        )
+        self.loading_task = self.progress.add_task("[cyan]Loading model...[/cyan]")
+
+    def finished_loading_model(self) -> None:
+        assert self.loading_task is not None
+        self.progress.remove_task(self.loading_task)
+
+    def estimating_batchsize(self, lo: int, hi: int | None) -> None:
+        hi_str = str(hi) if hi is not None else "?"
+        description = f"[cyan]Estimating batch size... ({lo}..{hi_str})[/cyan]"
+        if self.estimating_task is None:
+            self.estimating_task = self.progress.add_task(description)
+        else:
+            self.progress.update(self.estimating_task, description=description)
+
+    def finished_estimating_batchsize(self, batchsize: int) -> None:
+        if self.estimating_task is not None:
+            self.progress.remove_task(self.estimating_task)
+        self.stack.close()
+        console.print(f"Found maximum batch size: [cyan]{batchsize}[/cyan]")
+
+
 @speculator_app.command(help="Estimate maximum batch size at which a model can be run.")
 def estimate_batchsize(
     model_path: Annotated[
@@ -416,44 +437,64 @@ def estimate_batchsize(
 ) -> None:
     if vram_gb is not None:
         mem = vram_gb * 1024 * 1024 * 1024
-    else:
-        memory_stats = jax.local_devices()[0].memory_stats()
-        if memory_stats is None:
-            err_console.print("Cannot get the default device's memory stats, use --vram-gb")
-            raise Exit(1)
-        if "bytes_limit" not in memory_stats:
-            err_console.print("Cannot get the default device's bytes limit, use --vram-gb")
-            raise Exit(1)
-        mem = memory_stats["bytes_limit"]
+    elif (mem := get_default_device_memory()) is None:
+        err_console.print("Cannot get the default device's memory stats, use --vram-gb")
+        raise Exit(1)
 
-    with Progress(
-        SpinnerColumn(),
-        TextColumn("[progress.description]{task.description}"),
-        transient=True,
-    ) as progress:
-        loading_model_task = progress.add_task("[cyan]Loading model...[/cyan]")
-        model = LanguageModelConfig.load_model(model_path)
-        progress.remove_task(loading_model_task)
-
-        estimating_batchsize_task = progress.add_task("[cyan]Estimating batch size...[/cyan]")
-
-        def progress_callback(event: EstimateBatchsizeFromMemoryEvent) -> None:
-            lo = str(event.lo)
-            hi = str(event.hi) if event.hi is not None else "?"
-            description = f"[cyan]Estimating batch size... ({lo}..{hi})[/cyan]"
-            progress.update(estimating_batchsize_task, description=description)
-
-        bs = estimate_batchsize_from_memory(
-            model,
-            max_input_length,
-            max_output_length,
-            num_logits_per_token,
-            mem,
-            progress_callback,
+    callbacks_type = CliEstimateBatchsizeCallbacks
+
+    _estimate_batchsize(model_path, mem, max_input_length, max_output_length, num_logits_per_token, callbacks_type)
+
+
+@dataclass
+class CliCollectTracesCallbacks(CollectTracesCallbacks):
+    stack: ExitStack = field(default_factory=ExitStack)
+    live: Live | None = None
+    loading_task: TaskID | None = None
+    inference_task: TaskID | None = None
+
+    def loading_model(self) -> None:
+        self.live = self.stack.enter_context(Live(refresh_per_second=10))
+        self.progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            transient=True,
+        )
+        self.live.update(self.progress, refresh=True)
+        self.loading_task = self.progress.add_task("🧠 [cyan]Loading model...[/cyan]")
+
+    def finished_loading_model(self) -> None:
+        assert self.loading_task is not None
+        self.progress.remove_task(self.loading_task)
+
+    def loading_dataset(self) -> None:
+        self.loading_task = self.progress.add_task("🗂️ [cyan]Loading dataset...[/cyan]")
+
+    def finished_loading_dataset(self) -> None:
+        assert self.loading_task is not None
+        assert self.live is not None
+        self.progress.remove_task(self.loading_task)
+        self.progress = Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            MofNCompleteColumn(),
+            TimeElapsedColumn(),
+            TimeRemainingColumn(),
         )
-        progress.remove_task(estimating_batchsize_task)
+        self.live.update(self.progress, refresh=True)
+        self.inference_task = self.progress.add_task(
+            "🔮 [cyan]Running inference...[/cyan]",
+            total=self.num_tokens_to_generate,
+        )
+
+    def inference_progress(self, tokens_generated: int) -> None:
+        assert self.inference_task is not None
+        self.progress.update(self.inference_task, completed=tokens_generated)
 
-    console.print(f"Found maximum batch size: [cyan]{bs}[/cyan]")
+    def finished_inference(self) -> None:
+        assert self.inference_task is not None
+        self.progress.update(self.inference_task, description="✅ Completed")
+        self.stack.close()
 
 
 @speculator_app.command(help="Run model inference and collect traces for speculator training")
@@ -503,55 +544,17 @@ def collect_traces(
         ),
     ] = None,
 ) -> None:
-    with Live(refresh_per_second=10) as live:
-        with Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            transient=True,
-            disable=True,
-        ) as progress:
-            live.update(progress, refresh=True)
-            loading_model_task = progress.add_task("🧠 [cyan]Loading model...[/cyan]")
-            model = LanguageModelConfig.load_model(model_path)
-            progress.remove_task(loading_model_task)
-
-            loading_dataset_task = progress.add_task("🗂️ [cyan]Loading dataset...[/cyan]")
-            dataset = iter(import_hf_parquet(dataset_path))
-            dataset = chain([next(dataset)], dataset)  # iterator is lazy, force it to actually open the file
-            progress.remove_task(loading_dataset_task)
-
-        with Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            MofNCompleteColumn(),
-            TimeElapsedColumn(),
-            TimeRemainingColumn(),
-            disable=True,
-        ) as progress:
-            live.update(progress, refresh=True)
-            inference_task = progress.add_task("🔮 [cyan]Running inference...[/cyan]", total=num_tokens_to_generate)
-
-            def progress_callback(event: CollectTracesEvent) -> None:
-                progress.update(inference_task, completed=event.tokens_generated)
-
-            traces = inference_collect_traces(
-                model,
-                dataset,
-                num_logits_per_token,
-                batch_size,
-                max_input_length,
-                max_output_length,
-                num_tokens_to_generate,
-                progress_callback,
-            )
-
-            output_path.parent.mkdir(parents=True, exist_ok=True)
-            with open(output_path, "wb+") as output_fd:
-                for trace in traces:
-                    blob = trace.serialize()
-                    output_fd.write(blob)
-
-            progress.update(inference_task, description="✅ Completed")
+    _collect_traces(
+        model_path,
+        dataset_path,
+        output_path,
+        num_logits_per_token,
+        max_input_length,
+        max_output_length,
+        batch_size,
+        num_tokens_to_generate,
+        CliCollectTracesCallbacks,
+    )
 
 
 @speculator_app.command(help="View model inference traces")
@@ -597,6 +600,43 @@ def view_traces(
         console.print(table)
 
 
+@dataclass
+class CliTrainCallbacks(TrainCallbacks):
+    stack: ExitStack = field(default_factory=ExitStack)
+    training_task: TaskID | None = None
+
+    def started(self) -> None:
+        self.progress = self.stack.enter_context(
+            Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+                TimeRemainingColumn(),
+            ),
+        )
+        self.training_task = self.progress.add_task(
+            "🔮 [cyan]Training speculator...[/cyan]",
+            total=self.subsample_size,
+        )
+
+    def training_progress(self, trained_tokens: int) -> None:
+        assert self.training_task is not None
+        self.progress.update(self.training_task, completed=trained_tokens)
+
+    def finished_training(self) -> None:
+        assert self.training_task is not None
+        self.progress.update(self.training_task, description="✅ Completed")
+        self.progress.remove_task(self.training_task)
+        self.stack.close()
+
+    def saving_speculator(self) -> None:
+        pass
+
+    def finished_saving_speculator(self) -> None:
+        console.print(f"💾 Speculator saved to [cyan]{self.output_path}[/cyan]")
+
+
 @speculator_app.command(help="Train a speculator from inference traces")
 def train(
     trace_path: Annotated[
@@ -633,30 +673,15 @@ def train(
         ),
     ] = None,
 ) -> None:
-    with open(trace_path, "rb") as trace_fd:
-        traces = LalamoCompletion.deserialize_many(trace_fd)
-
-        speculator = NGramSpeculator.new(hashtable_size, num_logits_per_token, ngram_size)
-
-        with Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            MofNCompleteColumn(),
-            TimeElapsedColumn(),
-            TimeRemainingColumn(),
-        ) as progress:
-            inference_task = progress.add_task("🔮 [cyan]Training speculator...[/cyan]", total=subsample_size)
-
-            def progress_callback(event: SpeculatorTrainingEvent) -> None:
-                progress.update(inference_task, completed=event.trained_tokens)
-
-            train_speculator(speculator, traces, subsample_size, progress_callback)
-
-            progress.update(inference_task, description="✅ Completed")
-
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    with open(output_path, "wb+") as fd:
-        fd.write(speculator.serialize())
+    _train(
+        trace_path,
+        output_path,
+        hashtable_size,
+        num_logits_per_token,
+        ngram_size,
+        subsample_size,
+        CliTrainCallbacks,
+    )
 
 
 @speculator_app.command(help="Run speculator as an autoregressive llm")

lalamo/model_import/model_specs/common.py

@@ -15,7 +15,8 @@ from jaxtyping import Array, DTypeLike
 
 from lalamo.model_import.decoder_configs import ForeignConfig
 from lalamo.quantization import QuantizationMode
-from lalamo.utils import MapDictValues, open_safetensors
+from lalamo.safetensors import safe_read
+from lalamo.utils import MapDictValues
 
 __all__ = [
     "ConfigMap",
@@ -52,7 +53,8 @@ class WeightsType(Enum):
         float_dtype: DTypeLike,
     ) -> Iterator[tuple[Mapping[str, jnp.ndarray], Mapping[str, str]]]:
         if self == WeightsType.SAFETENSORS:
-            with open_safetensors(filename) as (weights_dict, metadata_dict):
+            with Path(filename).open("rb") as fd:
+                (metadata_dict, weights_dict) = safe_read(fd)
                 yield MapDictValues(lambda v: cast_if_float(v, float_dtype), weights_dict), metadata_dict or {}
         else:
             import torch

lalamo/models/common.py

@@ -15,7 +15,7 @@ from lalamo.message_processor import Message, MessageProcessor, MessageProcessor
 from lalamo.modules import Classifier, Decoder, LalamoModule, config_converter
 from lalamo.modules.classifier import ClassifierConfig, ClassifierResult
 from lalamo.modules.decoder import DecoderConfig, DecoderResult
-from lalamo.utils import open_safetensors
+from lalamo.safetensors import safe_read
 
 __all__ = [
     "TextModel",
@@ -42,8 +42,8 @@ class TextModelConfig[ConfigT: ClassifierConfig | DecoderConfig](ABC):
         with open(path / "config.json") as config_file:
             config_json = json.load(config_file)
         config = config_converter.structure(config_json["model_config"], cls)
-        with open_safetensors(path / "model.safetensors") as open_results:
-            weights_dict, _ = open_results
+        with Path(path / "model.safetensors").open("rb") as fd:
+            _, weights_dict = safe_read(fd)
             weights = unflatten_parameters(weights_dict)
             model = config.model_config.empty().import_weights(weights)
         tokenizer = Tokenizer.from_file(str(path / "tokenizer.json"))

lalamo/safetensors.py

@@ -0,0 +1,97 @@
+import json
+import struct
+from collections.abc import Mapping
+from dataclasses import dataclass
+from io import BufferedReader, BufferedWriter
+from typing import Any, ClassVar, Self
+
+import cattrs
+import jax
+import jax.numpy as jnp
+import numpy as np
+from jaxtyping import Array
+
+from lalamo.utils import LazyDict
+
+SF2J = {
+    "BOOL": jnp.dtype(jnp.bool_),
+    "U8": jnp.dtype(jnp.uint8),
+    "I8": jnp.dtype(jnp.int8),
+    "I16": jnp.dtype(jnp.int16),
+    "U16": jnp.dtype(jnp.uint16),
+    "F16": jnp.dtype(jnp.float16),
+    "BF16": jnp.dtype(jnp.bfloat16),
+    "I32": jnp.dtype(jnp.int32),
+    "U32": jnp.dtype(jnp.uint32),
+    "F32": jnp.dtype(jnp.float32),
+    "C64": jnp.dtype(jnp.complex64),
+    "F64": jnp.dtype(jnp.float64),
+    "I64": jnp.dtype(jnp.int64),
+    "U64": jnp.dtype(jnp.uint64),
+}
+
+J2SF = {v: k for k, v in SF2J.items()}
+
+
+@dataclass(frozen=True)
+class SFTensorInfo:
+    _converter: ClassVar[cattrs.Converter] = cattrs.Converter()
+    _converter.register_structure_hook(jnp.dtype, lambda x, _: SF2J[x])
+    _converter.register_unstructure_hook(jnp.dtype, lambda x: J2SF[x])
+
+    dtype: jnp.dtype
+    shape: tuple[int, ...]
+    data_offsets: tuple[int, int]
+
+    @property
+    def start(self) -> int:
+        return self.data_offsets[0]
+
+    @property
+    def end(self) -> int:
+        return self.data_offsets[1]
+
+    @property
+    def size(self) -> int:
+        return self.end - self.start
+
+    @classmethod
+    def from_dict(cls, obj: dict) -> Self:
+        return cls._converter.structure(obj, cls)
+
+    def to_dict(self) -> dict:
+        return self._converter.unstructure(self)
+
+
+def safe_read(fd: BufferedReader) -> tuple[dict[str, str] | None, LazyDict[str, Array]]:
+    header_size = struct.unpack("<Q", fd.read(8))[0]
+    header: dict[str, dict[str, Any]] = json.loads(fd.read(header_size))
+    metadata: dict[str, str] | None = header.pop("__metadata__", None)
+    data_offset = fd.tell()
+
+    def _load_tensor(key: str) -> Array:
+        info = SFTensorInfo.from_dict(header[key])
+        fd.seek(data_offset + info.start)
+        return jnp.asarray(np.fromfile(fd, info.dtype, info.size // info.dtype.itemsize)).reshape(info.shape)
+
+    lazy_tensors = LazyDict(set(header.keys()), _load_tensor)
+    return (metadata, lazy_tensors)
+
+
+def safe_write(fd: BufferedWriter, tensors: Mapping[str, Array]) -> None:
+    sorted_tensors = dict(sorted(tensors.items(), key=lambda x: (-x[1].dtype.alignment, x[0])))
+
+    header_dict = {}
+    offset = 0
+    for key, tensor in sorted_tensors.items():
+        assert offset % tensor.dtype.alignment == 0
+        header_dict[key] = SFTensorInfo(tensor.dtype, tensor.shape, (offset, offset + tensor.nbytes)).to_dict()
+        offset += tensor.nbytes
+
+    data_alignment = max(8, next((t.dtype.alignment for t in sorted_tensors.values()), 1))
+    header = json.dumps(header_dict).encode()
+    header += b" " * (-len(header) % data_alignment)
+    fd.write(struct.pack("<Q", len(header)) + header)
+
+    for tensor in sorted_tensors.values():
+        jax.device_get(tensor).tofile(fd)

lalamo/speculator/estimator.py

@@ -9,6 +9,13 @@ import jax.numpy as jnp
 from lalamo.models import LanguageModel
 
 
+def get_default_device_memory() -> int | None:
+    memory_stats = jax.local_devices()[0].memory_stats()
+    if memory_stats is None or "bytes_limit" not in memory_stats:
+        return None
+    return memory_stats["bytes_limit"]
+
+
 def estimate_memory_from_batchsize(
     model: LanguageModel,
     max_input_length: int,
@@ -23,7 +30,7 @@ def estimate_memory_from_batchsize(
                 max_output_length=max_output_length,
                 num_top_logits_to_return=num_logits_per_token,
             ),
-            backend="cpu", # cuda backend tries to allocate in .compile() and ooms
+            backend="cpu",  # cuda backend tries to allocate in .compile() and ooms
         )
         .lower(
             model,
@@ -41,7 +48,7 @@ def estimate_memory_from_batchsize(
     return (
         memory_analysis.argument_size_in_bytes  # type: ignore (pyright bug)
         + memory_analysis.output_size_in_bytes  # type: ignore (pyright bug)
-        + memory_analysis.temp_size_in_bytes    # type: ignore (pyright bug)
+        + memory_analysis.temp_size_in_bytes  # type: ignore (pyright bug)
     )
 
 

lalamo/utils.py

@@ -9,21 +9,17 @@ from collections.abc import (
     Sequence,
     ValuesView,
 )
-from contextlib import contextmanager
 from dataclasses import dataclass
-from pathlib import Path
 from typing import overload
 
 import einops
 import jax.numpy as jnp
 from jaxtyping import Array
-from safetensors import safe_open
 
 __all__ = [
     "MapDictValues",
     "MapSequence",
     "jax_uint4_to_packed_uint8",
-    "open_safetensors",
     "process_chat_template",
 ]
 
@@ -45,15 +41,6 @@ class LazyDict[K, V](Mapping[K, V]):
         return len(self.stored_keys)
 
 
-@contextmanager
-def open_safetensors(filename: Path | str) -> Iterator[tuple[Mapping[str, Array], Mapping[str, str]]]:
-    with safe_open(filename, framework="flax") as safetensors_nonsense:
-        yield (
-            LazyDict(set(safetensors_nonsense.keys()), safetensors_nonsense.get_tensor),
-            safetensors_nonsense.metadata(),
-        )
-
-
 @dataclass(frozen=True)
 class MapIterable[OldT, NewT](Iterable[NewT]):
     map_func: Callable[[OldT], NewT]

{lalamo-0.5.16.dist-info → lalamo-0.5.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lalamo
-Version: 0.5.16
+Version: 0.5.17
 Summary: JAX library for optimization and export of models for use with the UZU inference engine.
 Requires-Python: <4,>=3.12
 Description-Content-Type: text/markdown
@@ -19,7 +19,6 @@ Requires-Dist: rich>=14.0.0
 Requires-Dist: thefuzz>=0.22.1
 Requires-Dist: tokenizers>=0.21.2
 Requires-Dist: typer>=0.15.1
-Requires-Dist: safetensors>=0.6.2
 Requires-Dist: polars>=1.33.1
 Requires-Dist: xxhash>=3.5.0
 Provides-Extra: cpu

{lalamo-0.5.16.dist-info → lalamo-0.5.17.dist-info}/RECORD

@@ -1,11 +1,13 @@
-lalamo/__init__.py,sha256=FjfGsBVSl14mNsDoFJEwXMRUq1-Kg_lessRzlJNG3KM,815
+lalamo/__init__.py,sha256=asVMPmQ7BUt7bYlcuNZ7SnOSJDJUiN9QhlU5lRUehSo,1387
+lalamo/commands.py,sha256=rU9T8Mx6s7itpk-dj5ToQ4PUpGPfdmmKlrF02l2kIS0,9967
 lalamo/common.py,sha256=5NUFD26yQgOnEEk3LaQnce8n-VwJxILkEpFesHZhtQU,3820
-lalamo/main.py,sha256=GgUT7lT48-XQuAEH7qzsDKG8Lx9iBf-sYBIRhZL9q7E,23978
+lalamo/main.py,sha256=dE7Us9L6sfz9bp5rUSzGHUkG0Uon4xdju9dGGtXidZI,23888
 lalamo/message_processor.py,sha256=bSUAQg7CemLTnBV4LtPxJBicAalruDCA-JXjkTYPZ8U,5797
 lalamo/quantization.py,sha256=8o6ryIZLzzDYQuvBTboPfaVVdfijAKGpTxOcg3GKVD8,2752
 lalamo/registry_abc.py,sha256=ENjXiD_wEH100fNjG-W5Em1L_EQ0Lf0pdRhRGvf3qZk,2197
+lalamo/safetensors.py,sha256=kUiTSgx2zhfD1hxV_AA1DOLaKAKzjRd_vOYZCFf0em0,3048
 lalamo/sampling.py,sha256=g_dNiJyZrRqoQIiLid4cr6nRT9N5tSz3GtHr8Bt4n-E,3404
-lalamo/utils.py,sha256=QwATVXAeHBsQEDyt_31SHgxFphFVZYHpv3ZaklXks9Y,4585
+lalamo/utils.py,sha256=c88IP110gHZJ6hYDq7p36A9u-vLRM_YdavFom56gsNQ,4111
 lalamo/data/__init__.py,sha256=exfhBLxHrg7BWutM0tAln5QuIWlNQmOhaG2noFYxfPI,189
 lalamo/data/huggingface_message.py,sha256=-7lN9eIcETQzt1Pnx3d4d8p3_I7WYMNf4mp1P91N7fI,1115
 lalamo/data/lalamo_completions.py,sha256=U_m3UNSJASUFz3rJq_taZOtL_U4B8Oj-ndkTF-JH-v4,1509
@@ -35,7 +37,7 @@ lalamo/model_import/loaders/executorch.py,sha256=t2Ey_mBMNC8bTSTdYWjuGXdPTRoohFl
 lalamo/model_import/loaders/huggingface.py,sha256=qWdzoSvHvb_3prn2kwfxgnYPW2bVB0Q49m_wyRYha8Q,34677
 lalamo/model_import/loaders/utils.py,sha256=eiX3WKFRrAfBY-dugodscNInl5o5w3KmVcgma4atpGY,2456
 lalamo/model_import/model_specs/__init__.py,sha256=JISqwJkloQkGD2jvi1MakNEWapIwlNXXVi5giZyXB74,1275
-lalamo/model_import/model_specs/common.py,sha256=RLySCIkmGiA1IVZgLeemssMBMo4hMYMpmBjV0cRwBb4,6586
+lalamo/model_import/model_specs/common.py,sha256=8ALKxHrt8uK4XiqjK25NwZj1CC7DM7jlYcFVZPGkFrw,6643
 lalamo/model_import/model_specs/deepseek.py,sha256=Umef93_ZBuq93yYsejIRNwj3udoln1gHfrv3SK5jyMo,417
 lalamo/model_import/model_specs/essential_ai.py,sha256=xbHcwRpAWhR9gOgypVzcgunFspoUEk3iNsw-46CVR4o,390
 lalamo/model_import/model_specs/gemma.py,sha256=dwKwOHU1sBJNLFAwtEyydsRUF9QENN3SHtjbfqtOSic,3876
@@ -52,7 +54,7 @@ lalamo/model_import/model_specs/qwen.py,sha256=HvN080ILpOwkqJbRLMqCa8Z8ImlLfTwiE
 lalamo/model_import/model_specs/reka.py,sha256=dOUYbEMMvovQdzQuBO_DCsjGI39syhoKCvnxLkNEDCw,423
 lalamo/models/__init__.py,sha256=Vn5PcvSqKppIchkSZwQVTn_GpRvOOzZVxo5PUeDl6N8,283
 lalamo/models/classifier.py,sha256=LvL54crCVi4HVSIXuoaSLB_5jtcx74GL7kgdy2Y16Zc,2094
-lalamo/models/common.py,sha256=PDteofGxjSBWYw_mPxbN1DTUba70aOURrAIjl13SSHc,2954
+lalamo/models/common.py,sha256=uU6eCHtIqMeC_aRGVo09NdpAtvQ6RKSbm6pumVvL8pc,2943
 lalamo/models/language_model.py,sha256=QPeVEyhutSze7fSNhvOvwSoYt24QMk-dtTJkos38amY,13465
 lalamo/modules/__init__.py,sha256=OHIQn08jx2c3L2KIQA-7SJ4yVb2E5m6T6FqTHFJTDdM,4006
 lalamo/modules/activations.py,sha256=U3qTQtZawPAUcoqbkIJnmTYcaNiQuSPMLcBeJ398GhI,1022
@@ -81,13 +83,13 @@ lalamo/modules/token_mixers/state/mamba_state.py,sha256=LHzJvNE6MkB7nrsZSNto6pxb
 lalamo/modules/token_mixers/state/short_conv_state.py,sha256=osjcDHoeFWQaUoOROzeJe8F1qC8rvqunimGD4CuIDHo,895
 lalamo/speculator/__init__.py,sha256=9-tmZcbCom_lIGpJYn6xLlnEahFLFidpqmgkafmu--k,456
 lalamo/speculator/common.py,sha256=PudF_gkpe5_nQ-57sAC-foE1xCy_H2Axh5KwRoA86lo,587
-lalamo/speculator/estimator.py,sha256=4D8dPZCWsrpORb7y8pQ6VsiIg1Cblvvxe6gXCoYtcD4,2530
+lalamo/speculator/estimator.py,sha256=j-zmhy3RxYDmQ7W0FMTmDk3i275r_Vg1s4NCaS4c_SQ,2760
 lalamo/speculator/inference.py,sha256=5GntUgj0HQLeLn3HIHnVX8EEO0EBzmKeP5-_U7kdFAM,3670
 lalamo/speculator/ngram.py,sha256=95mdfAWhx4d5XOnOwhyhElnvcy6nlUjYhcbJzqDs414,5875
 lalamo/speculator/utils.py,sha256=0wZoMMIzzk0Q-3zq5H5f-JBplePNHxywndkrNtOJOyo,1697
-lalamo-0.5.16.dist-info/licenses/LICENSE,sha256=diHRfjSEJHD1nnEeMIfMRCjR3UERf8bT3eseD6b1ayA,1072
-lalamo-0.5.16.dist-info/METADATA,sha256=dcs0vT9RULTxt4cxJJmfjP-4UJi7ZkrifXAaSMAgKeU,3147
-lalamo-0.5.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lalamo-0.5.16.dist-info/entry_points.txt,sha256=qli7qTfnBk5WP10rOGXXEckHMtt-atJMDWd8jN89Uks,43
-lalamo-0.5.16.dist-info/top_level.txt,sha256=VHvWL5JN5XRG36NsN_MieJ7EwRihEOrEjyDaTdFJ-aI,7
-lalamo-0.5.16.dist-info/RECORD,,
+lalamo-0.5.17.dist-info/licenses/LICENSE,sha256=diHRfjSEJHD1nnEeMIfMRCjR3UERf8bT3eseD6b1ayA,1072
+lalamo-0.5.17.dist-info/METADATA,sha256=16-W1J0wiwrmgMTgqiE9r3vxKRmZbGgZ-zS7bNACwTA,3113
+lalamo-0.5.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lalamo-0.5.17.dist-info/entry_points.txt,sha256=qli7qTfnBk5WP10rOGXXEckHMtt-atJMDWd8jN89Uks,43
+lalamo-0.5.17.dist-info/top_level.txt,sha256=VHvWL5JN5XRG36NsN_MieJ7EwRihEOrEjyDaTdFJ-aI,7
+lalamo-0.5.17.dist-info/RECORD,,