tokenfold-select 0.5.2__py3-none-any.whl

tokenfold_select/__init__.py

@@ -0,0 +1,7 @@
+"""Optional development-only scorer; importing never loads a model/framework."""
+from .artifact import inspect_artifact
+from .runtime import DevelopmentScorer
+from .onnx import OnnxScorer, inspect_onnx
+
+__all__ = ["OnnxScorer", "inspect_onnx", "DevelopmentScorer", "inspect_artifact"]
+__version__ = "0.5.2"

tokenfold_select/__main__.py

@@ -0,0 +1,133 @@
+"""Inspection or the existing one-shot Select file protocol; no downloads."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import sys
+import threading
+
+from .artifact import inspect_artifact
+from .protocol import REQUEST_BYTES, encode_response, load_request
+from .runtime import DevelopmentScorer
+from .onnx import DEFAULT_MANIFEST_SHA256, DEFAULT_REVISION, OnnxScorer, inspect_onnx
+
+
+def configure(args):
+    """Generate approval only on explicit request; never download or invoke a model."""
+    model = inspect_onnx(args.artifact, args.manifest_sha256)
+    if model['source_checkpoint_sha256'] != args.revision:
+        raise ValueError('model_revision_mismatch')
+    if (not args.approved_by.strip() or not args.scratch_root.is_absolute()
+            or not args.scratch_root.is_dir() or not args.site_packages.is_absolute()
+            or not args.site_packages.is_dir()):
+        raise ValueError('explicit_approval_and_absolute_existing_directories_required')
+    executable = Path(sys._base_executable).resolve(strict=True)
+    if executable.stat().st_size > 64 * 1024 * 1024:
+        raise ValueError('executable_byte_limit')
+    bootstrap = Path(__file__).with_name('bootstrap.py').resolve()
+    with executable.open('rb') as stream:
+        pin = hashlib.file_digest(stream, 'sha256').hexdigest()
+    return dict(executable=str(executable), executable_sha256=pin,
+                arguments=['-I', '-S', str(bootstrap), '--site-packages', str(args.site_packages.resolve()),
+                           'worker', '--artifact', str(args.artifact.resolve()), '--revision', args.revision,
+                           '--manifest-sha256', args.manifest_sha256, '--development'],
+                model_revision=args.revision, approved_by=args.approved_by,
+                scratch_root=str(args.scratch_root.resolve()))
+
+
+def _worker(args):
+    # A native direct child is required. This timer never starts on import/inspection.
+    timer = threading.Timer(30, lambda: os._exit(124))
+    timer.daemon = True
+    timer.start()
+    code = 1
+    try:
+        # Set before importing any inference library. ONNX is explicitly CPU-only.
+        if args.backend == 'onnx':
+            os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+            os.environ['TOKENIZERS_PARALLELISM'] = 'false'
+            if os.name == 'nt':
+                # The Rust parent clears the environment. Obtain the system path
+                # from Windows, not an inherited/user-controlled environment value.
+                import ctypes
+                system = ctypes.create_unicode_buffer(32768)
+                length = ctypes.windll.kernel32.GetWindowsDirectoryW(system, len(system))
+                if not 0 < length < len(system):
+                    raise ValueError('windows_system_directory_unavailable')
+                os.environ['SystemRoot'] = system.value
+        request_path = Path(os.environ['TOKENFOLD_SELECT_REQUEST_PATH'])
+        response_path = Path(os.environ['TOKENFOLD_SELECT_RESPONSE_PATH'])
+        if (not request_path.is_absolute() or not response_path.is_absolute()
+                or request_path.resolve().parent != response_path.resolve().parent
+                or request_path.resolve() == response_path.resolve() or response_path.exists()):
+            raise ValueError('caller_owned_request_response_paths_required')
+        def offline(event, _):
+            if event in ('socket.connect', 'socket.getaddrinfo', 'socket.bind', 'subprocess.Popen', 'os.system'):
+                raise ValueError('offline_direct_child_only')
+        sys.addaudithook(offline)
+        with request_path.open('rb') as stream:
+            request = load_request(stream.read(REQUEST_BYTES + 1), args.revision)
+        if args.backend == 'onnx':
+            scorer = OnnxScorer(args.artifact, args.manifest_sha256, development=args.development)
+            if scorer.revision != args.revision:
+                raise ValueError('model_revision_mismatch')
+        else:
+            scorer = DevelopmentScorer(args.artifact, args.revision, cache_directory=request_path.parent / 'model-cache',
+                                       development=args.development, precision='fp32')
+        response = scorer.score(request)
+        raw = encode_response(request, [value for _, value in response['scores']])
+        with response_path.open('xb') as stream:
+            stream.write(raw)
+        if args.backend == 'pytorch':
+            scorer._torch.cuda.synchronize()
+        code = 0
+    except BaseException:
+        print('tokenfold-select: refused', file=sys.stderr, flush=True)
+    finally:
+        # Avoid potentially unbounded framework teardown; native parent owns kill/wait.
+        sys.stdout.flush()
+        sys.stderr.flush()
+        os._exit(code)
+
+
+def main(arguments=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest='command', required=True)
+    for command in ('inspect', 'inspect-onnx', 'configure', 'worker'):
+        mode = sub.add_parser(command)
+        mode.add_argument('--artifact', type=Path, required=True)
+        mode.add_argument('--revision', default=DEFAULT_REVISION)
+        mode.add_argument('--manifest-sha256', default=DEFAULT_MANIFEST_SHA256)
+        if command == 'worker':
+            mode.add_argument('--backend', choices=('onnx', 'pytorch'), default='onnx')
+            mode.add_argument('--development', action='store_true', required=True)
+        if command == 'configure':
+            mode.add_argument('--approved-by', required=True)
+            mode.add_argument('--site-packages', type=Path, required=True)
+            mode.add_argument('--scratch-root', type=Path, required=True)
+    args = parser.parse_args(arguments)
+    if args.command == 'worker':
+        _worker(args)
+    try:
+        if args.command == 'configure':
+            print(json.dumps(configure(args)))
+            return 0
+        if args.command == 'inspect-onnx':
+            report = inspect_onnx(args.artifact, args.manifest_sha256)
+            if report['source_checkpoint_sha256'] != args.revision:
+                raise ValueError('model_revision_mismatch')
+            report = dict(checkpoint_sha256=report['source_checkpoint_sha256'])
+        else:
+            report = inspect_artifact(args.artifact, args.revision)
+    except (OSError, ValueError, KeyError, TypeError, AttributeError):
+        print('tokenfold-select: refused', file=sys.stderr)
+        return 1
+    print(json.dumps(dict(verification='metadata_and_hashes_only', checkpoint_tensors_loaded=False,
+                          model_revision=report['checkpoint_sha256'], development_only=True,
+                          qualification=False, release_approved=False, redistribution_cleared=False)))
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

tokenfold_select/_model.py

@@ -0,0 +1,62 @@
+"""Development-only joint group encoder. Random head: NOT a usable Select scorer.
+
+No downloads, truncation, windowing, generated text or production integration.
+The caller owns request validation and supplies the provisioned encoder/tokenizer.
+"""
+import torch
+from torch import nn
+
+
+def pack_groups(tokenizer, query, groups, cap=8192):
+    """Return one sequence and explicit content ranges in caller order."""
+    if not 1 <= cap <= 8192 or not query or not 1 <= len(groups) <= 512:
+        raise ValueError("invalid_admission")
+    ids_seen = set()
+    cls, sep = tokenizer.cls_token_id, tokenizer.sep_token_id
+    if cls is None or sep is None:
+        raise ValueError("missing_special_tokens")
+    ids = [cls] + tokenizer.encode(query, add_special_tokens=False, truncation=False) + [sep]
+    ranges = []
+    for group in groups:
+        gid, text = group['id'], group['text']
+        if not isinstance(gid, str) or not gid or gid in ids_seen or not isinstance(text, str) or not text:
+            raise ValueError("invalid_group")
+        ids_seen.add(gid)
+        content = tokenizer.encode(text, add_special_tokens=False, truncation=False)
+        if not content:
+            raise ValueError("empty_group_range")
+        start = len(ids)
+        ids.extend(content)
+        ranges.append((start, len(ids)))
+        ids.append(sep)
+        if len(ids) > cap:
+            raise ValueError("token_cap")
+    return ids, ranges
+
+
+class JointGroupModel(nn.Module):
+    """One encoder invocation, masked content mean and a shared linear head."""
+
+    def __init__(self, encoder):
+        super().__init__()
+        self.encoder = encoder
+        self.head = nn.Linear(encoder.config.hidden_size, 1)
+
+    def forward(self, input_ids, attention_mask, ranges):
+        if input_ids.ndim != 2 or input_ids.shape[0] != 1 or attention_mask.shape != input_ids.shape:
+            raise ValueError("single_sequence_required")
+        previous = 0
+        if not 1 <= len(ranges) <= 512:
+            raise ValueError("invalid_ranges")
+        for start, end in ranges:
+            if not previous <= start < end <= input_ids.shape[1] or not attention_mask[0, start:end].all():
+                raise ValueError("invalid_ranges")
+            previous = end
+        positions = torch.arange(input_ids.shape[1], device=input_ids.device).unsqueeze(0)
+        states = self.encoder(input_ids=input_ids, attention_mask=attention_mask,
+                              position_ids=positions).last_hidden_state[0]
+        pooled = torch.stack([states[start:end].mean(0) for start, end in ranges])
+        scores = self.head(pooled).squeeze(-1)
+        if not torch.isfinite(scores).all():
+            raise ValueError("nonfinite_scores")
+        return scores

tokenfold_select/_windows.py

@@ -0,0 +1,107 @@
+"""Bounded source-conditioned windows, shared by future training and inference.
+
+Original groups are never cropped or split. A bounded auxiliary source-tail copy
+is unscored encoder context, not extra retained text or consumer input.
+"""
+
+import hashlib
+from pathlib import Path
+
+PROFILE = "source-tail-1536-capacity-v2"
+CONTEXT_TOKENS = 1536
+WINDOW_TOKENS = 8192
+MAX_GROUPS = 512
+MAX_WINDOWS = 64
+MAX_TOTAL_TOKENS = MAX_WINDOWS * WINDOW_TOKENS
+
+
+def code_hash():
+    return hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
+
+
+def verify_profile(report):
+    recipe = report.get("recipe", {})
+    if (recipe.get("scoring_window_profile") != PROFILE
+            or recipe.get("scoring_context_token_cap") != CONTEXT_TOKENS
+            or recipe.get("scoring_window_code_sha256") != code_hash()):
+        raise ValueError("matching_trained_source_context_profile_required")
+
+
+def pack_windows(tokenizer, config, request):
+    """Return legacy-shaped windows and a non-qualifying packing receipt.
+
+    Tokenization is once per query/group. Each original group has one scored
+    range; auxiliary copies have no ranges. All admission finishes before forward.
+    """
+    groups = request["groups"]
+    if not isinstance(groups, list) or not 1 <= len(groups) <= MAX_GROUPS:
+        raise ValueError("bounded_nonempty_groups_required")
+    if (not isinstance(request.get("query"), str) or not request["query"].strip()
+            or any(not isinstance(g, dict) or not isinstance(g.get("text"), str) or not g["text"] for g in groups)):
+        raise ValueError("original_query_and_group_text_required")
+    tokenizer.no_truncation(); tokenizer.no_padding()
+    def special(name):
+        value = config[name]
+        token = tokenizer.token_to_id(value["content"] if isinstance(value, dict) else value)
+        if token is None: raise ValueError("missing_special_token")
+        return token
+    cls, sep = special("cls_token"), special("sep_token")
+    query = [cls] + tokenizer.encode(request["query"], add_special_tokens=False).ids + [sep]
+    if len(query) >= WINDOW_TOKENS: raise ValueError("query_token_cap")
+    encoded = [tokenizer.encode(group["text"], add_special_tokens=False).ids for group in groups]
+    if any(not ids or len(query) + len(ids) + 1 > WINDOW_TOKENS for ids in encoded):
+        raise ValueError("whole_group_token_cap")
+    source_tokens = sum(len(ids) + 1 for ids in encoded)
+    if source_tokens + len(query) > MAX_TOTAL_TOKENS:
+        raise ValueError("total_token_cap")
+    context = []; capacity_checks = 0; compatible_allowance = 0; reduced = False
+    if source_tokens + len(query) > WINDOW_TOKENS:
+        allowance = min(CONTEXT_TOKENS, WINDOW_TOKENS - len(query) - max(map(len, encoded)) - 2)
+        if allowance < 1: raise ValueError("whole_group_leaves_no_source_context")
+        compatible_allowance = allowance
+        def demand(amount):
+            nonlocal capacity_checks
+            capacity_checks += 1
+            header = len(query) + amount + 1
+            count = 1; used = header
+            for content in encoded:
+                length = len(content) + 1
+                if used + length > WINDOW_TOKENS:
+                    count += 1; used = header
+                used += length
+            return count, source_tokens + count * header
+        count, total = demand(allowance)
+        if count > MAX_WINDOWS or total > MAX_TOTAL_TOKENS:
+            count, total = demand(1)
+            if count > MAX_WINDOWS: raise ValueError("window_count_cap")
+            if total > MAX_TOTAL_TOKENS: raise ValueError("total_token_cap")
+            # Largest feasible auxiliary copy, never fewer original source tokens.
+            low, high = 1, allowance
+            while low < high:
+                mid = (low + high + 1) // 2
+                count, total = demand(mid)
+                if count <= MAX_WINDOWS and total <= MAX_TOTAL_TOKENS: low = mid
+                else: high = mid - 1
+            allowance = low; reduced = True
+        for ids in encoded:
+            context = (context + [sep] + ids)[-allowance:]
+    header = query + context + [sep] if context else query
+    windows = []
+    ids, ranges, indices = list(header), [], []
+    for index, content in enumerate(encoded):
+        if len(ids) + len(content) + 1 > WINDOW_TOKENS:
+            windows.append((ids, ranges, indices))
+            if len(windows) >= MAX_WINDOWS: raise ValueError("window_count_cap")
+            ids, ranges, indices = list(header), [], []
+        start = len(ids); ids.extend(content); ids.append(sep)
+        ranges.append((start, start + len(content))); indices.append(index)
+    windows.append((ids, ranges, indices))
+    total = sum(len(window[0]) for window in windows)
+    if total > MAX_TOTAL_TOKENS: raise ValueError("total_token_cap")
+    return windows, {"profile": PROFILE, "context_tokens_per_window": len(context),
+                     "max_context_tokens": CONTEXT_TOKENS,
+                     "whole_group_compatible_context_tokens": compatible_allowance,
+                     "context_reduced_for_capacity": reduced, "capacity_checks": capacity_checks,
+                     "scored_source_tokens_with_separators": source_tokens,
+                     "total_encoded_tokens_including_copies": total,
+                     "windows": len(windows), "groups": len(groups), "qualification": False}

tokenfold_select/artifact.py

@@ -0,0 +1,83 @@
+"""Portable integrity/development admission, never a model or rights certificate."""
+import hashlib
+from pathlib import Path
+
+from . import _windows
+from .protocol import decode
+
+TOKENIZER_FILES = {'tokenizer.json', 'tokenizer_config.json', 'special_tokens_map.json'}
+
+
+def sha(path, limit):
+    total = 0
+    digest = hashlib.sha256()
+    with Path(path).open('rb') as stream:
+        while chunk := stream.read(1024 * 1024):
+            total += len(chunk)
+            if total > limit:
+                raise ValueError('artifact_byte_limit')
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _file(root, name):
+    path = (root / name).resolve(strict=True)
+    if not path.is_relative_to(root) or not path.is_file():
+        raise ValueError('artifact_path_escape')
+    return path
+
+
+def _json(root, name, limit):
+    path = _file(root, name)
+    with path.open('rb') as stream:
+        data = stream.read(limit + 1)
+    if len(data) > limit:
+        raise ValueError('artifact_byte_limit')
+    return decode(data)
+
+
+def inspect_artifact(directory, revision):
+    """Check bytes/declared completion only, with no imports of ML frameworks."""
+    if (not isinstance(revision, str) or len(revision) != 64
+            or any(c not in '0123456789abcdef' for c in revision)):
+        raise ValueError('checkpoint_SHA256_required')
+    root = Path(directory)
+    if not root.is_absolute():
+        raise ValueError('absolute_artifact_path_required')
+    root = root.resolve(strict=True)
+    report = _json(root, 'training-report.json', 1024 * 1024)
+    if (not isinstance(report, dict) or not isinstance(report.get('recipe'), dict)
+            or report.get('status') != 'completed_development_only'
+            or report.get('checkpoint_sha256') != revision
+            or report.get('mode') != 'supervised-foundation-feature-distilled'
+            or report.get('recipe', {}).get('objective_profile') != 'optional-balanced-window-hardpair-headwarm-v4'
+            or report.get('recipe', {}).get('precision') != 'fp32'
+            or report.get('recipe', {}).get('student_token_cap') != 8192):
+        raise ValueError('completed_clean_compact12_development_artifact_required')
+    _windows.verify_profile(report)
+    if sha(Path(__file__).with_name('_model.py'), 128 * 1024) != report.get('joint_model_code_sha256'):
+        raise ValueError('model_code_hash_mismatch')
+    config = _json(root, 'config.json', 65536)
+    if (not isinstance(config, dict) or config.get('model_type') != 'modernbert' or config.get('num_hidden_layers') != 12
+            or config.get('hidden_size') != 768 or config.get('max_position_embeddings') != 8192
+            or config.get('layer_types') != ['full_attention' if i % 3 == 0 else 'sliding_attention' for i in range(12)]):
+        raise ValueError('compact12_configuration_required')
+    if sha(_file(root, 'config.json'), 65536) != report.get('config_sha256'):
+        raise ValueError('config_hash_mismatch')
+    if sha(_file(root, 'checkpoint.pt'), 1024**3) != revision:
+        raise ValueError('weights_hash_mismatch')
+    pins = report.get('tokenizer_sha256')
+    if not isinstance(pins, dict) or set(pins) != TOKENIZER_FILES:
+        raise ValueError('complete_tokenizer_pins_required')
+    if {p.name for p in (root / 'tokenizer').iterdir()} != TOKENIZER_FILES:
+        raise ValueError('tokenizer_files_mismatch')
+    for name, digest in pins.items():
+        if sha(_file(root, 'tokenizer/' + name), 32 * 1024**2) != digest:
+            raise ValueError('tokenizer_hash_mismatch')
+    decision = _json(root, 'DEVELOPMENT-DECISION.json', 1024 * 1024)
+    if (not isinstance(decision, dict) or decision.get('checkpoint_sha256') != revision
+            or any(decision.get(key) is not True for key in
+                   ('training_completed', 'completion_content_audit_passed', 'development_stop_go_passed'))
+            or any(decision.get(key) is not False for key in ('qualification', 'release_approved', 'redistribution_cleared'))):
+        raise ValueError('accepted_development_decision_required_not_release')
+    return report

tokenfold_select/bootstrap.py

@@ -0,0 +1,21 @@
+"""Direct-native entrypoint for an approved caller-owned Python environment."""
+import argparse
+from pathlib import Path
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(add_help=False)
+    parser.add_argument('--site-packages', type=Path, required=True)
+    args, remaining = parser.parse_known_args()
+    if (not sys.flags.isolated or not sys.flags.no_site
+            or not args.site_packages.is_absolute() or not args.site_packages.is_dir()):
+        raise ValueError('isolated_native_caller_environment_required')
+    # No .pth processing, checkout imports, inherited PYTHONPATH or shell/shim.
+    sys.path[:0] = [str(Path(__file__).resolve().parents[1]), str(args.site_packages.resolve())]
+    from tokenfold_select.__main__ import main as run
+    return run(remaining)
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

tokenfold_select/onnx.py

@@ -0,0 +1,85 @@
+"""Bounded ONNX CPU reference scorer; no PyTorch, downloads or generated text."""
+import hashlib
+import importlib.metadata
+import json
+from pathlib import Path
+import re
+
+from . import _windows as packing
+from . import protocol
+
+CAPS={'model.onnx':64*1024**2,'model.onnx.data':1024**3,
+      'tokenizer/tokenizer.json':32*1024**2,'tokenizer/tokenizer_config.json':65536,
+      'tokenizer/special_tokens_map.json':65536,
+      'runtime/onnx_score.py':65536,'runtime/packing.py':65536,'runtime/protocol.py':65536}
+DEFAULT_REVISION='4dbdb5071e7d6316d8eb70dc161b1f2f77f555012f6616d7fa34311188e24cec'
+DEFAULT_MANIFEST_SHA256='267313718e80aa91a56923dea30363fd5c24fde3782c716d6d048891860a2054'
+ENVIRONMENT={'onnxruntime':'1.30.0','numpy':'2.5.3','tokenizers':'0.23.2'}
+
+def digest(path):
+    with Path(path).open('rb') as stream:return hashlib.file_digest(stream,'sha256').hexdigest()
+
+def inspect_onnx(directory,manifest_sha256=DEFAULT_MANIFEST_SHA256):
+    root=Path(directory)
+    if not root.is_absolute() or not re.fullmatch(r'[0-9a-f]{64}',manifest_sha256):
+        raise ValueError('absolute_root_and_manifest_pin_required')
+    root=root.resolve(strict=True);path=root/'onnx-manifest.json'
+    if path.is_symlink() or path.stat().st_size>65536 or digest(path)!=manifest_sha256:
+        raise ValueError('manifest_pin_mismatch')
+    m=protocol.decode(path.read_bytes())
+    if (m.get('status')!='development_ONNX_export' or m.get('precision')!='fp32'
+        or m.get('provider')!='CPUExecutionProvider' or m.get('opset')!=18
+        or m.get('limits')!={'tokens':8192,'groups':512,'request_bytes':1048576,'response_bytes':65536}
+        or not re.fullmatch(r'[0-9a-f]{64}',m.get('source_checkpoint_sha256',''))
+        or m.get('qualification') is not False or set(m.get('files',{}))!=set(CAPS)):
+        raise ValueError('bounded_development_ONNX_profile_required')
+    for name,cap in CAPS.items():
+        path=root/name;pin=m['files'][name]
+        if (path.is_symlink() or not path.resolve(strict=True).is_relative_to(root)
+            or type(pin.get('bytes')) is not int or not 0<pin['bytes']<=cap
+            or path.stat().st_size!=pin['bytes'] or digest(path)!=pin['sha256']):
+            raise ValueError('executed_file_pin_mismatch')
+    # Package code is installed locally, never imported from the model directory.
+    for module,name in ((packing,'packing'),(protocol,'protocol')):
+        if digest(module.__file__)!=m['files'][f'runtime/{name}.py']['sha256']:
+            raise ValueError('installed_helpers_must_match_model')
+    return m
+
+class OnnxScorer:
+    """One complete window; callers own process lifetime and final allocation."""
+    def __init__(self,directory,manifest_sha256=DEFAULT_MANIFEST_SHA256,*,development=False):
+        if development is not True:raise ValueError('explicit_development_opt_in_required')
+        self.root=Path(directory)
+        self.manifest_sha256=manifest_sha256
+        self.manifest=inspect_onnx(self.root,manifest_sha256)
+        self.revision=self.manifest['source_checkpoint_sha256']
+        self._tokenizer=self._session=None
+
+    def score(self,request):
+        request=protocol.validate_request(request,self.revision)
+        for name,version in ENVIRONMENT.items():
+            if importlib.metadata.version(name)!=version:raise ValueError('pinned_ONNX_environment_required')
+        if self._tokenizer is None:
+            from tokenizers import Tokenizer
+            self._tokenizer=Tokenizer.from_file(str(self.root/'tokenizer/tokenizer.json'))
+            self._token_config=protocol.decode((self.root/'tokenizer/tokenizer_config.json').read_bytes())
+        windows,receipt=packing.pack_windows(self._tokenizer,self._token_config,request)
+        if len(windows)!=1 or receipt['context_tokens_per_window']:
+            raise ValueError('single_complete_window_required_no_truncation')
+        ids,ranges,indices=windows[0]
+        if indices!=list(range(len(request['groups']))):raise ValueError('group_alignment')
+        import numpy as np
+        if self._session is None:
+            inspect_onnx(self.root,self.manifest_sha256)
+            import onnxruntime as ort
+            options=ort.SessionOptions();options.intra_op_num_threads=4;options.inter_op_num_threads=1
+            session=ort.InferenceSession(str(self.root/'model.onnx'),options,providers=['CPUExecutionProvider'])
+            session.disable_fallback()
+            if session.get_providers()!=['CPUExecutionProvider']:
+                raise ValueError('explicit_CPU_provider_required')
+            self._session=session
+        inputs=np.asarray([ids],dtype=np.int64)
+        output=self._session.run(['token_scores'],{'input_ids':inputs,'attention_mask':np.ones_like(inputs)})[0]
+        if output.shape!=inputs.shape or not np.isfinite(output).all():raise ValueError('finite_token_scores_required')
+        scores=[float(output[0,a:b].mean()) for a,b in ranges]
+        return protocol.decode(protocol.encode_response(request,scores))

tokenfold_select/protocol.py

@@ -0,0 +1,69 @@
+"""Existing Select schema one, with bounded fail-closed JSON handling."""
+import json
+import math
+
+REQUEST_BYTES = 1024 * 1024
+RESPONSE_BYTES = 65536
+
+
+def _unique(pairs):
+    value = {}
+    for key, item in pairs:
+        if key in value:
+            raise ValueError('duplicate_JSON_key')
+        value[key] = item
+    return value
+
+
+def _number(text):
+    value = float(text)
+    if not math.isfinite(value):
+        raise ValueError('nonfinite_JSON')
+    return value
+
+
+def _constant(_):
+    raise ValueError('nonfinite_JSON')
+
+
+def decode(data):
+    return json.loads(data, object_pairs_hook=_unique, parse_float=_number, parse_constant=_constant)
+
+
+def load_request(data, revision):
+    if not isinstance(data, bytes) or len(data) > REQUEST_BYTES:
+        raise ValueError('request_byte_limit')
+    request = decode(data)
+    return validate_request(request, revision)
+
+
+def validate_request(request, revision):
+    if (not isinstance(request, dict)
+            or set(request) != {'schema_version', 'model_revision', 'query', 'groups'}
+            or type(request['schema_version']) is not int or request['schema_version'] != 1
+            or request['model_revision'] != revision
+            or not isinstance(request['query'], str) or not request['query'].strip()
+            or len(request['query'].encode('utf8')) > 4096):
+        raise ValueError('invalid_select_request')
+    groups = request['groups']
+    if (not isinstance(groups, list) or not 1 <= len(groups) <= 512
+            or any(not isinstance(g, dict) or set(g) != {'id', 'text'}
+                   or not isinstance(g['id'], str) or not g['id'] or len(g['id'].encode('utf8')) > 256
+                   or not isinstance(g['text'], str) or not g['text'] for g in groups)
+            or len({g['id'] for g in groups}) != len(groups)):
+        raise ValueError('invalid_groups')
+    if len(json.dumps(request, ensure_ascii=False, allow_nan=False).encode('utf8')) > REQUEST_BYTES:
+        raise ValueError('request_byte_limit')
+    return request
+
+
+def encode_response(request, scores):
+    if (not isinstance(scores, list) or len(scores) != len(request['groups'])
+            or any(type(value) not in (int, float) or not math.isfinite(value) for value in scores)):
+        raise ValueError('invalid_scores')
+    response = dict(schema_version=1, model_revision=request['model_revision'],
+                    scores=[[g['id'], value] for g, value in zip(request['groups'], scores)])
+    raw = json.dumps(response, ensure_ascii=False, allow_nan=False).encode('utf8')
+    if len(raw) > RESPONSE_BYTES:
+        raise ValueError('response_byte_limit')
+    return raw

tokenfold_select/runtime.py

@@ -0,0 +1,122 @@
+"""Dedicated-process, explicitly opted-in development scoring; no text generation."""
+import importlib.metadata
+import json
+import os
+from pathlib import Path
+
+from . import _windows
+from .artifact import inspect_artifact
+from .protocol import encode_response, validate_request
+
+ENVIRONMENT = {
+    'torch': '2.11.0+cu128', 'transformers': '5.19.0', 'safetensors': '0.8.0',
+    'tokenizers': '0.23.2', 'huggingface_hub': '1.33.0', 'numpy': '2.5.3',
+}
+
+
+def initialize_windows_system_root():
+    if os.name == 'nt':
+        import ctypes
+        buffer = ctypes.create_unicode_buffer(32768)
+        get = ctypes.WinDLL('kernel32', use_last_error=True).GetWindowsDirectoryW
+        get.argtypes = [ctypes.c_wchar_p, ctypes.c_uint]
+        get.restype = ctypes.c_uint
+        size = get(buffer, len(buffer))
+        if not 0 < size < len(buffer):
+            raise ValueError('windows_directory_unavailable')
+        os.environ['SystemRoot'] = buffer.value
+
+
+class DevelopmentScorer:
+    """Single-window prototype. Use in its own process; GPU parity is still required."""
+
+    def __init__(self, artifact, revision, *, cache_directory, development=False, precision='fp32'):
+        if development is not True or precision not in ('fp32', 'fp16'):
+            raise ValueError('explicit_development_precision_required')
+        self.root = Path(artifact)
+        self.report = inspect_artifact(self.root, revision)
+        self.root = self.root.resolve()
+        self.revision, self.precision = revision, precision
+        self.cache = Path(cache_directory)
+        if not self.cache.is_absolute() or self.cache.resolve().is_relative_to(self.root):
+            raise ValueError('separate_absolute_cache_directory_required')
+        self._tokenizer = self._model = self._torch = None
+        self._token_config = None
+        self.last_metrics = None
+
+    def _prepare_environment(self):
+        initialization = self.report.get('initialization_recipe')
+        declared = initialization.get('environment', []) if isinstance(initialization, dict) else []
+        if (not isinstance(declared, list) or len(declared) != len(ENVIRONMENT)
+                or set(declared) != {name + '==' + value for name, value in ENVIRONMENT.items()}):
+            raise ValueError('supported_provisioned_environment_required')
+        for name, expected in ENVIRONMENT.items():
+            if importlib.metadata.version(name) != expected:
+                raise ValueError('supported_provisioned_environment_required')
+        initialize_windows_system_root()
+        self.cache.mkdir(parents=True, exist_ok=True)
+        os.environ.update(HF_HUB_OFFLINE='1', TRANSFORMERS_OFFLINE='1',
+                          HF_HOME=str(self.cache / 'hf'), TORCHINDUCTOR_CACHE_DIR=str(self.cache / 'torch'),
+                          TOKENIZERS_PARALLELISM='false', CUBLAS_WORKSPACE_CONFIG=':4096:8')
+
+    def _load_tokenizer(self):
+        self._prepare_environment()
+        from tokenizers import Tokenizer
+        self._tokenizer = Tokenizer.from_file(str(self.root / 'tokenizer/tokenizer.json'))
+        self._token_config = json.loads((self.root / 'tokenizer/tokenizer_config.json').read_bytes())
+
+    def _load_model(self):
+        self.report = inspect_artifact(self.root, self.revision)
+        import torch
+        if (not torch.cuda.is_available() or torch.cuda.get_device_capability() != (12, 0)
+                or torch.cuda.get_device_name() != 'NVIDIA GeForce RTX 5080'):
+            raise ValueError('tested_local_CUDA_target_required_no_fallback')
+        from transformers.initialization import no_init_weights
+        from transformers.models.modernbert.configuration_modernbert import ModernBertConfig
+        from transformers.models.modernbert.modeling_modernbert import ModernBertModel
+        from ._model import JointGroupModel
+        torch.set_num_threads(4)
+        torch.set_num_interop_threads(1)
+        torch.use_deterministic_algorithms(True)
+        torch.backends.cuda.matmul.allow_tf32 = False
+        torch.backends.cudnn.allow_tf32 = False
+        torch.backends.cudnn.benchmark = False
+        config = ModernBertConfig.from_pretrained(self.root, local_files_only=True)
+        config._attn_implementation = 'sdpa'
+        with no_init_weights():
+            model = JointGroupModel(ModernBertModel(config))
+        state = torch.load(self.root / 'checkpoint.pt', map_location='cpu', weights_only=True, mmap=True)
+        if any(not isinstance(value, torch.Tensor) or value.dtype != torch.float32
+               or not torch.isfinite(value).all() for value in state.values()):
+            raise ValueError('finite_FP32_source_tensors_required')
+        model.load_state_dict(state, strict=True, assign=True)
+        dtype = torch.float32 if self.precision == 'fp32' else torch.float16
+        self._model = model.to(device='cuda', dtype=dtype).eval()
+        self._torch = torch
+
+    def score(self, request):
+        """Admit the entire request before model creation, then one encoder forward."""
+        self.last_metrics = None
+        request = validate_request(request, self.revision)
+        if self._tokenizer is None:
+            self._load_tokenizer()
+        windows, packing = _windows.pack_windows(self._tokenizer, self._token_config, request)
+        if len(windows) != 1 or packing['context_tokens_per_window']:
+            raise ValueError('single_complete_window_required_no_truncation')
+        if self._model is None:
+            self._load_model()
+        torch = self._torch
+        dtype = torch.float32 if self.precision == 'fp32' else torch.float16
+        parameters = list(self._model.parameters())
+        if not parameters or any(p.device.type != 'cuda' or p.dtype != dtype for p in parameters):
+            raise ValueError('declared_CUDA_precision_required_no_fallback')
+        ids, ranges, indices = windows[0]
+        if indices != list(range(len(request['groups']))):
+            raise ValueError('score_alignment')
+        inputs = torch.tensor([ids], device='cuda')
+        with torch.inference_mode():
+            scores = self._model(inputs, torch.ones_like(inputs), ranges).cpu().tolist()
+        encoded = encode_response(request, scores)
+        self.last_metrics = dict(quality='unqualified', qualification=False, encoder_forwards=1,
+                                 device='cuda', precision=self.precision, packing=packing)
+        return json.loads(encoded)

tokenfold_select-0.5.2.dist-info/METADATA

@@ -0,0 +1,332 @@
+Metadata-Version: 2.4
+Name: tokenfold-select
+Version: 0.5.2
+Summary: Opt-in ONNX Select v2 scorer for Tokenfold
+License:                                  Apache License
+                                   Version 2.0, January 2004
+                                http://www.apache.org/licenses/
+        
+           TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+        
+           1. Definitions.
+        
+              "License" shall mean the terms and conditions for use, reproduction,
+              and distribution as defined by Sections 1 through 9 of this document.
+        
+              "Licensor" shall mean the copyright owner or entity authorized by
+              the copyright owner that is granting the License.
+        
+              "Legal Entity" shall mean the union of the acting entity and all
+              other entities that control, are controlled by, or are under common
+              control with that entity. For the purposes of this definition,
+              "control" means (i) the power, direct or indirect, to cause the
+              direction or management of such entity, whether by contract or
+              otherwise, or (ii) ownership of fifty percent (50%) or more of the
+              outstanding shares, or (iii) beneficial ownership of such entity.
+        
+              "You" (or "Your") shall mean an individual or Legal Entity
+              exercising permissions granted by this License.
+        
+              "Source" form shall mean the preferred form for making modifications,
+              including but not limited to software source code, documentation
+              source, and configuration files.
+        
+              "Object" form shall mean any form resulting from mechanical
+              transformation or translation of a Source form, including but
+              not limited to compiled object code, generated documentation,
+              and conversions to other media types.
+        
+              "Work" shall mean the work of authorship, whether in Source or
+              Object form, made available under the License, as indicated by a
+              copyright notice that is included in or attached to the work
+              (an example is provided in the Appendix below).
+        
+              "Derivative Works" shall mean any work, whether in Source or Object
+              form, that is based on (or derived from) the Work and for which the
+              editorial revisions, annotations, elaborations, or other modifications
+              represent, as a whole, an original work of authorship. For the purposes
+              of this License, Derivative Works shall not include works that remain
+              separable from, or merely link (or bind by name) to the interfaces of,
+              the Work and Derivative Works thereof.
+        
+              "Contribution" shall mean any work of authorship, including
+              the original version of the Work and any modifications or additions
+              to that Work or Derivative Works thereof, that is intentionally
+              submitted to Licensor for inclusion in the Work by the copyright owner
+              or by an individual or Legal Entity authorized to submit on behalf of
+              the copyright owner. For the purposes of this definition, "submitted"
+              means any form of electronic, verbal, or written communication sent
+              to the Licensor or its representatives, including but not limited to
+              communication on electronic mailing lists, source code control systems,
+              and issue tracking systems that are managed by, or on behalf of, the
+              Licensor for the purpose of discussing and improving the Work, but
+              excluding communication that is conspicuously marked or otherwise
+              designated in writing by the copyright owner as "Not a Contribution."
+        
+              "Contributor" shall mean Licensor and any individual or Legal Entity
+              on behalf of whom a Contribution has been received by Licensor and
+              subsequently incorporated within the Work.
+        
+           2. Grant of Copyright License. Subject to the terms and conditions of
+              this License, each Contributor hereby grants to You a perpetual,
+              worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+              copyright license to reproduce, prepare Derivative Works of,
+              publicly display, publicly perform, sublicense, and distribute the
+              Work and such Derivative Works in Source or Object form.
+        
+           3. Grant of Patent License. Subject to the terms and conditions of
+              this License, each Contributor hereby grants to You a perpetual,
+              worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+              (except as stated in this section) patent license to make, have made,
+              use, offer to sell, sell, import, and otherwise transfer the Work,
+              where such license applies only to those patent claims licensable
+              by such Contributor that are necessarily infringed by their
+              Contribution(s) alone or by combination of their Contribution(s)
+              with the Work to which such Contribution(s) was submitted. If You
+              institute patent litigation against any entity (including a
+              cross-claim or counterclaim in a lawsuit) alleging that the Work
+              or a Contribution incorporated within the Work constitutes direct
+              or contributory patent infringement, then any patent licenses
+              granted to You under this License for that Work shall terminate
+              as of the date such litigation is filed.
+        
+           4. Redistribution. You may reproduce and distribute copies of the
+              Work or Derivative Works thereof in any medium, with or without
+              modifications, and in Source or Object form, provided that You
+              meet the following conditions:
+        
+              (a) You must give any other recipients of the Work or
+                  Derivative Works a copy of this License; and
+        
+              (b) You must cause any modified files to carry prominent notices
+                  stating that You changed the files; and
+        
+              (c) You must retain, in the Source form of any Derivative Works
+                  that You distribute, all copyright, patent, trademark, and
+                  attribution notices from the Source form of the Work,
+                  excluding those notices that do not pertain to any part of
+                  the Derivative Works; and
+        
+              (d) If the Work includes a "NOTICE" text file as part of its
+                  distribution, then any Derivative Works that You distribute must
+                  include a readable copy of the attribution notices contained
+                  within such NOTICE file, excluding those notices that do not
+                  pertain to any part of the Derivative Works, in at least one
+                  of the following places: within a NOTICE text file distributed
+                  as part of the Derivative Works; within the Source form or
+                  documentation, if provided along with the Derivative Works; or,
+                  within a display generated by the Derivative Works, if and
+                  wherever such third-party notices normally appear. The contents
+                  of the NOTICE file are for informational purposes only and
+                  do not modify the License. You may add Your own attribution
+                  notices within Derivative Works that You distribute, alongside
+                  or as an addendum to the NOTICE text from the Work, provided
+                  that such additional attribution notices cannot be construed
+                  as modifying the License.
+        
+              You may add Your own copyright statement to Your modifications and
+              may provide additional or different license terms and conditions
+              for use, reproduction, or distribution of Your modifications, or
+              for any such Derivative Works as a whole, provided Your use,
+              reproduction, and distribution of the Work otherwise complies with
+              the conditions stated in this License.
+        
+           5. Submission of Contributions. Unless You explicitly state otherwise,
+              any Contribution intentionally submitted for inclusion in the Work
+              by You to the Licensor shall be under the terms and conditions of
+              this License, without any additional terms or conditions.
+              Notwithstanding the above, nothing herein shall supersede or modify
+              the terms of any separate license agreement you may have executed
+              with Licensor regarding such Contributions.
+        
+           6. Trademarks. This License does not grant permission to use the trade
+              names, trademarks, service marks, or product names of the Licensor,
+              except as required for reasonable and customary use in describing the
+              origin of the Work and reproducing the content of the NOTICE file.
+        
+           7. Disclaimer of Warranty. Unless required by applicable law or
+              agreed to in writing, Licensor provides the Work (and each
+              Contributor provides its Contributions) on an "AS IS" BASIS,
+              WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+              implied, including, without limitation, any warranties or conditions
+              of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+              PARTICULAR PURPOSE. You are solely responsible for determining the
+              appropriateness of using or redistributing the Work and assume any
+              risks associated with Your exercise of permissions under this License.
+        
+           8. Limitation of Liability. In no event and under no legal theory,
+              whether in tort (including negligence), contract, or otherwise,
+              unless required by applicable law (such as deliberate and grossly
+              negligent acts) or agreed to in writing, shall any Contributor be
+              liable to You for damages, including any direct, indirect, special,
+              incidental, or consequential damages of any character arising as a
+              result of this License or out of the use or inability to use the
+              Work (including but not limited to damages for loss of goodwill,
+              work stoppage, computer failure or malfunction, or any and all
+              other commercial damages or losses), even if such Contributor
+              has been advised of the possibility of such damages.
+        
+           9. Accepting Warranty or Additional Liability. While redistributing
+              the Work or Derivative Works thereof, You may choose to offer,
+              and charge a fee for, acceptance of support, warranty, indemnity,
+              or other liability obligations and/or rights consistent with this
+              License. However, in accepting such obligations, You may act only
+              on Your own behalf and on Your sole responsibility, not on behalf
+              of any other Contributor, and only if You agree to indemnify,
+              defend, and hold each Contributor harmless for any liability
+              incurred by, or claims asserted against, such Contributor by reason
+              of your accepting any such warranty or additional liability.
+        
+           END OF TERMS AND CONDITIONS
+        
+           APPENDIX: How to apply the Apache License to your work.
+        
+              To apply the Apache License to your work, attach the following
+              boilerplate notice, with the fields enclosed by brackets "[]"
+              replaced with your own identifying information. (Don't include
+              the brackets!)  The text should be enclosed in the appropriate
+              comment syntax for the file format. We also recommend that a
+              file or class name and description of purpose be included on the
+              same "printed page" as the copyright notice for easier
+              identification within third-party archives.
+        
+           Copyright [yyyy] [name of copyright owner]
+        
+           Licensed under the Apache License, Version 2.0 (the "License");
+           you may not use this file except in compliance with the License.
+           You may obtain a copy of the License at
+        
+               http://www.apache.org/licenses/LICENSE-2.0
+        
+           Unless required by applicable law or agreed to in writing, software
+           distributed under the License is distributed on an "AS IS" BASIS,
+           WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+           See the License for the specific language governing permissions and
+           limitations under the License.
+        
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: onnx
+Requires-Dist: onnxruntime==1.30.0; extra == "onnx"
+Requires-Dist: numpy==2.5.3; extra == "onnx"
+Requires-Dist: tokenizers==0.23.2; extra == "onnx"
+Dynamic: license-file
+
+# Tokenfold Select v2
+
+**Smaller contexts. Original words. Local ONNX scoring.**
+
+The optional `tokenfold-select` package defaults to the **Select v2 FP32 ONNX CPU
+scorer**. It ranks whole caller-declared groups jointly in one bounded encoder
+window; Tokenfold's existing allocator keeps required content, original order
+and the final token-budget check. It never generates replacement text.
+
+This is an **experimental preview**, not a quality- or performance-qualified
+production model. Core compression stays model-free and deterministic. Installing
+this package does not enable selection, download weights, send data to a service,
+or change the `tokenfold` Python/TypeScript compression APIs.
+
+## Install and inspect
+
+```sh
+python -m pip install 'tokenfold-select[onnx]==0.5.2'
+python -m tokenfold_select inspect-onnx --artifact /absolute/select-v2-model
+```
+
+Provision the separate ONNX model directory before use. It must contain
+`model.onnx`, `model.onnx.data`, the pinned tokenizer, runtime reference files and
+`onnx-manifest.json`. The default manifest hash is
+`267313718e80aa91a56923dea30363fd5c24fde3782c716d6d048891860a2054`;
+model revision is the source checkpoint hash
+`4dbdb5071e7d6316d8eb70dc161b1f2f77f555012f6616d7fa34311188e24cec`.
+The package verifies every computational file before inference. Do not substitute
+an untrusted manifest pin. Artifact inspection requires no inference dependencies.
+
+## Use with the CLI
+
+Create an existing, private scratch directory. Explicitly approve your native
+Python executable, installed package/dependency directory and model files:
+
+```sh
+python -m tokenfold_select configure \
+  --artifact /absolute/select-v2-model \
+  --site-packages /absolute/venv/lib/python3.12/site-packages \
+  --scratch-root /absolute/private-scratch \
+  --approved-by your-name > select-scorer.json
+
+tokenfold --experimental select context.json --query 'find the relevant item' \
+  --target-tokens 256 --scorer-config select-scorer.json \
+  --inference-timeout-ms 30000
+```
+
+Use actual absolute paths for your platform (Windows uses `Lib/site-packages`).
+`configure` checks model-file integrity and emits the existing approved-scorer
+JSON; it performs no inference. Inspect the file before use. The command pins the
+**native base interpreter**, not a Windows venv shim; the isolated bootstrap loads
+this installed package and the explicitly provisioned site-packages, without
+`.pth` processing. Protect auxiliary code, libraries, model files and scratch
+storage. This is not an OS sandbox.
+
+A minimal context document is:
+
+```json
+{"prefix":"","suffix":"","groups":[{"id":"required","text":"Keep this.\n","required":true},{"id":"optional","text":"Other context.\n","fallback_score":0.1}]}
+```
+
+The worker defaults to `--backend onnx`, explicitly selects
+`CPUExecutionProvider`, disables provider fallback and hides GPUs before loading
+inference libraries. It uses bounded file IPC, no network or descendants, and an
+owned direct-child deadline. A receipt with `used_scorer: false` means fallback,
+not successful model-backed selection. Missing approval retains the existing
+caller-declared deterministic fallback; there is no implicit model execution.
+
+## Resident API
+
+```python
+from tokenfold_select import OnnxScorer
+
+scorer = OnnxScorer('/absolute/select-v2-model', development=True)
+request = {
+    'schema_version': 1,
+    'model_revision': scorer.revision,
+    'query': 'find the relevant item',
+    'groups': [{'id': 'item', 'text': 'Original context.'}],
+}
+response = scorer.score(request)  # finite, id-aligned scores only
+```
+
+Use a dedicated process with caller-owned deadlines for the resident API. The
+CLI owns and waits for its one-shot worker; resident callers own their process.
+Neither API owns grouping, protected-content policy or final allocation.
+
+## Bounds and evidence
+
+One complete window: at most 8,192 tokens, 512 groups, a 1 MiB request and a
+64 KiB response. Overflow is refused, not truncated or silently windowed. ONNX
+Runtime 1.30.0, NumPy 2.5.3 and tokenizers 0.23.2 are pinned; no PyTorch inference
+dependency is required for this backend. No runtime binaries or weights are bundled.
+
+The exported graph passed six synthetic CPU parity cases against the source
+model, including dynamic lengths through 2,048 tokens and 512 pooled ranges.
+This is mechanics evidence, **not** full-8k execution, representative downstream
+quality, clean-user capacity or cold/warm compression latency qualification.
+Historical PyTorch FP16 timings do not measure this ONNX CPU backend. Keep
+`--experimental` and the explicit development acknowledgement until qualified.
+
+## Legacy development backend
+
+`DevelopmentScorer` and `inspect` retain the private PyTorch artifact contract.
+Use `worker --backend pytorch` explicitly for that FP32 direct-child prototype;
+it still requires its original pinned environment and artifact. The resident API
+also retains its explicit development-only FP16 option. Select v1's separately
+published LoRA model and historical benchmark results are not v2 evidence.
+
+## Safetensors model distribution
+
+The separate model release bundle also includes `model.safetensors` plus its
+custom joint-group architecture and pinned CPU reference loader. Safetensors is
+an alternative weight format, not a stock Transformers classification checkpoint
+or an automatic backend in this package. ONNX remains this worker's default.
+Use the model card's `SafetensorsScorer` instructions and manifest; representative
+quality/performance qualification is separate from format/parity validation.

tokenfold_select-0.5.2.dist-info/RECORD

@@ -0,0 +1,14 @@
+tokenfold_select/__init__.py,sha256=cIdHOhgD9FU5SHpU-Y6Q_r9ewl7YXL-MK_8uyW2s6u8,307
+tokenfold_select/__main__.py,sha256=6Fz-gEqNK0F6JyOG-htP5eH500ZY3ifVDf_xMq89-U0,6822
+tokenfold_select/_model.py,sha256=qQzx7TJ05NzZqWjitRF2q_iDK_o2B24HvhRn0cSaTWQ,2741
+tokenfold_select/_windows.py,sha256=-oefuJbjBaYP8Q8_WmojiH2hG5dBohDi3AWBVOFI2Sk,5353
+tokenfold_select/artifact.py,sha256=jMRnhT4bVkQyt_kEJCYZXWtQ6rJf47saYVuDiTMouqA,4096
+tokenfold_select/bootstrap.py,sha256=uk7Sg6pTUXGpITfi-s_hWaLHEatn-N7QyQHUT5R8LX0,848
+tokenfold_select/onnx.py,sha256=a5sLuAcZ0hNlDjZpCGDwcyvmjHn6OuFkEJOnJPDM6WQ,4985
+tokenfold_select/protocol.py,sha256=EVFf5elo1eOeoi6bMKcXkDqj-qH5c9T2XYmkvSkt6Qg,2635
+tokenfold_select/runtime.py,sha256=5zERlr59oGe6fo6VPrdKgw1IdFGzvJy8Z-jTFlQnMeY,6404
+tokenfold_select-0.5.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+tokenfold_select-0.5.2.dist-info/METADATA,sha256=1KGRtynpOCLAH41Ul4N_EWbcrxCHLQsgvNHq-_sUDKM,18921
+tokenfold_select-0.5.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+tokenfold_select-0.5.2.dist-info/top_level.txt,sha256=DTILb8zd3o0tMbQk4GXLjqKOCNxqd6J5sTtpOqOcYOo,17
+tokenfold_select-0.5.2.dist-info/RECORD,,

tokenfold_select-0.5.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tokenfold_select-0.5.2.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

tokenfold_select-0.5.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+tokenfold_select