embedflow 0.1.0__py3-none-any.whl

embedflow-0.1.0.dist-info/RECORD

@@ -0,0 +1,64 @@
+embedflow/__init__.py,sha256=ttyYjZlh_OhOxQFjmW9VeCDP6Ob8rHtxbUXYHI1sXRY,866
+embedflow/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+embedflow/analysis.py,sha256=O7HKJoYRTIQW_JO-dpaEsk0ALsM2EzDTUM3htz71CbM,8885
+embedflow/cli.py,sha256=n162kBnLvcso-YAKpx7YtYSIsePkyPOaTtZj95rM72g,73249
+embedflow/config.py,sha256=nRZBxAwUikcp47aA_hKn2kz1Z9pTpIcY8zHcshA6Elo,20970
+embedflow/runtime.py,sha256=z9n8YVekEFt_d4b_v78X6aWh4mbhIqtaxz41gM5jQ3c,6515
+embedflow/cache/__init__.py,sha256=tedr6nCGjocGGahMQDD1ZCk39npR35vvv6bO4EX5sj4,184
+embedflow/cache/base.py,sha256=FZNXTmROQDsxtMjAPif8NFepwW3LT3Cwy3UTgIZSLTI,692
+embedflow/cache/persistent_cache.py,sha256=oU_5zwscTjfVQQpubMCabZPJaiv5o0hl_i2ww9_YQ48,10390
+embedflow/compatibility/__init__.py,sha256=AJGj9A_jJr7zb7xOk1NOm6Ic_1JYGs9k6Z1DV0Tg7o4,964
+embedflow/compatibility/candidate_gap.py,sha256=8OBHqXywEozB8p1UA9pnkSGo7WT1ufbQrfWVUZQE-X8,4576
+embedflow/compatibility/containment.py,sha256=oR8YXtQgO_Xm78WdNEsq2L9Tub0cQQhl_WroW0Wz_w8,772
+embedflow/compatibility/evaluate.py,sha256=nqzNSQVo5jz1Ka36Xxgb98QNvhXL79NmRvSbfKJvAYg,14728
+embedflow/compatibility/metrics.py,sha256=WKZ3HdVH_8I5pBDEYOLZFxhIXL_AuGIuUXuTCplRNL4,3652
+embedflow/compatibility/migration_depth.py,sha256=XPs6QOtIosNnR30KRSqqubzatPlh8lQyC8j4874xtHo,3132
+embedflow/compatibility/probe.py,sha256=DXkxX--RNz6R3YF2_0PTuKM92iW0EEQ6x37V4EtFZFE,818
+embedflow/compatibility/report.py,sha256=WBHyxpqsP2FSciTzXxkZqy9fQPurKa621-G3Qh-0scU,5604
+embedflow/compatibility/t2.py,sha256=EMWVyjmaS6qEhLfuA8DJDeKzd7CmMtVpCsI4hwfcfmE,2536
+embedflow/data/__init__.py,sha256=sobXXMploY5aPtMW0CStpdqvJuINinVP7RzwoeF3kfs,49
+embedflow/data/registry/__init__.py,sha256=I3VOImw5Im7Nhe9qbdfJdlBHOwIzY4YOoqezXf-r-oU,54
+embedflow/data/registry/benchmark_profiles.jsonl,sha256=TvK9_J8YA-8QhACYa-agaRWHnTAuFKJ1f8CRwytbblQ,4388
+embedflow/data/registry/checksums.sha256,sha256=QdcuqJTKlkT3aPxFmahsDPQoqy8rXpJRR4Ba4KI7LGs,350
+embedflow/data/registry/migrations.jsonl,sha256=FHHnqTfYfuuljRmoAfuMFH9pgtLBUitdpD9phrkoANU,39100
+embedflow/data/registry/registry_manifest.json,sha256=y_AX10bhUid0BfGvnMVK9NXpz0RYm3ymf5dXuEKJuF8,788
+embedflow/data/registry/research_summaries.json,sha256=_L8voEVvw_wuMNG5RbBefX3PoRtxh5EM7Yo_oOafZyA,2494
+embedflow/data/registry/schema_version.json,sha256=w9hRvP5m2cwaq8Tsi2FRIRDYgCrAvMevMjmprGzsFKk,120
+embedflow/frozen/T2_V1_FROZEN_SPEC.md,sha256=bqNINz1f8lesD6TOCCB6hMZH-1ZwEvLWRZH7BXQo4Jo,2820
+embedflow/frozen/T2_V1_FROZEN_SPEC.sha256,sha256=sDh5kHsCAnLo6mWzV87PO2uhPtJR0mswvvVz1v5Ku2A,65
+embedflow/indexes/__init__.py,sha256=sGFYu1JJJnotXeJ9AcAueG8zyJ_dM5XthUiXtBCuz7c,252
+embedflow/indexes/base.py,sha256=TUzQEQPwJBfocFIFfByEi4FtoSc70tGS5XcQRBjNZ8Q,1723
+embedflow/indexes/faiss_backend.py,sha256=bf2betxhRFaaqgmCnhtlB8mT6iUnJN0PF2TumuBFVSI,13339
+embedflow/indexes/qdrant_backend.py,sha256=r0RyEmm7Qc5eNSc6uS9B-8UG6aa2Zp26-HJ5y_GbFP0,11995
+embedflow/metrics/__init__.py,sha256=quUSPg7lDGG9FTR1ViDUTiIic3DWFEqO50FhV3porL4,114
+embedflow/metrics/latency.py,sha256=8BHrks-5znViJn1CMPynAeoERPESRNHbZMOXvu44N-Q,2508
+embedflow/migration/__init__.py,sha256=wYgp_Gr8W3ZfPSMSJh4vGzRD2gsWQLXiyn1g2srhaPA,96
+embedflow/migration/compatibility.py,sha256=LZADCO9o9vNUMNX-b6gnjDm5L8l84mZlEnZlPtH54dc,8813
+embedflow/migration/facade.py,sha256=oeWrw4Bw7UfIw-9UW6mzyGsjtgieB9KSY1vix3m1GsU,13634
+embedflow/migration/materializer.py,sha256=FmPoYaL4BoomTQ8sZBjUvOzz5t4EhbmV77MU1bND1D0,9343
+embedflow/migration/planner.py,sha256=RMGTX-H0ceTHN-ZSZ4CZkRVHJA7H-X82iN4CloQZB3A,4048
+embedflow/migration/state.py,sha256=aSSXNI_l6y4yryUcaOw2P2ASNadeNxBWIl-bweRcSTY,4280
+embedflow/models/__init__.py,sha256=igk-3hE0CoBVO9TRlkzg1VLf6JB4BGGyBBhkOuBBIxU,231
+embedflow/models/base.py,sha256=E8WmLKc9w5UQM1edb_y1RHvOUUQHaWS4gVzP_TWGtI4,830
+embedflow/models/huggingface.py,sha256=Ns6KwQ16mda0H6b5nJ6UzYY4toqw9IOhVncMUYOIF3s,11950
+embedflow/registry/__init__.py,sha256=Ioms1I8IpCU6qI8Sbo_-FVgkOfj8LwoKdgdLwmYwSOM,1063
+embedflow/registry/loader.py,sha256=e-gSG3gXlCbJWK3ltAam1nS7QaOfx5FK7ZTMj-8UKqk,42104
+embedflow/registry/matcher.py,sha256=xrjA8qcsRRKD1yECJa9MHp3fMQJ0RNioN57Iru4S0IQ,8741
+embedflow/registry/schema.py,sha256=72OoY8WcfLusao69Xf9aQCq0r4x_SLNxkXLSjhND2Wo,13279
+embedflow/serving/__init__.py,sha256=6fMxYNv0TcGO2Kk-f3pMKD9l-p-8GxtliFLlYmYj0s4,67
+embedflow/serving/api.py,sha256=tvmuy3zdOP5QoNZ_1Pjc2GBMuv5ntkgCeMbfBW0mYK0,9718
+embedflow/serving/engine.py,sha256=GLex8TW4gOf9RqyS4Ts-NrLS2vGwffY2O_2v1XIuWMU,13965
+embedflow/serving/factory.py,sha256=81jp0PxbW3Nm0Sy96VsSSSw-yxkU7H2bOGfAKHxl380,61
+embedflow/serving/schemas.py,sha256=cNyTBfGfIuFs5Iv4xCqvCeft6jk51rRucNCLO7P0IAU,1258
+embedflow-0.1.0.dist-info/licenses/LICENSE,sha256=zNpul0odgfK2N1NDGJJObR26Wqk7rCgDEVJ5kRfV3a8,10205
+src/__init__.py,sha256=ND_FEmf-MoSPOTTZF3LjNPf5vfu4MKF13BD-PF5cRu4,62
+src/embed.py,sha256=9PHoyjUDaL1yk2k3nXOLgJsLoP-1_KnJGhr59TE4DrY,7767
+src/probe_features.py,sha256=t5AKGmlE60eqyVF27-ajgFur-7iqxy9-e_PiHZmYKZg,1749
+src/storage.py,sha256=L3uvThjIWY6gEO6bAX_Pn1tbsw6OipmKgPuImdlcNUQ,3166
+src/t2_v1.py,sha256=AH8ILrah2tAlmj2lA7WHiEDnwrt1PPrjjA1kuU-GXHM,1299
+src/utils.py,sha256=ul4w7p6CTUKO-FpQn6V4rXreEabMGIhwponbdNcUVMQ,2165
+embedflow-0.1.0.dist-info/METADATA,sha256=XSkm76irfA9f3FVwIGyljpBHX5PPyPUYNbdsI3LxVtA,7387
+embedflow-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+embedflow-0.1.0.dist-info/entry_points.txt,sha256=w7hwsj31zKortT7VcjPY7nUPQHGM2KEJzF80cxwcoUU,49
+embedflow-0.1.0.dist-info/top_level.txt,sha256=DYwVeyaEuIxHJJGK9uJGqZUQvdiCuJQSuVzpHT-cXbI,14
+embedflow-0.1.0.dist-info/RECORD,,

embedflow-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

embedflow-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+embedflow = embedflow.cli:main

embedflow-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,178 @@
+Copyright 2026 Arnav Srivastav
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

embedflow-0.1.0.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+embedflow
+src

src/__init__.py

@@ -0,0 +1 @@
+"""EmbedFlow million-document scale experiment components."""

src/embed.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+import gc, json, time
+from pathlib import Path
+import numpy as np
+from .storage import write_shard, valid_shard
+from .utils import progress
+
+class DummyEncoder:
+    def __init__(self, dimension:int, seed:int): self.dimension=dimension; self.seed=seed
+    def encode(self,texts,role='document'):
+        out=[]
+        for text in texts:
+            import hashlib
+            rng=np.random.default_rng(int(hashlib.sha256((str(self.seed)+role+'\0'+text).encode()).hexdigest()[:16],16))
+            v=rng.normal(size=self.dimension).astype('float32'); v/=max(np.linalg.norm(v),1e-12); out.append(v)
+        return np.asarray(out,dtype='float32'),sum(max(1,len(x.split())) for x in texts)
+    def close(self): pass
+
+class TransformersEncoder:
+    """Qwen-style encoder following the frozen model contract.
+
+    The implementation deliberately supports CPU fallback for the tiny local
+    verification run.  Remote jobs still select CUDA automatically and only
+    the batch size may be reduced after an OOM.
+    """
+    def __init__(self, config:dict, model_dir:Path, device='cuda'):
+        import torch
+        from transformers import AutoModel, AutoTokenizer
+        self.c=config; self.device=torch.device(device); self.tok=AutoTokenizer.from_pretrained(str(model_dir),revision=config['revision'],local_files_only=True,use_fast=True)
+        self.tok.padding_side=config.get('padding_side','left'); self.tok.truncation_side='right'; self.tok.pad_token=self.tok.pad_token or self.tok.eos_token
+        if self.device.type == 'cuda':
+            dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() and config.get('dtype_preference')=='bfloat16' else torch.float16
+        else:
+            dtype=torch.float32
+        self.net=AutoModel.from_pretrained(str(model_dir),revision=config['revision'],local_files_only=True,torch_dtype=dtype,low_cpu_mem_usage=True,attn_implementation='sdpa').eval().to(self.device)
+    def encode(self,texts,role='document'):
+        import torch
+        templ=self.c.get('query_instruction','') if role=='query' else self.c.get('document_instruction','')
+        texts=[templ.replace('{text}',x) if templ else x for x in texts]
+        e=self.tok(texts,padding=True,truncation=True,max_length=int(self.c.get('max_length',8192)),return_tensors='pt').to(self.device)
+        amp = self.device.type == 'cuda'
+        context = torch.autocast('cuda',dtype=next(self.net.parameters()).dtype) if amp else torch.autocast('cpu', enabled=False)
+        with torch.inference_mode(), context:
+            o=self.net(**e); h=o.last_hidden_state; pos=torch.full((h.shape[0],),h.shape[1]-1,dtype=torch.long,device=h.device); v=h[torch.arange(h.shape[0],device=h.device),pos]; v=torch.nn.functional.normalize(v.float(),p=2,dim=1)
+        return v.cpu().numpy().astype('float32'),int(e['attention_mask'].sum().item())
+    def close(self):
+        import torch
+        del self.net,self.tok; gc.collect()
+        if torch.cuda.is_available(): torch.cuda.empty_cache()
+
+class SentenceEncoder:
+    def __init__(self, config:dict, model_dir:Path, device='cuda'):
+        from sentence_transformers import SentenceTransformer
+        kwargs={'device':device}
+        # Older sentence-transformers versions accept local_files_only only
+        # through the underlying HF loader; a local snapshot is always passed
+        # here, so avoiding the kwarg keeps this compatible across versions.
+        self.model=SentenceTransformer(str(model_dir),**kwargs)
+        self.model.max_seq_length=int(config.get('max_length',512))
+    def encode(self,texts,role='document'):
+        v=self.model.encode(texts,batch_size=32,convert_to_numpy=True,normalize_embeddings=True,show_progress_bar=False)
+        return np.asarray(v,dtype='float32'),sum(max(1,len(x.split())) for x in texts)
+    def close(self): del self.model; gc.collect()
+
+def make_encoder(config, model_dir:Path, smoke=False, seed=20270823, device='cuda'):
+    if smoke: return DummyEncoder(int(config['dimension']),seed+hash(config['model_id'])%10000)
+    return SentenceEncoder(config,model_dir,device) if str(config['model_id']).startswith('sentence-transformers/') else TransformersEncoder(config,model_dir,device)
+
+def is_oom(exc: Exception) -> bool:
+    return 'out of memory' in str(exc).lower() or 'cuda oom' in str(exc).lower()
+
+def encode_resilient(encoder, texts:list[str], role:str, batch_size:int, desc: str = '') -> tuple[np.ndarray,int,int]:
+    values=[]; tokens=0; i=0; bs=max(1,int(batch_size)); min_bs=1
+    bar = None
+    if desc:
+        try:
+            from tqdm.auto import tqdm
+            import os
+            bar=tqdm(total=len(texts), desc=desc, unit='row', dynamic_ncols=True,
+                     disable=os.environ.get('EMBEDFLOW_NO_PROGRESS','').lower() in {'1','true','yes'})
+        except Exception:
+            bar=None
+    try:
+        while i<len(texts):
+            take=min(bs,len(texts)-i)
+            try:
+                v,t=encoder.encode(texts[i:i+take],role); values.append(v);tokens+=t;i+=take
+                if bar is not None: bar.update(take)
+            except Exception as e:
+                if not is_oom(e) or bs<=min_bs: raise
+                try:
+                    import torch; torch.cuda.empty_cache()
+                except Exception: pass
+                gc.collect(); bs=max(min_bs,bs//2)
+    finally:
+        if bar is not None: bar.close()
+    return (np.concatenate(values) if values else np.empty((0,getattr(encoder,'dimension',0)),dtype='float32')),tokens,bs
+
+def encode_pool(encoder, ids:list[str], texts:list[str], out:Path, dimension:int, shard_rows:int, role:str, metadata:dict, resume=True, batch_size:int=32):
+    out.mkdir(parents=True,exist_ok=True); total=0; tokens=0
+    for start in progress(range(0,len(ids),shard_rows), desc=f'{metadata.get("model_key", "model")} {role}', unit='shard', total=(len(ids)+shard_rows-1)//shard_rows):
+        idx=start//shard_rows; end=min(len(ids),start+shard_rows)
+        if resume and valid_shard(out,idx,dimension): total+=end-start; continue
+        batch=ids[start:end]; vals,tok,used_bs=encode_resilient(encoder,texts[start:end],role,batch_size); tokens+=tok
+        write_shard(out,idx,batch,vals,dict(metadata,start=start,end=end,tokens=tok,role=role,batch_size=used_bs))
+        total+=len(batch)
+    return {'rows':total,'tokens':tokens,'shards':(len(ids)+shard_rows-1)//shard_rows}
+
+def encode_pool_cached(encoder, ids:list[str], texts:list[str], out:Path, dimension:int, shard_rows:int, role:str, metadata:dict, cache:dict[str,np.ndarray], resume=True, batch_size:int=32):
+    """Encode a canonical row order while reusing already-produced vectors."""
+    out.mkdir(parents=True,exist_ok=True); total=0; tokens=0
+    for start in progress(range(0,len(ids),shard_rows), desc=f'{metadata.get("model_key", "model")} {role} (cached)', unit='shard', total=(len(ids)+shard_rows-1)//shard_rows):
+        idx=start//shard_rows; end=min(len(ids),start+shard_rows)
+        if resume and valid_shard(out,idx,dimension): total+=end-start; continue
+        chunk_ids=ids[start:end]; missing=[i for i,x in enumerate(chunk_ids) if x not in cache]
+        vals=np.empty((len(chunk_ids),dimension),dtype='float32')
+        for i,x in enumerate(chunk_ids):
+            if x in cache: vals[i]=cache[x]
+        if missing:
+            fresh,tok,used_bs=encode_resilient(encoder,[texts[start+i] for i in missing],role,batch_size); tokens+=tok
+            for j,i in enumerate(missing): vals[i]=fresh[j]
+        write_shard(out,idx,chunk_ids,vals,dict(metadata,start=start,end=end,tokens=tokens,role=role,batch_size=used_bs if missing else batch_size)); total+=len(chunk_ids)
+    return {'rows':total,'tokens':tokens,'shards':(len(ids)+shard_rows-1)//shard_rows}

src/probe_features.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+import numpy as np
+
+def top10(records): return [x[0] for x in records[:10]]
+def build_features(candidates:list[list[tuple[str,float,int]]], target_scores:dict[str,float], ks=(10,20,50,100,200,500)):
+    # candidates are source-ranked (id, source_score, source_rank); target_scores are finite-pool scores.
+    rerank={k:sorted(candidates[:k],key=lambda x:(-target_scores[x[0]],x[2])) for k in ks}; final=top10(rerank[max(ks)])
+    def stability(k): return len(set(top10(rerank[k]))&set(final))/10
+    def entrants(a,b): return len(set(top10(rerank[b]))-set(top10(rerank[a])))
+    e50=entrants(50,100); e100=entrants(100,200); e200=entrants(200,500)
+    margins=[]
+    for q in [candidates]:
+        t10=rerank[50][min(9,len(rerank[50])-1)][1] if not target_scores else target_scores[rerank[50][min(9,len(rerank[50])-1)][0]]
+        new=[target_scores[x[0]] for x in q[50:100]]; margins.append(t10-max(new) if new else 0.0)
+    deepest=max((x[2] for x in rerank[500][:10]),default=0)
+    vals={
+      'probe_residual_tail_50_mean':1-stability(50),'stability_to_500_50_mean':stability(50),
+      'deepest_p90':float(deepest),'late_tail_area':(e50/10+e100/10+e200/10)/3,
+      'last_shell_any_rate':float(e200>0),'fraction_margin_nonpositive':float(np.mean(np.asarray(margins)<=0)),
+      'p_any_entrant_50_100':float(e50>0),'p_any_entrant_100_200':float(e100>0),'p_any_entrant_200_500':float(e200>0),
+      'mean_entrants_50_100':float(e50),'mean_entrants_100_200':float(e100),'mean_entrants_200_500':float(e200),
+      'deepest_source_rank_top10':deepest,'stability_10_20':stability(10),'stability_20_50':stability(20),'stability_50_100':len(set(top10(rerank[50]))&set(top10(rerank[100])))/10,
+    }
+    return vals

src/storage.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+import json, hashlib
+from pathlib import Path
+import numpy as np
+from .utils import sha256_file, atomic_json
+
+def write_shard(root: Path, index: int, ids: list[str], vectors: np.ndarray, metadata: dict) -> dict:
+    root.mkdir(parents=True,exist_ok=True); arr=np.asarray(vectors)
+    if arr.ndim!=2 or len(ids)!=arr.shape[0] or not np.isfinite(arr).all(): raise ValueError('invalid shard')
+    tmp=root/f'.shard-{index:06d}.npy.tmp'; p=root/f'shard-{index:06d}.npy';
+    with tmp.open('wb') as f: np.save(f,arr.astype(np.float16),allow_pickle=False)
+    tmp.replace(p)
+    ip=root/f'shard-{index:06d}.ids.json'; ip.write_text(json.dumps(ids,ensure_ascii=False));
+    meta=dict(metadata,shard=index,rows=len(ids),dimension=int(arr.shape[1]),dtype='float16',array_sha256=sha256_file(p),ids_sha256=sha256_file(ip))
+    atomic_json(root/f'shard-{index:06d}.json',meta); (root/f'shard-{index:06d}.done').write_text(meta['array_sha256']+'\n'); return meta
+
+def valid_shard(root: Path,index:int,dimension:int|None=None)->bool:
+    p=root/f'shard-{index:06d}.npy'; m=root/f'shard-{index:06d}.json'; d=root/f'shard-{index:06d}.done'
+    if not (p.exists() and m.exists() and d.exists()): return False
+    try:
+        meta=json.loads(m.read_text()); a=np.load(p,mmap_mode='r')
+        return a.ndim==2 and (dimension is None or a.shape[1]==dimension) and meta.get('array_sha256')==sha256_file(p) and d.read_text().strip()==meta['array_sha256']
+    except Exception:return False
+
+def load_shards(root: Path, dimension:int|None=None)->tuple[list[str],np.ndarray]:
+    arrays=[];ids=[]
+    for p in sorted(root.glob('shard-*.npy')):
+        i=int(p.stem.split('-')[1]);
+        if not valid_shard(root,i,dimension): raise ValueError(f'invalid shard {p}')
+        arrays.append(np.load(p)); ids.extend(json.loads((root/f'shard-{i:06d}.ids.json').read_text()))
+    if not arrays: raise ValueError(f'no shards in {root}')
+    return ids,np.concatenate(arrays).astype(np.float32)
+
+def iter_shards(root: Path, dimension: int|None=None):
+    """Yield (ids, mmap vectors, metadata) without concatenating the corpus."""
+    for p in sorted(root.glob('shard-*.npy')):
+        i=int(p.stem.split('-')[1])
+        if not valid_shard(root,i,dimension): raise ValueError(f'invalid shard {p}')
+        ids=json.loads((root/f'shard-{i:06d}.ids.json').read_text())
+        yield ids,np.load(p,mmap_mode='r'),json.loads((root/f'shard-{i:06d}.json').read_text())
+
+def vector_lookup(root: Path, ids: list[str], dimension: int|None=None) -> np.ndarray:
+    """Read only requested rows, grouped by shard; output follows ``ids``."""
+    wanted={str(x):i for i,x in enumerate(ids)}; out=None; found=set()
+    for shard_ids, arr, _ in iter_shards(root,dimension):
+        local=[(j,wanted[x]) for j,x in enumerate(shard_ids) if x in wanted]
+        if local:
+            if out is None: out=np.empty((len(ids),arr.shape[1]),dtype='float32')
+            for j,pos in local: out[pos]=np.asarray(arr[j],dtype='float32'); found.add(shard_ids[j])
+    if out is None or len(found)!=len(wanted): raise KeyError(f'missing {len(wanted)-len(found)} vector IDs')
+    return out

src/t2_v1.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+from pathlib import Path
+import hashlib, json
+
+K_VALUES=(10,20,50,100,200,500)
+def verify_t2_hash(root:Path)->None:
+    p=root/'frozen/T2_V1_FROZEN_SPEC.md'; expected=(root/'frozen/T2_V1_FROZEN_SPEC.sha256').read_text().strip(); actual=hashlib.sha256(p.read_bytes()).hexdigest()
+    if expected!=actual: raise RuntimeError(f'T2 frozen hash mismatch: {actual} != {expected}')
+
+def decide(features:dict)->str:
+    safe=(features['probe_residual_tail_50_mean']<=.05 and features['deepest_p90']<=200 and features['late_tail_area']<=.20)
+    if safe:return 'SAFE'
+    guard=(features['probe_residual_tail_50_mean']>.10 or features['stability_to_500_50_mean']<.90 or features['last_shell_any_rate']>.25 or features['fraction_margin_nonpositive']>.50)
+    return 'UNSAFE_OR_UNCERTAIN' if guard else 'EXPAND'
+
+def blind_predictions(rows:list[dict], out:Path)->None:
+    keys=['pair','dataset','probe_residual_tail_50_mean','stability_to_500_50_mean','deepest_p90','late_tail_area','last_shell_any_rate','fraction_margin_nonpositive']
+    out.parent.mkdir(parents=True,exist_ok=True)
+    with out.open('w') as f:
+        f.write(','.join(keys+['T2_v1_decision'])+'\n')
+        for r in rows:f.write(','.join(str(r.get(k,'')) for k in keys+[None])[:-1]+str(decide(r))+'\n')

src/utils.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+import hashlib, json, os, platform, subprocess, time
+from pathlib import Path
+from typing import Any
+
+def progress(iterable, *, desc: str = '', total=None, unit: str = 'it', leave: bool = True):
+    """Return a tqdm iterator with optional nested-bar cleanup.
+
+    Set ``EMBEDFLOW_NO_PROGRESS=1`` for quiet runs.  ``leave=False`` is useful
+    for fine-grained query bars nested inside a shard bar.
+    """
+    try:
+        from tqdm.auto import tqdm
+        disabled = os.environ.get('EMBEDFLOW_NO_PROGRESS', '').lower() in {'1', 'true', 'yes'}
+        return tqdm(iterable, desc=desc, total=total, unit=unit,
+                    disable=disabled, dynamic_ncols=True, leave=leave)
+    except Exception:
+        return iterable
+
+def sha256_file(path: Path, block: int = 1024 * 1024) -> str:
+    h = hashlib.sha256()
+    with path.open('rb') as f:
+        for b in iter(lambda: f.read(block), b''): h.update(b)
+    return h.hexdigest()
+
+def stable_hash(value: str, seed: int = 0) -> str:
+    return hashlib.sha256(f'{seed}\0{value}'.encode()).hexdigest()
+
+def atomic_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + '.tmp')
+    tmp.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False))
+    tmp.replace(path)
+
+def load_yaml(path: Path) -> dict[str, Any]:
+    import yaml
+    return yaml.safe_load(path.read_text())
+
+def command_output(args: list[str]) -> str:
+    try: return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT, timeout=30)
+    except Exception as e: return f'command failed: {e}'
+
+def environment_snapshot() -> dict[str, Any]:
+    import sys
+    out = {'python': sys.version, 'platform': platform.platform(), 'hostname': platform.node(), 'cpu': platform.processor()}
+    try:
+        import torch
+        out.update({'torch': torch.__version__, 'cuda': torch.version.cuda, 'cuda_count': torch.cuda.device_count()})
+    except Exception as e: out['torch_error'] = repr(e)
+    out['nvidia_smi'] = command_output(['nvidia-smi'])
+    return out
+
+def now() -> str: return time.strftime('%Y%m%d-%H%M%S')