dstpr 0.2.0__tar.gz → 0.2.2__tar.gz

{dstpr-0.2.0 → dstpr-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstpr
-Version: 0.2.0
+Version: 0.2.2
 Summary: A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate.
 Author-email: Stephen Meisenbacher <sjmeis@gtgd.com>
 Maintainer-email: Stephen Meisenbacher <sjmeis@gtgd.com>
@@ -94,8 +94,8 @@ If you want to integrate specific pipeline layers directly into an existing work
 ```python
 from dstpr.cleaners import clean_and_split_chunks
 from dstpr.segmenters import SemanticSectionRouter
-from dstpr.rankers import HybridTaskRanker
-from sentence_transformers import Transformer, SentenceTransformer
+from dstpr.rankers import HybridRanker
+from sentence_transformers import SentenceTransformer
 
 embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
 
@@ -110,7 +110,7 @@ buckets = router.route_sentences(sentences)
 target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
 
 # Grade and sort text features
-ranker = HybridRanker(model=shared_model)
+ranker = HybridRanker()
 final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
 ```
 

{dstpr-0.2.0 → dstpr-0.2.2}/README.md

@@ -54,8 +54,8 @@ If you want to integrate specific pipeline layers directly into an existing work
 ```python
 from dstpr.cleaners import clean_and_split_chunks
 from dstpr.segmenters import SemanticSectionRouter
-from dstpr.rankers import HybridTaskRanker
-from sentence_transformers import Transformer, SentenceTransformer
+from dstpr.rankers import HybridRanker
+from sentence_transformers import SentenceTransformer
 
 embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
 
@@ -70,7 +70,7 @@ buckets = router.route_sentences(sentences)
 target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
 
 # Grade and sort text features
-ranker = HybridRanker(model=shared_model)
+ranker = HybridRanker()
 final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
 ```
 

{dstpr-0.2.0 → dstpr-0.2.2}/dstpr/pipeline.py

@@ -47,14 +47,14 @@ class ParallelPreprocessingPipeline:
                 print(f"GPU Auto-Calibrated Batch Size: {self.inference_batch_size}")
 
                 dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16
-                bi_encoder = SentenceTransformer(
+                self.bi_encoder = SentenceTransformer(
                     self.embedding_model,
                     device="cuda",
                     model_kwargs={"dtype": dtype}
                 )
 
-                cross_encoder = CrossEncoder(
-                    "cross-encoder/ms-marco-MiniLM-L-6-v2",
+                self.cross_encoder = CrossEncoder(
+                    "cross-encoder/ms-marco-MiniLM-L6-v2",
                     device="cuda",
                     model_kwargs={"dtype": dtype}
                 )
@@ -63,11 +63,11 @@ class ParallelPreprocessingPipeline:
                 self.bi_encoder = SentenceTransformer(self.embedding_model, device=device)
                 self.cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2", device=device)
 
-            self.router = SemanticSectionRouter(profile=self.profile, model=bi_encoder)
+            self.router = SemanticSectionRouter(profile=self.profile, model=self.bi_encoder)
             self.ranker = HybridRanker(
                 profile=self.profile,
-                model=bi_encoder,
-                cross_encoder_model=cross_encoder,
+                model=self.bi_encoder,
+                cross_encoder_model=self.cross_encoder,
             )
             self._is_initialized = True
 
@@ -115,8 +115,7 @@ class ParallelPreprocessingPipeline:
 
         filtered_doc_sentences = self.ranker.rank_and_filter(
             sentences=doc_candidates,
-            batch_size=self.inference_batch_size,
-            final_threshold=threshold
+            batch_size=self.inference_batch_size
         )
 
         return [sanitize_outputs(sents, self.profile) for sents in filtered_doc_sentences]

{dstpr-0.2.0 → dstpr-0.2.2}/dstpr/rankers/hybrid.py

@@ -14,7 +14,7 @@ class HybridRanker:
         self.cross_encoder = (
             cross_encoder_model
             if cross_encoder_model
-            else CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+            else CrossEncoder("cross-encoder/ms-marco-MiniLM-L6-v2")
         )
 
         with torch.inference_mode():
@@ -61,7 +61,6 @@ class HybridRanker:
         sentences: list[list[str]],
         high_certainty_threshold: float = 0.60,
         low_certainty_threshold: float = 0.20,
-        final_threshold: float = 0.40,
         batch_size: int = 512
     ) -> list[list[str]]:
         flat_candidates = []
@@ -108,6 +107,24 @@ class HybridRanker:
             )
 
         cross_keep_set = set()
+        if needs_cross:
+            primary_anchor = self.profile.positive_semantic_anchors[0]
+            cross_inputs = [[primary_anchor, sent] for sent in needs_cross]
+
+            cross_scores = self.cross_encoder.predict(
+                cross_inputs,
+                batch_size=batch_size,
+                show_progress_bar=False,
+                convert_to_tensor=True,
+            )
+
+            with torch.inference_mode():
+                survived_flags = (cross_scores >= 0.0).cpu().tolist()
+
+            cross_keep_set = set(
+                [sent for sent, keep in zip(needs_cross, survived_flags) if keep]
+            )
+
         surviving_mask = [False] * len(flat_candidates)
         for idx, is_auto in enumerate(auto_mask):
             if is_auto:

{dstpr-0.2.0 → dstpr-0.2.2}/dstpr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstpr
-Version: 0.2.0
+Version: 0.2.2
 Summary: A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate.
 Author-email: Stephen Meisenbacher <sjmeis@gtgd.com>
 Maintainer-email: Stephen Meisenbacher <sjmeis@gtgd.com>
@@ -94,8 +94,8 @@ If you want to integrate specific pipeline layers directly into an existing work
 ```python
 from dstpr.cleaners import clean_and_split_chunks
 from dstpr.segmenters import SemanticSectionRouter
-from dstpr.rankers import HybridTaskRanker
-from sentence_transformers import Transformer, SentenceTransformer
+from dstpr.rankers import HybridRanker
+from sentence_transformers import SentenceTransformer
 
 embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
 
@@ -110,7 +110,7 @@ buckets = router.route_sentences(sentences)
 target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
 
 # Grade and sort text features
-ranker = HybridRanker(model=shared_model)
+ranker = HybridRanker()
 final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
 ```
 

{dstpr-0.2.0 → dstpr-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dstpr"
-version = "0.2.0"
+version = "0.2.2"
 description = "A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate."
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}