dstpr 0.2.2__tar.gz → 0.2.4__tar.gz

{dstpr-0.2.2 → dstpr-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstpr
-Version: 0.2.2
+Version: 0.2.4
 Summary: A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate.
 Author-email: Stephen Meisenbacher <sjmeis@gtgd.com>
 Maintainer-email: Stephen Meisenbacher <sjmeis@gtgd.com>
@@ -76,7 +76,7 @@ This is the recommended approach for large-scale data pipelines.
 ```python
 from dstpr import ParallelPreprocessingPipeline, JOB_POSTING_PROFILE
 
-pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE, batch_size=64)
+pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE)
 
 raw_documents = [
     "DOC 1",
@@ -84,41 +84,13 @@ raw_documents = [
     "..."
 ]
 
-cleaned_documents = pipeline.process(raw_documents, num_workers=8, threshold=0.25)
-```
-
-### Advanced Usage
-
-If you want to integrate specific pipeline layers directly into an existing workflow, or tweak the internal parameters, you can import individual modules manually:
-
-```python
-from dstpr.cleaners import clean_and_split_chunks
-from dstpr.segmenters import SemanticSectionRouter
-from dstpr.rankers import HybridRanker
-from sentence_transformers import SentenceTransformer
-
-embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
-
-text = "Job posting text goes here!"
-
-# Clean and segment sentences
-sentences = clean_and_split_chunks(text)
-
-# Route by context
-router = SemanticSectionRouter(model=embedding_model)
-buckets = router.route_sentences(sentences)
-target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
-
-# Grade and sort text features
-ranker = HybridRanker()
-final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
+cleaned_documents = pipeline.process(raw_documents, threshold=0.25)
 ```
 
 ### Parameter Tuning
 
 To adjust the trade-off between strict filtering and execution speed, consider tweaking these variables in `ParallelPreprocessingPipeline`:
  - `threshold` (Default: `0.4`): Controls how aggressively sentences are discarded. Raising this value toward `0.6`, for example, ensures only stronger matching sentences are passed through. Lowering it toward `0.3`, on the other hand, acts as a wider net.
- - `batch_size` (Default: `256`): This depends on your hardware! Adjust for best performance.
 
 ## Creating Custom Domain Profiles (via the provided wizard)
 

{dstpr-0.2.2 → dstpr-0.2.4}/README.md

@@ -36,7 +36,7 @@ This is the recommended approach for large-scale data pipelines.
 ```python
 from dstpr import ParallelPreprocessingPipeline, JOB_POSTING_PROFILE
 
-pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE, batch_size=64)
+pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE)
 
 raw_documents = [
     "DOC 1",
@@ -44,41 +44,13 @@ raw_documents = [
     "..."
 ]
 
-cleaned_documents = pipeline.process(raw_documents, num_workers=8, threshold=0.25)
-```
-
-### Advanced Usage
-
-If you want to integrate specific pipeline layers directly into an existing workflow, or tweak the internal parameters, you can import individual modules manually:
-
-```python
-from dstpr.cleaners import clean_and_split_chunks
-from dstpr.segmenters import SemanticSectionRouter
-from dstpr.rankers import HybridRanker
-from sentence_transformers import SentenceTransformer
-
-embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
-
-text = "Job posting text goes here!"
-
-# Clean and segment sentences
-sentences = clean_and_split_chunks(text)
-
-# Route by context
-router = SemanticSectionRouter(model=embedding_model)
-buckets = router.route_sentences(sentences)
-target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
-
-# Grade and sort text features
-ranker = HybridRanker()
-final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
+cleaned_documents = pipeline.process(raw_documents, threshold=0.25)
 ```
 
 ### Parameter Tuning
 
 To adjust the trade-off between strict filtering and execution speed, consider tweaking these variables in `ParallelPreprocessingPipeline`:
  - `threshold` (Default: `0.4`): Controls how aggressively sentences are discarded. Raising this value toward `0.6`, for example, ensures only stronger matching sentences are passed through. Lowering it toward `0.3`, on the other hand, acts as a wider net.
- - `batch_size` (Default: `256`): This depends on your hardware! Adjust for best performance.
 
 ## Creating Custom Domain Profiles (via the provided wizard)
 

{dstpr-0.2.2 → dstpr-0.2.4}/dstpr/cleaners/post_process.py

@@ -1,15 +1,18 @@
 import re
 from dstpr.profiles import DomainProfile
 
-def sanitize_outputs(sentences: list[str], profile: DomainProfile, min_length: int = 35) -> str:
+LEADING_JUNK_RE = re.compile(r'^[\s\-\d\.\,\*•■\:\>]+')
+TRAILING_NUM_RE = re.compile(r'\b\d+[\.,]\d+$|\b\d+$')
+MULTI_DOT_SPACE_RE = re.compile(r'\s*\.+\s*')
+
+def sanitize_outputs(sentences: list[str], profile: DomainProfile, min_length: int = 20) -> str:
     if not sentences:
         return ""
         
     seen = set()
     cleaned_pool = []
     
-    leading_junk_pattern = re.compile(r'^[\s\-\d\.\,\*•■\:\>]+', re.IGNORECASE)
-    junk_matcher = re.compile(profile.junk_line_regex, re.IGNORECASE) if profile.junk_line_regex else None
+    junk_matcher = profile._junk_line_re
     
     for sentence in sentences:
         sent = sentence.strip()
@@ -21,18 +24,18 @@ def sanitize_outputs(sentences: list[str], profile: DomainProfile, min_length: i
         if junk_matcher and junk_matcher.search(sent):
             continue
         
-        sent = leading_junk_pattern.sub('', sent)
+        sent = LEADING_JUNK_RE.sub('', sent)
         
         if sent:
             sent = sent[0].upper() + sent[1:]
-            sent = re.sub(r'\s*\.+\s*', '. ', sent).strip()
+            sent = MULTI_DOT_SPACE_RE.sub('. ', sent).strip()
             if not sent.endswith('.'):
                 sent += '.'
             
         if len(sent) < min_length:
             continue
             
-        if re.search(r'\b\d+[\.,]\d+$|\b\d+$', sent):
+        if TRAILING_NUM_RE.search(sent):
             continue
             
         cleaned_pool.append(sent)
@@ -41,6 +44,5 @@ def sanitize_outputs(sentences: list[str], profile: DomainProfile, min_length: i
     final_text = re.sub(r'\s+', ' ', final_text)
     final_text = re.sub(r'\s*\.\s*', '. ', final_text)
     final_text = re.sub(r'\.+', '.', final_text)
-    final_text = re.sub(r'\.+', '.', final_text)
     
     return final_text.strip()

{dstpr-0.2.2 → dstpr-0.2.4}/dstpr/cleaners/pre_process.py

@@ -2,16 +2,19 @@ import re
 import nltk
 from dstpr.profiles import DomainProfile
 
+INLINE_SPLIT_RE = re.compile(r'(?:[\•\·\■\*]|\(\d+\)|\b\d+\)|\;\s+(?=[A-Z]))')
+
 def clean_and_split_chunks(text: str, profile: DomainProfile, min_length: int = 15) -> list[str]:
     if not text or not isinstance(text, str):
         return []
 
-    if profile.raw_noise_regex:
-        text = re.sub(profile.raw_noise_regex, '', text, flags=re.IGNORECASE)
-    lines = [line.strip() for line in text.split("\n") if line.strip()]
+    if profile._raw_noise_re:
+        text = profile._raw_noise_re.sub(' ', text)
 
-    meta_matcher = re.compile(profile.metadata_line_regex, re.IGNORECASE) if profile.metadata_line_regex else None
-    
+    text = INLINE_SPLIT_RE.sub('\n', text)
+    lines = [line.strip() for line in text.split("\n") if line.strip()]
+    meta_matcher = profile._meta_line_re
+        
     normalized_chunks = []
     for line in lines:
         if meta_matcher and meta_matcher.search(line):
@@ -20,9 +23,8 @@ def clean_and_split_chunks(text: str, profile: DomainProfile, min_length: int =
         if not line.endswith(('.', '!', '?', ';', ':')):
             # If it's short (< 6 words) and lacks punctuation, it's a header line, do not append
             if len(line.split()) < 6:
-                line = line + ":"
-            else:
-                line = line + "."
+                continue
+            line = line + "."
         normalized_chunks.append(line)
         
     normalized_text = " ".join(normalized_chunks)
@@ -33,11 +35,9 @@ def clean_and_split_chunks(text: str, profile: DomainProfile, min_length: int =
     clean_sentences = []
     for sentence in sentences:
         sentence_clean = sentence.strip()
-        sentence_lower = sentence_clean.lower()
-        
         if len(sentence_clean) < min_length:
             continue
-        if profile.global_stop_phrases and any(phrase in sentence_lower for phrase in profile.global_stop_phrases):
+        if profile._stop_phrases_re and profile._stop_phrases_re.search(sentence_clean):
             continue
             
         clean_sentences.append(sentence_clean)

{dstpr-0.2.2 → dstpr-0.2.4}/dstpr/pipeline.py

@@ -74,7 +74,6 @@ class ParallelPreprocessingPipeline:
     def _cpu_preprocess_chunk(self, raw_docs: list[str]) -> dict:
         doc_sentence_counts = []
         flat_sentences = []
-        flat_windows = []
 
         for raw_text in raw_docs:
             if not raw_text or not isinstance(raw_text, str):
@@ -86,36 +85,32 @@ class ParallelPreprocessingPipeline:
             doc_sentence_counts.append(len(sents))
             flat_sentences.extend(sents)
 
-            for i in range(n_sents):
-                window = " ".join(sents[max(0, i - 1) : min(n_sents, i + 2)])
-                flat_windows.append(window)
-
         return {
             "doc_sentence_counts": doc_sentence_counts,
             "flat_sentences": flat_sentences,
-            "flat_windows": flat_windows,
             "raw_docs_count": len(raw_docs)
         }
 
     def _gpu_execution_pass(self, prep_data: dict, threshold: float) -> list[str]:
         flat_sentences = prep_data["flat_sentences"]
-        flat_windows = prep_data["flat_windows"]
         doc_sentence_counts = prep_data["doc_sentence_counts"]
         raw_docs_count = prep_data["raw_docs_count"]
 
         if not flat_sentences:
             return [""] * raw_docs_count
 
-        doc_candidates = self.router.route_sentences(
+        flat_candidates, flat_cand_embs, doc_cand_counts = self.router.route_sentences(
             sentences=flat_sentences,
-            windows=flat_windows,
             doc_sentence_counts=doc_sentence_counts,
             batch_size=self.inference_batch_size,
         )
 
         filtered_doc_sentences = self.ranker.rank_and_filter(
-            sentences=doc_candidates,
-            batch_size=self.inference_batch_size
+            flat_candidates=flat_candidates,
+            flat_cand_embs=flat_cand_embs,
+            doc_cand_counts=doc_cand_counts,
+            cross_prob_threshold=threshold,
+            batch_size=self.inference_batch_size,
         )
 
         return [sanitize_outputs(sents, self.profile) for sents in filtered_doc_sentences]

{dstpr-0.2.2 → dstpr-0.2.4}/dstpr/profiles.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass, field, asdict
 import json
 from pathlib import Path
+import re
 
 @dataclass
 class DomainProfile:
@@ -20,12 +21,24 @@ class DomainProfile:
     metadata_line_regex: str | None = None
     junk_line_regex: str | None = None
 
+    def __post_init__(self):
+        self._raw_noise_re = re.compile(self.raw_noise_regex, re.IGNORECASE) if self.raw_noise_regex else None
+        self._meta_line_re = re.compile(self.metadata_line_regex, re.IGNORECASE) if self.metadata_line_regex else None
+        self._junk_line_re = re.compile(self.junk_line_regex, re.IGNORECASE) if self.junk_line_regex else None
+        self._action_kw_re = re.compile(self.action_keywords_regex, re.IGNORECASE) if self.action_keywords_regex else None
+        self._syntax_start_re = re.compile(self.syntax_start_pattern, re.IGNORECASE) if self.syntax_start_pattern else None
+        if self.global_stop_phrases:
+            pattern = "|".join(map(re.escape, self.global_stop_phrases))
+            self._stop_phrases_re = re.compile(rf"(?:{pattern})", re.IGNORECASE)
+        else:
+            self._stop_phrases_re = None
+
     def validate(self):
         if not self.positive_semantic_anchors:
             raise ValueError("DomainProfile must define at least one positive_semantic_anchor.")
-        if not self.name.isalnum():
-            raise ValueError("Profile name must be alphanumeric (no spaces or special characters).")
-        if not self.section_anchors:
+        if not self.name.replace(" ", "").replace("_", "").isalnum():
+            raise ValueError("Profile name must be alphanumeric.")
+        if not self.anchors:
             raise ValueError("DomainProfile must define at least one section anchor.")
         if self.default_section not in self.anchors:
             raise ValueError(f"Default section '{self.default_section}' must be one of the defined section anchors.")
@@ -75,12 +88,12 @@ JOB_POSTING_PROFILE = DomainProfile(
     default_section="CORE",
     target_sections=['CORE', 'REQUIREMENTS'],
     positive_semantic_anchors=[
-        "execute daily responsibilities, manage operational workflows, deliver projects, and accomplish work assignments",
-        "design, build, develop, and maintain technical systems, codebases, software, or infrastructure",
-        "analyze data, conduct research, prepare comprehensive reports, formulate budgets, and monitor key metrics",
-        "collaborate with cross-functional teams, communicate with stakeholders, and partner across departments",
-        "ensure regulatory compliance, perform audits or inspections, manage inventory, and deliver service solutions",
-        "lead teams, assign tasks to staff, direct project timelines, and manage program resources"
+        "What are the core daily duties, responsibilities, and operational tasks performed in this role?",
+        "Responsible for designing, building, developing, and maintaining technical systems, software, or equipment.",
+        "Performs data analysis, conducts research, prepares reports, formulates budgets, and monitors program metrics.",
+        "Collaborates with cross-functional teams, advises stakeholders, and coordinates activities across departments.",
+        "Ensures regulatory compliance, performs inspections or clinical care, manages inventory, and delivers services.",
+        "Supervises staff, assigns work to subordinates, leads projects, and manages organizational resources."
     ],
     negative_semantic_anchors=[
         "company history, corporate mission statement, organizational culture, workplace values, company overview",

{dstpr-0.2.2 → dstpr-0.2.4}/dstpr/rankers/hybrid.py

@@ -58,79 +58,69 @@ class HybridRanker:
 
     def rank_and_filter(
         self,
-        sentences: list[list[str]],
-        high_certainty_threshold: float = 0.60,
+        flat_candidates: list[str],
+        flat_cand_embs: torch.Tensor,
+        doc_cand_counts: list[int],
+        high_certainty_threshold: float = 0.40,
         low_certainty_threshold: float = 0.20,
+        cross_prob_threshold: float = 0.30,
+        min_fallback_sentences: int = 3,
         batch_size: int = 512
     ) -> list[list[str]]:
-        flat_candidates = []
-        doc_cand_counts = []
-        for c_list in sentences:
-            doc_cand_counts.append(len(c_list))
-            flat_candidates.extend(c_list)
-
         if not flat_candidates:
-            return [[] for _ in sentences]
-
-        unique_candidates = list(dict.fromkeys(flat_candidates))
-        norm_unique = self._encode_sorted(unique_candidates, batch_size=batch_size)
+            return [[] for _ in doc_cand_counts]
+
+        action_re = self.profile._action_kw_re
+        if action_re:
+            lexical_boost = torch.tensor(
+                [0.06 if action_re.search(s) else 0.0 for s in flat_candidates],
+                dtype=flat_cand_embs.dtype,
+                device=flat_cand_embs.device
+            )
+        else:
+            lexical_boost = 0.0
 
         with torch.inference_mode():
-            emb_map = {s: norm_unique[i] for i, s in enumerate(unique_candidates)}
-            flat_cand_embs = torch.stack([emb_map[s] for s in flat_candidates])
-
             pos_sim = torch.matmul(flat_cand_embs, self.pos_anchors.T)
             max_pos, _ = torch.max(pos_sim, dim=1)
 
             if self.neg_anchors is not None:
                 neg_sim = torch.matmul(flat_cand_embs, self.neg_anchors.T)
                 max_neg, _ = torch.max(neg_sim, dim=1)
-                net_scores = max_pos - (0.5 * max_neg)
+                # Only penalize when negative similarity is genuinely high (> 0.30)
+                neg_penalty = torch.clamp(max_neg - 0.30, min=0.0) * 0.6
+                net_scores = max_pos - neg_penalty + lexical_boost
             else:
-                net_scores = max_pos
+                net_scores = max_pos + lexical_boost
 
-            score_min, score_max = net_scores.min().item(), net_scores.max().item()
-            if (score_max - score_min) > 1e-6:
-                norm_scores = (net_scores - score_min) / (score_max - score_min)
-            else:
-                norm_scores = net_scores
-
-            auto_mask = (norm_scores >= high_certainty_threshold).cpu().tolist()
-            cross_mask = (
-                (norm_scores >= low_certainty_threshold)
-                & (norm_scores < high_certainty_threshold)
-            ).cpu().tolist()
-            needs_cross = list(
-                dict.fromkeys(
-                    [s for s, flag in zip(flat_candidates, cross_mask) if flag]
-                )
-            )
+            auto_mask = (net_scores >= high_certainty_threshold).cpu().tolist()
+            cross_mask = ((net_scores >= low_certainty_threshold) & (net_scores < high_certainty_threshold)).cpu().tolist()
+            net_scores_cpu = net_scores.cpu().tolist()
 
+        needs_cross = list(dict.fromkeys([s for s, flag in zip(flat_candidates, cross_mask) if flag]))
         cross_keep_set = set()
         if needs_cross:
+            needs_cross_sorted = sorted(needs_cross, key=len)
             primary_anchor = self.profile.positive_semantic_anchors[0]
-            cross_inputs = [[primary_anchor, sent] for sent in needs_cross]
+            cross_inputs = [[primary_anchor, sent] for sent in needs_cross_sorted]
 
             cross_scores = self.cross_encoder.predict(
                 cross_inputs,
                 batch_size=batch_size,
                 show_progress_bar=False,
-                convert_to_tensor=True,
+                convert_to_tensor=True
             )
 
             with torch.inference_mode():
-                survived_flags = (cross_scores >= 0.0).cpu().tolist()
+                cross_probs = torch.sigmoid(cross_scores)
+                survived_flags = (cross_probs >= cross_prob_threshold).cpu().tolist()
 
-            cross_keep_set = set(
-                [sent for sent, keep in zip(needs_cross, survived_flags) if keep]
-            )
+            cross_keep_set = {sent for sent, keep in zip(needs_cross, survived_flags) if keep}
 
-        surviving_mask = [False] * len(flat_candidates)
-        for idx, is_auto in enumerate(auto_mask):
-            if is_auto:
-                surviving_mask[idx] = True
-            elif cross_mask[idx] and (flat_candidates[idx] in cross_keep_set):
-                surviving_mask[idx] = True
+        surviving_mask = [
+            is_auto or (is_cross and (flat_candidates[idx] in cross_keep_set))
+            for idx, (is_auto, is_cross) in enumerate(zip(auto_mask, cross_mask))
+        ]
 
         filtered_docs = []
         c_offset = 0
@@ -141,7 +131,21 @@ class HybridRanker:
 
             doc_pool = flat_candidates[c_offset : c_offset + c_count]
             doc_mask = surviving_mask[c_offset : c_offset + c_count]
-            filtered_docs.append([sent for sent, keep in zip(doc_pool, doc_mask) if keep])
+            kept = [sent for sent, keep in zip(doc_pool, doc_mask) if keep]
+
+            if not kept and min_fallback_sentences > 0:
+                doc_scores = net_scores_cpu[c_offset : c_offset + c_count]
+                # Rank indices by score descending, keep top K that have at least > 0.10 score
+                ranked_local_idxs = sorted(
+                    range(c_count), key=lambda i: doc_scores[i], reverse=True
+                )
+                rescued_idxs = sorted([
+                    i for i in ranked_local_idxs[:min_fallback_sentences]
+                    if doc_scores[i] > 0.10
+                ])
+                kept = [doc_pool[i] for i in rescued_idxs]
+
+            filtered_docs.append(kept)
             c_offset += c_count
 
         return filtered_docs

{dstpr-0.2.2 → dstpr-0.2.4}/dstpr/segmenters/section_split.py

@@ -45,8 +45,7 @@ class SemanticSectionRouter:
 
     def route_sentences(
             self, 
-            sentences: list[str], 
-            windows: list[str],
+            sentences: list[str],
             doc_sentence_counts: list[int],
             min_confidence: float = 0.20,
             batch_size: int = 512
@@ -54,39 +53,46 @@ class SemanticSectionRouter:
         if not sentences:
             return [[] for _ in doc_sentence_counts]
 
-        window_embs = self._encode_sorted(windows, batch_size=batch_size) 
+        unique_sents = list(dict.fromkeys(sentences))
+        norm_unique = self._encode_sorted(unique_sents, batch_size=batch_size)
 
         with torch.inference_mode():
+            emb_map = {s: norm_unique[i] for i, s in enumerate(unique_sents)}
+            sent_embs = torch.stack([emb_map[s] for s in sentences])
+
+            # 2. Compute 3-sentence context windows in VRAM via 1D avg pooling (0ms model cost!)
+            padded = torch.nn.functional.pad(sent_embs.unsqueeze(0).transpose(1, 2), (1, 1), mode="replicate")
+            window_embs = torch.nn.functional.avg_pool1d(padded, kernel_size=3, stride=1).transpose(1, 2).squeeze(0)
+            window_embs = torch.nn.functional.normalize(window_embs, p=2, dim=1)
+
             sim_sections = torch.matmul(window_embs, self.anchor_embeddings.T)
             best_scores, best_indices = torch.max(sim_sections, dim=1)
 
             confident_mask = best_scores >= min_confidence
             fallback_tensor = torch.full_like(best_indices, self.default_idx)
-            routed_indices = torch.where(
-                confident_mask, best_indices, fallback_tensor
-            ).cpu().tolist()
+            routed_indices = torch.where(confident_mask, best_indices, fallback_tensor).cpu().tolist()
 
-        target_section_set = set(self.profile.target_sections)
-        doc_candidates = []
+        target_idx_set = {self.section_keys.index(k) for k in self.profile.target_sections if k in self.section_keys}
+        flat_candidates = []
+        keep_emb_indices = []
+        doc_cand_counts = []
         curr_offset = 0
 
         for count in doc_sentence_counts:
             if count == 0:
-                doc_candidates.append([])
+                doc_cand_counts.append(0)
                 continue
 
-            doc_sents = sentences[curr_offset : curr_offset + count]
-            doc_secs = [
-                self.section_keys[idx]
-                for idx in routed_indices[curr_offset : curr_offset + count]
+            doc_indices = [
+                curr_offset + i for i in range(count) 
+                if routed_indices[curr_offset + i] in target_idx_set
             ]
+            if len(doc_indices) < min(3, count):
+                doc_indices = list(range(curr_offset, curr_offset + count))
 
-            targets = [s for s, sec in zip(doc_sents, doc_secs) if sec in target_section_set]
-
-            if not targets:
-                targets = doc_sents
-
-            doc_candidates.append(targets)
+            doc_cand_counts.append(len(doc_indices))
+            keep_emb_indices.extend(doc_indices)
+            flat_candidates.extend([sentences[i] for i in doc_indices])
             curr_offset += count
 
-        return doc_candidates
+        return flat_candidates, sent_embs[keep_emb_indices], doc_cand_counts

{dstpr-0.2.2 → dstpr-0.2.4}/dstpr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dstpr
-Version: 0.2.2
+Version: 0.2.4
 Summary: A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate.
 Author-email: Stephen Meisenbacher <sjmeis@gtgd.com>
 Maintainer-email: Stephen Meisenbacher <sjmeis@gtgd.com>
@@ -76,7 +76,7 @@ This is the recommended approach for large-scale data pipelines.
 ```python
 from dstpr import ParallelPreprocessingPipeline, JOB_POSTING_PROFILE
 
-pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE, batch_size=64)
+pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE)
 
 raw_documents = [
     "DOC 1",
@@ -84,41 +84,13 @@ raw_documents = [
     "..."
 ]
 
-cleaned_documents = pipeline.process(raw_documents, num_workers=8, threshold=0.25)
-```
-
-### Advanced Usage
-
-If you want to integrate specific pipeline layers directly into an existing workflow, or tweak the internal parameters, you can import individual modules manually:
-
-```python
-from dstpr.cleaners import clean_and_split_chunks
-from dstpr.segmenters import SemanticSectionRouter
-from dstpr.rankers import HybridRanker
-from sentence_transformers import SentenceTransformer
-
-embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
-
-text = "Job posting text goes here!"
-
-# Clean and segment sentences
-sentences = clean_and_split_chunks(text)
-
-# Route by context
-router = SemanticSectionRouter(model=embedding_model)
-buckets = router.route_sentences(sentences)
-target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
-
-# Grade and sort text features
-ranker = HybridRanker()
-final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
+cleaned_documents = pipeline.process(raw_documents, threshold=0.25)
 ```
 
 ### Parameter Tuning
 
 To adjust the trade-off between strict filtering and execution speed, consider tweaking these variables in `ParallelPreprocessingPipeline`:
  - `threshold` (Default: `0.4`): Controls how aggressively sentences are discarded. Raising this value toward `0.6`, for example, ensures only stronger matching sentences are passed through. Lowering it toward `0.3`, on the other hand, acts as a wider net.
- - `batch_size` (Default: `256`): This depends on your hardware! Adjust for best performance.
 
 ## Creating Custom Domain Profiles (via the provided wizard)
 

{dstpr-0.2.2 → dstpr-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dstpr"
-version = "0.2.2"
+version = "0.2.4"
 description = "A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate."
 readme = {file = "README.md", content-type = "text/markdown"}
 license = {file = "LICENSE"}