dstpr 0.1.0__tar.gz

dstpr-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Stephen Meisenbacher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dstpr-0.1.0/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.4
+Name: dstpr
+Version: 0.1.0
+Summary: A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate.
+Author-email: Stephen Meisenbacher <sjmeis@gtgd.com>
+Maintainer-email: Stephen Meisenbacher <sjmeis@gtgd.com>
+License: MIT License
+        
+        Copyright (c) 2026 Stephen Meisenbacher
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: numpy>=1.22.0
+Requires-Dist: datasets>=2.12.0
+Requires-Dist: spacy<4.0.0,>=3.8.0
+Dynamic: license-file
+
+# DSTPR: Domain-specific Text Pre-processing and Ranking
+
+[![PyPI version](https://img.shields.io/pypi/v/dstpr.svg)](https://pypi.org/project/dstpr/)
+[![License](https://img.shields.io/github/license/sjmeis/DSTPR.svg)](https://github.com/sjmeis/DSTPR/blob/main/LICENSE)
+
+A high-throughput text preprocessing pipeline designed to filter, segment, and rank core sentences from noisy, plaintext documents. 
+
+This package strips away boilerplate, disclaimers, and application instructions *before* moving this data onto to heavier processing pipelines.
+
+---
+
+## Pipeline Architecture
+
+Instead of running heavy transformer models over every sentence in a document, `DSTPR` pipes documents through progressively stricter layers:
+
+1. **Heuristic Cleansing & Tokenization:** Uses `PySBD` (Python Sentence Boundary Disambiguation) paired with regular expressions to fix punctuation caused by flattening, then filters out sentences containing stop phrases.
+2. **Semantic Section Routing:** Utilizes a lightweight encoder (Default: `all-MiniLM-L6-v2`) to dynamically find structural transitions in input texts.
+3. **Hybrid Ranking:** Scores remaining sentences using a combination of **Semantics** (cosine similarity to anchors) and **Lexical Syntax** (regex-supported detection).
+4. **Parallel Execution Engine:** Wraps the entire pipeline inside efficient datasets to allow for batch processing at scale, with CPU or GPU.
+
+---
+
+## Installation
+
+To install the package, simply run
+
+```bash
+pip install dstpr
+```
+
+## Usage
+
+### High-Throughput Batch Processing
+
+This is the recommended approach for large-scale data pipelines. 
+
+```python
+from dstpr import ParallelPreprocessingPipeline, JOB_POSTING_PROFILE
+
+pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE, batch_size=64)
+
+raw_documents = [
+    "DOC 1",
+    "DOC 2",
+    "..."
+]
+
+cleaned_documents = pipeline.process(raw_documents, num_workers=8, threshold=0.25)
+```
+
+### Advanced Usage
+
+If you want to integrate specific pipeline layers directly into an existing workflow, or tweak the internal parameters, you can import individual modules manually:
+
+```python
+from dstpr.cleaners import clean_and_split_chunks
+from dstpr.segmenters import SemanticSectionRouter
+from dstpr.rankers import HybridTaskRanker
+from sentence_transformers import Transformer, SentenceTransformer
+
+embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
+
+text = "Job posting text goes here!"
+
+# Clean and segment sentences
+sentences = clean_and_split_chunks(text)
+
+# Route by context
+router = SemanticSectionRouter(model=embedding_model)
+buckets = router.route_sentences(sentences)
+target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
+
+# Grade and sort text features
+ranker = HybridRanker(model=shared_model)
+final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
+```
+
+### Parameter Tuning
+
+To adjust the trade-off between strict filtering and execution speed, consider tweaking these variables in `ParallelPreprocessingPipeline`:
+ - `threshold` (Default: `0.4`): Controls how aggressively sentences are discarded. Raising this value toward `0.6`, for example, ensures only stronger matching sentences are passed through. Lowering it toward `0.3`, on the other hand, acts as a wider net.
+ - `batch_size` (Default: `256`): This depends on your hardware! Adjust for best performance.
+
+## Creating Custom Domain Profiles (via the provided wizard)
+
+A `DomainProfile` is required to use `DSTPR` (see the usage example). Out of the box, `dstpr` ships with pre-configured configurations for job postings (`JOB_POSTINGS_PROFILE`). However, you can easily generate an domain-specific pipeline for **any specific domain** using our built-in interactive configuration wizard.
+
+To spin up the profile creation walkthrough, simply open your terminal and run:
+
+```bash
+task-profile-wizard
+```
+
+All profiles generated via the terminal wizard are automatically validated and written out as JSON to a local cache: `~/.config/dstpr/profiles/`

dstpr-0.1.0/README.md

@@ -0,0 +1,94 @@
+# DSTPR: Domain-specific Text Pre-processing and Ranking
+
+[![PyPI version](https://img.shields.io/pypi/v/dstpr.svg)](https://pypi.org/project/dstpr/)
+[![License](https://img.shields.io/github/license/sjmeis/DSTPR.svg)](https://github.com/sjmeis/DSTPR/blob/main/LICENSE)
+
+A high-throughput text preprocessing pipeline designed to filter, segment, and rank core sentences from noisy, plaintext documents. 
+
+This package strips away boilerplate, disclaimers, and application instructions *before* moving this data onto to heavier processing pipelines.
+
+---
+
+## Pipeline Architecture
+
+Instead of running heavy transformer models over every sentence in a document, `DSTPR` pipes documents through progressively stricter layers:
+
+1. **Heuristic Cleansing & Tokenization:** Uses `PySBD` (Python Sentence Boundary Disambiguation) paired with regular expressions to fix punctuation caused by flattening, then filters out sentences containing stop phrases.
+2. **Semantic Section Routing:** Utilizes a lightweight encoder (Default: `all-MiniLM-L6-v2`) to dynamically find structural transitions in input texts.
+3. **Hybrid Ranking:** Scores remaining sentences using a combination of **Semantics** (cosine similarity to anchors) and **Lexical Syntax** (regex-supported detection).
+4. **Parallel Execution Engine:** Wraps the entire pipeline inside efficient datasets to allow for batch processing at scale, with CPU or GPU.
+
+---
+
+## Installation
+
+To install the package, simply run
+
+```bash
+pip install dstpr
+```
+
+## Usage
+
+### High-Throughput Batch Processing
+
+This is the recommended approach for large-scale data pipelines. 
+
+```python
+from dstpr import ParallelPreprocessingPipeline, JOB_POSTING_PROFILE
+
+pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE, batch_size=64)
+
+raw_documents = [
+    "DOC 1",
+    "DOC 2",
+    "..."
+]
+
+cleaned_documents = pipeline.process(raw_documents, num_workers=8, threshold=0.25)
+```
+
+### Advanced Usage
+
+If you want to integrate specific pipeline layers directly into an existing workflow, or tweak the internal parameters, you can import individual modules manually:
+
+```python
+from dstpr.cleaners import clean_and_split_chunks
+from dstpr.segmenters import SemanticSectionRouter
+from dstpr.rankers import HybridTaskRanker
+from sentence_transformers import Transformer, SentenceTransformer
+
+embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
+
+text = "Job posting text goes here!"
+
+# Clean and segment sentences
+sentences = clean_and_split_chunks(text)
+
+# Route by context
+router = SemanticSectionRouter(model=embedding_model)
+buckets = router.route_sentences(sentences)
+target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
+
+# Grade and sort text features
+ranker = HybridRanker(model=shared_model)
+final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
+```
+
+### Parameter Tuning
+
+To adjust the trade-off between strict filtering and execution speed, consider tweaking these variables in `ParallelPreprocessingPipeline`:
+ - `threshold` (Default: `0.4`): Controls how aggressively sentences are discarded. Raising this value toward `0.6`, for example, ensures only stronger matching sentences are passed through. Lowering it toward `0.3`, on the other hand, acts as a wider net.
+ - `batch_size` (Default: `256`): This depends on your hardware! Adjust for best performance.
+
+## Creating Custom Domain Profiles (via the provided wizard)
+
+A `DomainProfile` is required to use `DSTPR` (see the usage example). Out of the box, `dstpr` ships with pre-configured configurations for job postings (`JOB_POSTINGS_PROFILE`). However, you can easily generate an domain-specific pipeline for **any specific domain** using our built-in interactive configuration wizard.
+
+To spin up the profile creation walkthrough, simply open your terminal and run:
+
+```bash
+task-profile-wizard
+```
+
+All profiles generated via the terminal wizard are automatically validated and written out as JSON to a local cache: `~/.config/dstpr/profiles/`

dstpr-0.1.0/dstpr/__init__.py

@@ -0,0 +1,13 @@
+import sys
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
+    sys.modules['lzma'] = lzma
+
+from dstpr.pipeline import ParallelPreprocessingPipeline
+from dstpr.profiles import DomainProfile, JOB_POSTING_PROFILE
+from dstpr.utils.cli import run_profile_wizard, DEFAULT_STORAGE_DIR
+from dstpr.utils.setup import setup_resources
+
+__all__ = ["ParallelPreprocessingPipeline", "DomainProfile", "JOB_POSTING_PROFILE", "run_profile_wizard", "DEFAULT_STORAGE_DIR", "setup_resources"]

dstpr-0.1.0/dstpr/pipeline.py

@@ -0,0 +1,86 @@
+import os
+import torch
+from datasets import Dataset
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from dstpr.cleaners.pre_process import clean_and_split_chunks
+from dstpr.cleaners.post_process import sanitize_outputs
+from dstpr.segmenters.section_split import SemanticSectionRouter
+from dstpr.rankers.hybrid import HybridRanker
+from dstpr.profiles import DomainProfile
+
+class ParallelPreprocessingPipeline:
+    def __init__(self, profile: DomainProfile, batch_size: int = 256, embedding_model: str = "all-MiniLM-L6-v2"):
+        self.profile = profile
+        self.batch_size = batch_size
+        self.embedding_model = embedding_model
+        
+        self.router = None
+        self.ranker = None
+        self._is_initialized = False
+
+    def _lazy_init_local_resources(self):
+
+        if not self._is_initialized:
+            torch.set_num_threads(1)
+            
+            bi_encoder = SentenceTransformer(self.embedding_model)
+            cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+            
+            self.router = SemanticSectionRouter(profile=self.profile, model=bi_encoder)
+            self.ranker = HybridRanker(
+                profile=self.profile, 
+                model=bi_encoder, 
+                cross_encoder_model=cross_encoder
+            )
+            self._is_initialized = True
+
+    def _process_batch(self, batch: dict, threshold: float) -> dict:
+        self._lazy_init_local_resources()
+        
+        cleaned_documents = []
+        for raw_text in batch["raw_text"]:
+            if not raw_text or not isinstance(raw_text, str):
+                cleaned_documents.append("")
+                continue
+                
+            sentences = clean_and_split_chunks(raw_text)
+            buckets = self.router.route_sentences(sentences)
+            print(buckets)
+            
+            target_sentences = []
+            for section_key in self.router.profile.target_sections:
+                target_sentences.extend(buckets.get(section_key, []))
+                
+            if not target_sentences:
+                for sec_content in buckets.values():
+                    target_sentences.extend(sec_content)
+            
+            filtered_tasks = self.ranker.rank_and_filter(target_sentences, final_threshold=threshold)
+            pp_outputs = sanitize_outputs(filtered_tasks)
+            cleaned_documents.append(pp_outputs)
+            
+        return {"cleaned_text": cleaned_documents}
+
+    def process(self, texts: list[str], num_workers: int = None, threshold: float = 0.50) -> list[str]:
+        if num_workers is None:
+            num_workers = max(1, os.cpu_count() - 1)
+
+        os.environ["TOKENIZERS_PARALLELISM"] = "false"
+            
+        print(f"Creating Dataset for {len(texts)} documents...")
+        dataset = Dataset.from_dict({"raw_text": texts})
+        
+        print(f"Running pipeline using {num_workers} parallel worker(s)...")
+        
+        processed_dataset = dataset.map(
+            self._process_batch,
+            batched=True,
+            batch_size=self.batch_size,
+            num_proc=num_workers,
+            fn_kwargs={
+                "threshold": threshold
+            },
+            desc="Filtering Documents"
+        )
+        
+        return processed_dataset["cleaned_text"]

dstpr-0.1.0/dstpr/profiles.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass, field, asdict
+import json
+from pathlib import Path
+
+@dataclass
+class DomainProfile:
+    name: str
+    global_stop_phrases: set[str] = field(default_factory=set)
+    anchors: dict[str, str] = field(default_factory=dict)
+    default_section: str = "CORE"
+    target_sections: list[str] = field(default_factory=list)
+    target_semantic_anchor: str = "core actions and focus areas"
+    action_keywords_regex: str = r"\b(do|execute|perform)\b"
+    syntax_start_pattern: str = r"^.*" 
+
+    def validate(self):
+        if not self.name.isalnum():
+            raise ValueError("Profile name must be alphanumeric (no spaces or special characters).")
+        if not self.section_anchors:
+            raise ValueError("DomainProfile must define at least one section anchor.")
+        if self.default_section not in self.anchors:
+            raise ValueError(f"Default section '{self.default_section}' must be one of the defined section anchors.")
+        for section in self.target_sections:
+            if section not in self.anchors:
+                raise ValueError(f"Target section '{section}' must match a key in section_anchors.")
+        
+    def save_to_json(self, target_dir: Path | str) -> Path:
+        self.validate()
+        dir_path = Path(target_dir)
+        dir_path.mkdir(parents=True, exist_ok=True)
+        
+        file_path = dir_path / f"{self.name}.json"
+        
+        data = asdict(self)
+        data['global_stop_phrases'] = list(data['global_stop_phrases'])
+        
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, indent=4)
+            
+        return file_path
+
+    @classmethod
+    def load_from_json(cls, file_path: Path | str) -> 'DomainProfile':
+        with open(file_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            
+        data['global_stop_phrases'] = set(data['global_stop_phrases'])
+        return cls(**data)
+        
+
+JOB_POSTING_PROFILE = DomainProfile(
+    name="job postings",
+    global_stop_phrases={"apply online", "click here", "equal opportunity employer", "visa sponsorship"},
+    anchors={
+        'CORE': "core job duties, daily responsibilities, tasks, project assignments, what you will build and execute on the job",
+        'REQUIREMENTS': "candidate qualifications, required experience, required skills, degrees, interview assessment process, hiring criteria",
+        'BOILERPLATE': "company overview, history, mission statement, office locations, perks and benefits, scam alerts, recruitment partners, privacy policies"
+    },
+    default_section="CORE",
+    target_sections=['CORE', 'REQUIREMENTS'],
+    target_semantic_anchor="execute responsibilities, perform duties, manage projects, build and design systems",
+    action_keywords_regex=r"\b(write|design|build|develop|manage|lead|collaborate|implement|maintain)\b",
+    syntax_start_pattern=r"^(?:you\s+will\s+|responsibilities\s+include\s+)?",
+)

dstpr-0.1.0/dstpr.egg-info/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.4
+Name: dstpr
+Version: 0.1.0
+Summary: A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate.
+Author-email: Stephen Meisenbacher <sjmeis@gtgd.com>
+Maintainer-email: Stephen Meisenbacher <sjmeis@gtgd.com>
+License: MIT License
+        
+        Copyright (c) 2026 Stephen Meisenbacher
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: numpy>=1.22.0
+Requires-Dist: datasets>=2.12.0
+Requires-Dist: spacy<4.0.0,>=3.8.0
+Dynamic: license-file
+
+# DSTPR: Domain-specific Text Pre-processing and Ranking
+
+[![PyPI version](https://img.shields.io/pypi/v/dstpr.svg)](https://pypi.org/project/dstpr/)
+[![License](https://img.shields.io/github/license/sjmeis/DSTPR.svg)](https://github.com/sjmeis/DSTPR/blob/main/LICENSE)
+
+A high-throughput text preprocessing pipeline designed to filter, segment, and rank core sentences from noisy, plaintext documents. 
+
+This package strips away boilerplate, disclaimers, and application instructions *before* moving this data onto to heavier processing pipelines.
+
+---
+
+## Pipeline Architecture
+
+Instead of running heavy transformer models over every sentence in a document, `DSTPR` pipes documents through progressively stricter layers:
+
+1. **Heuristic Cleansing & Tokenization:** Uses `PySBD` (Python Sentence Boundary Disambiguation) paired with regular expressions to fix punctuation caused by flattening, then filters out sentences containing stop phrases.
+2. **Semantic Section Routing:** Utilizes a lightweight encoder (Default: `all-MiniLM-L6-v2`) to dynamically find structural transitions in input texts.
+3. **Hybrid Ranking:** Scores remaining sentences using a combination of **Semantics** (cosine similarity to anchors) and **Lexical Syntax** (regex-supported detection).
+4. **Parallel Execution Engine:** Wraps the entire pipeline inside efficient datasets to allow for batch processing at scale, with CPU or GPU.
+
+---
+
+## Installation
+
+To install the package, simply run
+
+```bash
+pip install dstpr
+```
+
+## Usage
+
+### High-Throughput Batch Processing
+
+This is the recommended approach for large-scale data pipelines. 
+
+```python
+from dstpr import ParallelPreprocessingPipeline, JOB_POSTING_PROFILE
+
+pipeline = ParallelPreprocessingPipeline(profile=JOB_POSTING_PROFILE, batch_size=64)
+
+raw_documents = [
+    "DOC 1",
+    "DOC 2",
+    "..."
+]
+
+cleaned_documents = pipeline.process(raw_documents, num_workers=8, threshold=0.25)
+```
+
+### Advanced Usage
+
+If you want to integrate specific pipeline layers directly into an existing workflow, or tweak the internal parameters, you can import individual modules manually:
+
+```python
+from dstpr.cleaners import clean_and_split_chunks
+from dstpr.segmenters import SemanticSectionRouter
+from dstpr.rankers import HybridTaskRanker
+from sentence_transformers import Transformer, SentenceTransformer
+
+embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
+
+text = "Job posting text goes here!"
+
+# Clean and segment sentences
+sentences = clean_and_split_chunks(text)
+
+# Route by context
+router = SemanticSectionRouter(model=embedding_model)
+buckets = router.route_sentences(sentences)
+target_sentences = buckets['CORE'] + buckets['REQUIREMENTS']
+
+# Grade and sort text features
+ranker = HybridRanker(model=shared_model)
+final = ranker.rank_and_filter(target_sentences, bi_cutoff_pct=0.25, final_threshold=0.40)
+```
+
+### Parameter Tuning
+
+To adjust the trade-off between strict filtering and execution speed, consider tweaking these variables in `ParallelPreprocessingPipeline`:
+ - `threshold` (Default: `0.4`): Controls how aggressively sentences are discarded. Raising this value toward `0.6`, for example, ensures only stronger matching sentences are passed through. Lowering it toward `0.3`, on the other hand, acts as a wider net.
+ - `batch_size` (Default: `256`): This depends on your hardware! Adjust for best performance.
+
+## Creating Custom Domain Profiles (via the provided wizard)
+
+A `DomainProfile` is required to use `DSTPR` (see the usage example). Out of the box, `dstpr` ships with pre-configured configurations for job postings (`JOB_POSTINGS_PROFILE`). However, you can easily generate an domain-specific pipeline for **any specific domain** using our built-in interactive configuration wizard.
+
+To spin up the profile creation walkthrough, simply open your terminal and run:
+
+```bash
+task-profile-wizard
+```
+
+All profiles generated via the terminal wizard are automatically validated and written out as JSON to a local cache: `~/.config/dstpr/profiles/`

dstpr-0.1.0/dstpr.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+LICENSE
+README.md
+pyproject.toml
+dstpr/__init__.py
+dstpr/pipeline.py
+dstpr/profiles.py
+dstpr.egg-info/PKG-INFO
+dstpr.egg-info/SOURCES.txt
+dstpr.egg-info/dependency_links.txt
+dstpr.egg-info/entry_points.txt
+dstpr.egg-info/requires.txt
+dstpr.egg-info/top_level.txt

dstpr-0.1.0/dstpr.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dstpr-0.1.0/dstpr.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+profile-wizard = dstpr.utils.cli:run_profile_wizard

dstpr-0.1.0/dstpr.egg-info/requires.txt

@@ -0,0 +1,5 @@
+sentence-transformers>=3.0.0
+scikit-learn>=1.0.0
+numpy>=1.22.0
+datasets>=2.12.0
+spacy<4.0.0,>=3.8.0

dstpr-0.1.0/dstpr.egg-info/top_level.txt

@@ -0,0 +1 @@
+dstpr

dstpr-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=61.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dstpr"
+version = "0.1.0"
+description = "A high-throughput, domain-specific text preprocessing cascading pipeline filter to rank core sentences in texts and filter away boilerplate."
+readme = {file = "README.md", content-type = "text/markdown"}
+license = {file = "LICENSE"}
+authors = [
+  {name = "Stephen Meisenbacher", email = "sjmeis@gtgd.com"},
+]
+maintainers = [
+  {name = "Stephen Meisenbacher", email = "sjmeis@gtgd.com"}
+]
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent"
+]
+dependencies = [
+    "sentence-transformers>=3.0.0",
+    "scikit-learn>=1.0.0",
+    "numpy>=1.22.0",
+    "datasets>=2.12.0",
+    "spacy>=3.8.0,<4.0.0"
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["dstpr"]
+
+[project.scripts]
+profile-wizard = "dstpr.utils.cli:run_profile_wizard"

dstpr-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+