jaat2 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
jaat2/__init__.py ADDED
@@ -0,0 +1,87 @@
1
+ # Copyright 2026 Stephen Meisenbacher
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import os
16
+ import sys
17
+ import time
18
+ import threading
19
+ import re
20
+ from .config import JAAT_ART, SHOW_ART, VERSION, COPYRIGHT, DESCRIPTION, CYAN, GREEN, RESET, BOLD, DIM
21
+
22
+ def get_visible_len(text: str) -> int:
23
+ ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
24
+ return len(ansi_escape.sub('', text))
25
+
26
+ class Loader:
27
+ def __init__(self, desc="Loading JAAT..."):
28
+ self.desc = desc
29
+ self.done = False
30
+ self.thread = None
31
+ self.width = 64
32
+
33
+ def _animate(self):
34
+ chars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
35
+ idx = 0
36
+ while not self.done:
37
+ sys.stdout.write(f"\r {CYAN}{chars[idx % len(chars)]}{RESET} {self.desc}...")
38
+ sys.stdout.flush()
39
+ idx += 1
40
+ time.sleep(0.08)
41
+
42
+ def __enter__(self):
43
+ if SHOW_ART:
44
+ top_bottom = "+" + "-" * (self.width - 2) + "+"
45
+ side = f"{DIM}|{RESET}"
46
+
47
+ print(f"\n{DIM}{top_bottom}{RESET}")
48
+
49
+ for line in JAAT_ART.strip("\n").split("\n"):
50
+ content = line.ljust(self.width - 4)
51
+ print(f"{side} {CYAN}{content}{RESET} {side}")
52
+
53
+ print(f"{side}" + " " * (self.width - 2) + f"{side}")
54
+
55
+ welcome_msg = f"Welcome to JAAT v{VERSION}".center(self.width - 4)
56
+ copyright_msg = COPYRIGHT.center(self.width - 4)
57
+ desc_msg = DESCRIPTION.center(self.width - 4)
58
+
59
+ print(f"{side} {BOLD}{welcome_msg}{RESET} {side}")
60
+ print(f"{side} {DIM}{copyright_msg}{RESET} {side}")
61
+ print(f"{side} {DIM}{desc_msg}{RESET} {side}")
62
+ print(f"{DIM}{top_bottom}{RESET}\n")
63
+
64
+ self.thread = threading.Thread(target=self._animate, daemon=True)
65
+ self.thread.start()
66
+ return self
67
+
68
+ def __exit__(self, exc_type, exc_value, traceback):
69
+ self.done = True
70
+ if self.thread:
71
+ self.thread.join()
72
+ sys.stdout.write(f"\r {GREEN}✅ {self.desc}... Ready.{RESET}\n\n")
73
+ sys.stdout.flush()
74
+
75
+ with Loader():
76
+ os.environ["USE_TORCH"] = "1"
77
+ os.environ["USE_TF"] = "0"
78
+ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
79
+ os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
80
+
81
+ from .match import Match
82
+ from .utils import setup, chunker, validate_inputs, diagnostic, clear_cache, shutdown, toggle_progress, toggle_threads
83
+
84
+ __all__ = [
85
+ "setup", "chunker", "validate_inputs", "diagnostic", "clear_cache", "shutdown", "toggle_progress", "toggle_threads",
86
+ "Match",
87
+ ]
jaat2/base.py ADDED
@@ -0,0 +1,250 @@
1
+ # Copyright 2026 Stephen Meisenbacher
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import os
16
+ import psutil
17
+ from typing import List, Tuple, Any, Optional
18
+ import torch
19
+ import re
20
+ from torch.utils.data import Dataset
21
+ from transformers.utils import is_flash_attn_2_available
22
+ from wtpsplit import SaT
23
+
24
+ from .config import logger, MODEL_CACHE
25
+
26
+ BULLET_PATTERN = re.compile(r"(^|\n)[\s•·\-\*]+\s*")
27
+ DELIM_PATTERN = re.compile(r"[;]\s*")
28
+ CLEAN_PATTERN = re.compile(r"(;|\s\+\s|\s\*\s|\s\-\s|\s•\s|\s·\s|--|\*\*)")
29
+ MULTI_PUNCT = re.compile(r"\.{2,}")
30
+ MULTI_SPACE = re.compile(r"[ \t]+")
31
+ TRAILING_PUNCT = re.compile(r"^[ \t\n\r.\-•*]+|[ \t\n\r.\-•*]+$")
32
+ CLAUSE_DELIMS = re.compile(r"(\s*;\s*|\s*—\s*|\s*--\s*)")
33
+
34
+ def get_device_settings() -> Tuple[str, int]:
35
+ """
36
+ Detects hardware capabilities and derives a sensible batch size.
37
+ """
38
+
39
+ # CUDA
40
+ if torch.cuda.is_available():
41
+ device = "cuda"
42
+
43
+ try:
44
+ device_props = torch.cuda.get_device_properties(0)
45
+ total_vram_gb = device_props.total_memory / (1024 ** 3)
46
+ except Exception:
47
+ total_vram_gb = 8.0
48
+
49
+ if total_vram_gb >= 40.0:
50
+ batch_size = 2048
51
+ elif total_vram_gb >= 20.0:
52
+ batch_size = 1024
53
+ elif total_vram_gb >= 14.0:
54
+ batch_size = 512
55
+ elif total_vram_gb >= 8.0:
56
+ batch_size = 256
57
+ else:
58
+ batch_size = 64
59
+
60
+ return device, batch_size
61
+
62
+ # Apple Silicon (MPS)
63
+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
64
+ total_ram_gb = psutil.virtual_memory().total / (1024 ** 3)
65
+ device = "mps"
66
+
67
+ if total_ram_gb >= 32.0:
68
+ batch_size = 256
69
+ elif total_ram_gb >= 16.0:
70
+ batch_size = 128
71
+ else:
72
+ batch_size = 64
73
+
74
+ return device, batch_size
75
+
76
+ # CPU Fallback
77
+ device = "cpu"
78
+ cpu_count = os.cpu_count() or 4
79
+ total_ram_gb = psutil.virtual_memory().total / (1024 ** 3)
80
+
81
+ if cpu_count >= 16 and total_ram_gb >= 32.0:
82
+ batch_size = 64
83
+ else:
84
+ batch_size = 32
85
+
86
+ return device, batch_size
87
+
88
+ def get_optimal_attn_implementation() -> str:
89
+ """
90
+ Selects the fastest attention backend available for the current hardware.
91
+ """
92
+ if not torch.cuda.is_available():
93
+ return "sdpa" if hasattr(torch.nn.functional, "scaled_dot_product_attention") else "eager"
94
+
95
+ major, _ = torch.cuda.get_device_capability(0)
96
+ has_compatible_arch = major >= 8
97
+
98
+ if has_compatible_arch and is_flash_attn_2_available():
99
+ return "flash_attention_2"
100
+
101
+ if hasattr(torch.nn.functional, "scaled_dot_product_attention"):
102
+ return "sdpa"
103
+
104
+ return "eager"
105
+
106
+ class ListDataset(Dataset):
107
+ def __init__(self, original_list: List[Any]) -> None:
108
+ self.original_list = original_list
109
+
110
+ def __len__(self) -> int:
111
+ return len(self.original_list)
112
+
113
+ def __getitem__(self, i: int) -> Any:
114
+ return self.original_list[i]
115
+
116
+ def clean_text(text: str) -> str:
117
+ """Pre-normalizes delimiters, bullets, and line breaks into standard clause boundaries."""
118
+ if not text:
119
+ return ""
120
+ lines = [line.strip() for line in text.split("\n") if line.strip()]
121
+ normalized_lines = [
122
+ line if line.endswith((".", "!", "?")) else f"{line}."
123
+ for line in (l.strip() for l in text.splitlines())
124
+ if line
125
+ ]
126
+ text = " ".join(normalized_lines)
127
+ text = BULLET_PATTERN.sub(" ", text)
128
+
129
+ text = CLAUSE_DELIMS.sub(". ", text)
130
+ text = MULTI_PUNCT.sub(".", text)
131
+ text = MULTI_SPACE.sub(" ", text).strip()
132
+ return text
133
+
134
+ def filter_valid_sentence(sent: str, min_words: int = 2, max_words: int = 64) -> Optional[str]:
135
+ s = TRAILING_PUNCT.sub("", sent)
136
+ words = s.split()
137
+ word_count = len(words)
138
+ if min_words <= word_count <= max_words:
139
+ return s
140
+ return None
141
+
142
+ class BatchedSaTSplitter:
143
+ def __init__(
144
+ self,
145
+ model_name: str = "sat-3l-sm",
146
+ device: Optional[str] = None
147
+ ) -> None:
148
+ self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
149
+ self.model_name = model_name
150
+ self.sat = None
151
+
152
+ cache_key = f"sat_{model_name}"
153
+ if cache_key not in MODEL_CACHE:
154
+ logger.info(f"Loading SaT segmenter '{model_name}' on {self.device}...")
155
+ sat_instance = SaT(model_name)
156
+ if self.device == "cuda":
157
+ sat_instance.half().to("cuda")
158
+ MODEL_CACHE[cache_key] = sat_instance
159
+ self.sat = MODEL_CACHE[cache_key]
160
+
161
+ def split_two_pass(
162
+ self,
163
+ docs: List[str],
164
+ batch_size: int = 1024,
165
+ pass1_threshold: float = 0.50,
166
+ pass2_threshold: float = 0.20,
167
+ min_words: int = 2,
168
+ max_words: int = 48,
169
+ pass2_char_length: int = 80
170
+ ) -> List[List[str]]:
171
+ if not docs:
172
+ return []
173
+
174
+ cleaned_docs = [clean_text(d) for d in docs]
175
+
176
+ doc_sentences_raw = self.sat.split(
177
+ cleaned_docs,
178
+ batch_size=batch_size,
179
+ threshold=pass1_threshold,
180
+ outer_batch_size=batch_size * 2,
181
+ verbose=False
182
+ )
183
+
184
+ doc_sentence_groups: List[List[str]] = []
185
+ pass2_candidates: List[str] = []
186
+ pass2_mapping: List[Tuple[int, int]] = []
187
+
188
+ for d_idx, raw_sents in enumerate(doc_sentences_raw):
189
+ filtered_doc_sents = []
190
+ for s in raw_sents:
191
+ clean_s = s.strip()
192
+ if clean_s:
193
+ filtered_doc_sents.append(clean_s)
194
+
195
+ doc_sentence_groups.append(filtered_doc_sents)
196
+
197
+ for s_idx, sent in enumerate(filtered_doc_sents):
198
+ if len(sent) >= pass2_char_length or len(sent.split()) > 15: # > (max_words // 2):
199
+ pass2_candidates.append(sent)
200
+ pass2_mapping.append((d_idx, s_idx))
201
+
202
+ replacement_map: dict[Tuple[int, int], List[str]] = {}
203
+ if pass2_candidates:
204
+ pass2_results = self.sat.split(
205
+ pass2_candidates,
206
+ batch_size=batch_size,
207
+ threshold=pass2_threshold,
208
+ outer_batch_size=batch_size * 2,
209
+ verbose=False
210
+ )
211
+
212
+ for (d_idx, s_idx), refined_sents in zip(pass2_mapping, pass2_results):
213
+ valid_splits = [s.strip(" ,;.-•*") for s in refined_sents if s.strip()]
214
+ cleaned_splits = []
215
+ for split in valid_splits:
216
+ split_cleaned = re.sub(r"^(and|or|whether that's|whether|such as|including)\s+", "", split, flags=re.IGNORECASE)
217
+ if len(split_cleaned.split()) >= min_words:
218
+ cleaned_splits.append(split_cleaned)
219
+
220
+ if len(cleaned_splits) > 1:
221
+ replacement_map[(d_idx, s_idx)] = cleaned_splits
222
+
223
+ final_docs: List[List[str]] = []
224
+ for d_idx, sents in enumerate(doc_sentence_groups):
225
+ doc_refined = []
226
+ for s_idx, sent in enumerate(sents):
227
+ key = (d_idx, s_idx)
228
+ candidates = replacement_map.get(key, [sent])
229
+
230
+ for candidate in candidates:
231
+ valid_item = filter_valid_sentence(candidate, min_words, max_words)
232
+ if valid_item is not None:
233
+ doc_refined.append(valid_item)
234
+
235
+ final_docs.append(doc_refined)
236
+
237
+ return final_docs
238
+
239
+ _GLOBAL_SPLITTER: Optional[BatchedSaTSplitter] = None
240
+
241
+ def get_sentence_splitter(device: Optional[str] = None) -> BatchedSaTSplitter:
242
+ global _GLOBAL_SPLITTER
243
+ if _GLOBAL_SPLITTER is None:
244
+ _GLOBAL_SPLITTER = BatchedSaTSplitter(device=device)
245
+ return _GLOBAL_SPLITTER
246
+
247
+ def sent_tokenize(text: str) -> List[str]:
248
+ splitter = get_sentence_splitter()
249
+ results = splitter.split_two_pass([text])
250
+ return results[0] if results else []
jaat2/config.py ADDED
@@ -0,0 +1,49 @@
1
+ import logging
2
+ import os
3
+
4
+ logger = logging.getLogger("JAATv2")
5
+
6
+ if not logger.handlers:
7
+ handler = logging.StreamHandler()
8
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
9
+ handler.setFormatter(formatter)
10
+ logger.addHandler(handler)
11
+ logger.setLevel(logging.INFO)
12
+
13
+ GLOBAL_SETTINGS = {
14
+ "show_progress": True,
15
+ "single_threaded": False
16
+ }
17
+
18
+ MODEL_CACHE = {}
19
+
20
+ VERSION = "2.0.0beta"
21
+ COPYRIGHT = "© 2026 Stephen Meisenbacher. All rights reserved."
22
+ DESCRIPTION = "An NLP-powered toolkit for job ad text analysis."
23
+
24
+ CYAN = "\033[96m"
25
+ GREEN = "\033[92m"
26
+ RESET = "\033[0m"
27
+ BOLD = "\033[1m"
28
+ DIM = "\033[2m"
29
+
30
+ # Note: 3D Diagonal
31
+ JAAT_ART = r"""
32
+ ,----,
33
+ ,---._ ,/ .`|
34
+ .-- -.' \ ,---, ,---, ,` .' : ,----,
35
+ | | : ' .' \ ' .' \ ; ; / .' .' \
36
+ : ; | / ; '. / ; '. .'___,/ ,' ,----,' |
37
+ : |: : \ : : \ | : | .---. | : . ;
38
+ | : :: | /\ \ : | /\ \ ; |.'; ; /. ./| ; |.' /
39
+ : | : ' ;. :| : ' ;. :`----' | | .-' . ' | `----'/ ;
40
+ | ; || | ;/ \ \ | ;/ \ \ ' : ;/___/ \: | / ; /
41
+ ___ l ' : | \ \ ,' : | \ \ ,' | | '. \ ' . ; / /-,
42
+ / /\ J :| | ' '--' | | ' '--' ' : | \ \ ' / / /.`|
43
+ / ../ `..- ,| : : | : : ; |.' \ \ ./__; :
44
+ \ \ ; | | ,' | | ,' '---' \ \ | : .'
45
+ \ \ ,' `--'' `--'' '---"; | .'
46
+ "---....--' `---'
47
+
48
+ """
49
+ SHOW_ART = os.isatty(1)