jaat2 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jaat2/__init__.py +87 -0
- jaat2/base.py +250 -0
- jaat2/config.py +49 -0
- jaat2/match.py +667 -0
- jaat2/taxonomy.py +291 -0
- jaat2/utils.py +146 -0
- jaat2-2.0.0.dist-info/METADATA +140 -0
- jaat2-2.0.0.dist-info/RECORD +12 -0
- jaat2-2.0.0.dist-info/WHEEL +5 -0
- jaat2-2.0.0.dist-info/licenses/LICENSE +202 -0
- jaat2-2.0.0.dist-info/licenses/NOTICE +9 -0
- jaat2-2.0.0.dist-info/top_level.txt +1 -0
jaat2/__init__.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# Copyright 2026 Stephen Meisenbacher
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
import threading
|
|
19
|
+
import re
|
|
20
|
+
from .config import JAAT_ART, SHOW_ART, VERSION, COPYRIGHT, DESCRIPTION, CYAN, GREEN, RESET, BOLD, DIM
|
|
21
|
+
|
|
22
|
+
def get_visible_len(text: str) -> int:
|
|
23
|
+
ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
|
|
24
|
+
return len(ansi_escape.sub('', text))
|
|
25
|
+
|
|
26
|
+
class Loader:
|
|
27
|
+
def __init__(self, desc="Loading JAAT..."):
|
|
28
|
+
self.desc = desc
|
|
29
|
+
self.done = False
|
|
30
|
+
self.thread = None
|
|
31
|
+
self.width = 64
|
|
32
|
+
|
|
33
|
+
def _animate(self):
|
|
34
|
+
chars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
|
|
35
|
+
idx = 0
|
|
36
|
+
while not self.done:
|
|
37
|
+
sys.stdout.write(f"\r {CYAN}{chars[idx % len(chars)]}{RESET} {self.desc}...")
|
|
38
|
+
sys.stdout.flush()
|
|
39
|
+
idx += 1
|
|
40
|
+
time.sleep(0.08)
|
|
41
|
+
|
|
42
|
+
def __enter__(self):
|
|
43
|
+
if SHOW_ART:
|
|
44
|
+
top_bottom = "+" + "-" * (self.width - 2) + "+"
|
|
45
|
+
side = f"{DIM}|{RESET}"
|
|
46
|
+
|
|
47
|
+
print(f"\n{DIM}{top_bottom}{RESET}")
|
|
48
|
+
|
|
49
|
+
for line in JAAT_ART.strip("\n").split("\n"):
|
|
50
|
+
content = line.ljust(self.width - 4)
|
|
51
|
+
print(f"{side} {CYAN}{content}{RESET} {side}")
|
|
52
|
+
|
|
53
|
+
print(f"{side}" + " " * (self.width - 2) + f"{side}")
|
|
54
|
+
|
|
55
|
+
welcome_msg = f"Welcome to JAAT v{VERSION}".center(self.width - 4)
|
|
56
|
+
copyright_msg = COPYRIGHT.center(self.width - 4)
|
|
57
|
+
desc_msg = DESCRIPTION.center(self.width - 4)
|
|
58
|
+
|
|
59
|
+
print(f"{side} {BOLD}{welcome_msg}{RESET} {side}")
|
|
60
|
+
print(f"{side} {DIM}{copyright_msg}{RESET} {side}")
|
|
61
|
+
print(f"{side} {DIM}{desc_msg}{RESET} {side}")
|
|
62
|
+
print(f"{DIM}{top_bottom}{RESET}\n")
|
|
63
|
+
|
|
64
|
+
self.thread = threading.Thread(target=self._animate, daemon=True)
|
|
65
|
+
self.thread.start()
|
|
66
|
+
return self
|
|
67
|
+
|
|
68
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
69
|
+
self.done = True
|
|
70
|
+
if self.thread:
|
|
71
|
+
self.thread.join()
|
|
72
|
+
sys.stdout.write(f"\r {GREEN}✅ {self.desc}... Ready.{RESET}\n\n")
|
|
73
|
+
sys.stdout.flush()
|
|
74
|
+
|
|
75
|
+
with Loader():
|
|
76
|
+
os.environ["USE_TORCH"] = "1"
|
|
77
|
+
os.environ["USE_TF"] = "0"
|
|
78
|
+
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
|
79
|
+
os.environ["TF_ENABLE_ONEDNN_OPTS"] = "0"
|
|
80
|
+
|
|
81
|
+
from .match import Match
|
|
82
|
+
from .utils import setup, chunker, validate_inputs, diagnostic, clear_cache, shutdown, toggle_progress, toggle_threads
|
|
83
|
+
|
|
84
|
+
__all__ = [
|
|
85
|
+
"setup", "chunker", "validate_inputs", "diagnostic", "clear_cache", "shutdown", "toggle_progress", "toggle_threads",
|
|
86
|
+
"Match",
|
|
87
|
+
]
|
jaat2/base.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
# Copyright 2026 Stephen Meisenbacher
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import os
|
|
16
|
+
import psutil
|
|
17
|
+
from typing import List, Tuple, Any, Optional
|
|
18
|
+
import torch
|
|
19
|
+
import re
|
|
20
|
+
from torch.utils.data import Dataset
|
|
21
|
+
from transformers.utils import is_flash_attn_2_available
|
|
22
|
+
from wtpsplit import SaT
|
|
23
|
+
|
|
24
|
+
from .config import logger, MODEL_CACHE
|
|
25
|
+
|
|
26
|
+
BULLET_PATTERN = re.compile(r"(^|\n)[\s•·\-\*]+\s*")
|
|
27
|
+
DELIM_PATTERN = re.compile(r"[;]\s*")
|
|
28
|
+
CLEAN_PATTERN = re.compile(r"(;|\s\+\s|\s\*\s|\s\-\s|\s•\s|\s·\s|--|\*\*)")
|
|
29
|
+
MULTI_PUNCT = re.compile(r"\.{2,}")
|
|
30
|
+
MULTI_SPACE = re.compile(r"[ \t]+")
|
|
31
|
+
TRAILING_PUNCT = re.compile(r"^[ \t\n\r.\-•*]+|[ \t\n\r.\-•*]+$")
|
|
32
|
+
CLAUSE_DELIMS = re.compile(r"(\s*;\s*|\s*—\s*|\s*--\s*)")
|
|
33
|
+
|
|
34
|
+
def get_device_settings() -> Tuple[str, int]:
|
|
35
|
+
"""
|
|
36
|
+
Detects hardware capabilities and derives a sensible batch size.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
# CUDA
|
|
40
|
+
if torch.cuda.is_available():
|
|
41
|
+
device = "cuda"
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
device_props = torch.cuda.get_device_properties(0)
|
|
45
|
+
total_vram_gb = device_props.total_memory / (1024 ** 3)
|
|
46
|
+
except Exception:
|
|
47
|
+
total_vram_gb = 8.0
|
|
48
|
+
|
|
49
|
+
if total_vram_gb >= 40.0:
|
|
50
|
+
batch_size = 2048
|
|
51
|
+
elif total_vram_gb >= 20.0:
|
|
52
|
+
batch_size = 1024
|
|
53
|
+
elif total_vram_gb >= 14.0:
|
|
54
|
+
batch_size = 512
|
|
55
|
+
elif total_vram_gb >= 8.0:
|
|
56
|
+
batch_size = 256
|
|
57
|
+
else:
|
|
58
|
+
batch_size = 64
|
|
59
|
+
|
|
60
|
+
return device, batch_size
|
|
61
|
+
|
|
62
|
+
# Apple Silicon (MPS)
|
|
63
|
+
if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
|
|
64
|
+
total_ram_gb = psutil.virtual_memory().total / (1024 ** 3)
|
|
65
|
+
device = "mps"
|
|
66
|
+
|
|
67
|
+
if total_ram_gb >= 32.0:
|
|
68
|
+
batch_size = 256
|
|
69
|
+
elif total_ram_gb >= 16.0:
|
|
70
|
+
batch_size = 128
|
|
71
|
+
else:
|
|
72
|
+
batch_size = 64
|
|
73
|
+
|
|
74
|
+
return device, batch_size
|
|
75
|
+
|
|
76
|
+
# CPU Fallback
|
|
77
|
+
device = "cpu"
|
|
78
|
+
cpu_count = os.cpu_count() or 4
|
|
79
|
+
total_ram_gb = psutil.virtual_memory().total / (1024 ** 3)
|
|
80
|
+
|
|
81
|
+
if cpu_count >= 16 and total_ram_gb >= 32.0:
|
|
82
|
+
batch_size = 64
|
|
83
|
+
else:
|
|
84
|
+
batch_size = 32
|
|
85
|
+
|
|
86
|
+
return device, batch_size
|
|
87
|
+
|
|
88
|
+
def get_optimal_attn_implementation() -> str:
|
|
89
|
+
"""
|
|
90
|
+
Selects the fastest attention backend available for the current hardware.
|
|
91
|
+
"""
|
|
92
|
+
if not torch.cuda.is_available():
|
|
93
|
+
return "sdpa" if hasattr(torch.nn.functional, "scaled_dot_product_attention") else "eager"
|
|
94
|
+
|
|
95
|
+
major, _ = torch.cuda.get_device_capability(0)
|
|
96
|
+
has_compatible_arch = major >= 8
|
|
97
|
+
|
|
98
|
+
if has_compatible_arch and is_flash_attn_2_available():
|
|
99
|
+
return "flash_attention_2"
|
|
100
|
+
|
|
101
|
+
if hasattr(torch.nn.functional, "scaled_dot_product_attention"):
|
|
102
|
+
return "sdpa"
|
|
103
|
+
|
|
104
|
+
return "eager"
|
|
105
|
+
|
|
106
|
+
class ListDataset(Dataset):
|
|
107
|
+
def __init__(self, original_list: List[Any]) -> None:
|
|
108
|
+
self.original_list = original_list
|
|
109
|
+
|
|
110
|
+
def __len__(self) -> int:
|
|
111
|
+
return len(self.original_list)
|
|
112
|
+
|
|
113
|
+
def __getitem__(self, i: int) -> Any:
|
|
114
|
+
return self.original_list[i]
|
|
115
|
+
|
|
116
|
+
def clean_text(text: str) -> str:
|
|
117
|
+
"""Pre-normalizes delimiters, bullets, and line breaks into standard clause boundaries."""
|
|
118
|
+
if not text:
|
|
119
|
+
return ""
|
|
120
|
+
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
|
121
|
+
normalized_lines = [
|
|
122
|
+
line if line.endswith((".", "!", "?")) else f"{line}."
|
|
123
|
+
for line in (l.strip() for l in text.splitlines())
|
|
124
|
+
if line
|
|
125
|
+
]
|
|
126
|
+
text = " ".join(normalized_lines)
|
|
127
|
+
text = BULLET_PATTERN.sub(" ", text)
|
|
128
|
+
|
|
129
|
+
text = CLAUSE_DELIMS.sub(". ", text)
|
|
130
|
+
text = MULTI_PUNCT.sub(".", text)
|
|
131
|
+
text = MULTI_SPACE.sub(" ", text).strip()
|
|
132
|
+
return text
|
|
133
|
+
|
|
134
|
+
def filter_valid_sentence(sent: str, min_words: int = 2, max_words: int = 64) -> Optional[str]:
|
|
135
|
+
s = TRAILING_PUNCT.sub("", sent)
|
|
136
|
+
words = s.split()
|
|
137
|
+
word_count = len(words)
|
|
138
|
+
if min_words <= word_count <= max_words:
|
|
139
|
+
return s
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
class BatchedSaTSplitter:
|
|
143
|
+
def __init__(
|
|
144
|
+
self,
|
|
145
|
+
model_name: str = "sat-3l-sm",
|
|
146
|
+
device: Optional[str] = None
|
|
147
|
+
) -> None:
|
|
148
|
+
self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
|
149
|
+
self.model_name = model_name
|
|
150
|
+
self.sat = None
|
|
151
|
+
|
|
152
|
+
cache_key = f"sat_{model_name}"
|
|
153
|
+
if cache_key not in MODEL_CACHE:
|
|
154
|
+
logger.info(f"Loading SaT segmenter '{model_name}' on {self.device}...")
|
|
155
|
+
sat_instance = SaT(model_name)
|
|
156
|
+
if self.device == "cuda":
|
|
157
|
+
sat_instance.half().to("cuda")
|
|
158
|
+
MODEL_CACHE[cache_key] = sat_instance
|
|
159
|
+
self.sat = MODEL_CACHE[cache_key]
|
|
160
|
+
|
|
161
|
+
def split_two_pass(
|
|
162
|
+
self,
|
|
163
|
+
docs: List[str],
|
|
164
|
+
batch_size: int = 1024,
|
|
165
|
+
pass1_threshold: float = 0.50,
|
|
166
|
+
pass2_threshold: float = 0.20,
|
|
167
|
+
min_words: int = 2,
|
|
168
|
+
max_words: int = 48,
|
|
169
|
+
pass2_char_length: int = 80
|
|
170
|
+
) -> List[List[str]]:
|
|
171
|
+
if not docs:
|
|
172
|
+
return []
|
|
173
|
+
|
|
174
|
+
cleaned_docs = [clean_text(d) for d in docs]
|
|
175
|
+
|
|
176
|
+
doc_sentences_raw = self.sat.split(
|
|
177
|
+
cleaned_docs,
|
|
178
|
+
batch_size=batch_size,
|
|
179
|
+
threshold=pass1_threshold,
|
|
180
|
+
outer_batch_size=batch_size * 2,
|
|
181
|
+
verbose=False
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
doc_sentence_groups: List[List[str]] = []
|
|
185
|
+
pass2_candidates: List[str] = []
|
|
186
|
+
pass2_mapping: List[Tuple[int, int]] = []
|
|
187
|
+
|
|
188
|
+
for d_idx, raw_sents in enumerate(doc_sentences_raw):
|
|
189
|
+
filtered_doc_sents = []
|
|
190
|
+
for s in raw_sents:
|
|
191
|
+
clean_s = s.strip()
|
|
192
|
+
if clean_s:
|
|
193
|
+
filtered_doc_sents.append(clean_s)
|
|
194
|
+
|
|
195
|
+
doc_sentence_groups.append(filtered_doc_sents)
|
|
196
|
+
|
|
197
|
+
for s_idx, sent in enumerate(filtered_doc_sents):
|
|
198
|
+
if len(sent) >= pass2_char_length or len(sent.split()) > 15: # > (max_words // 2):
|
|
199
|
+
pass2_candidates.append(sent)
|
|
200
|
+
pass2_mapping.append((d_idx, s_idx))
|
|
201
|
+
|
|
202
|
+
replacement_map: dict[Tuple[int, int], List[str]] = {}
|
|
203
|
+
if pass2_candidates:
|
|
204
|
+
pass2_results = self.sat.split(
|
|
205
|
+
pass2_candidates,
|
|
206
|
+
batch_size=batch_size,
|
|
207
|
+
threshold=pass2_threshold,
|
|
208
|
+
outer_batch_size=batch_size * 2,
|
|
209
|
+
verbose=False
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
for (d_idx, s_idx), refined_sents in zip(pass2_mapping, pass2_results):
|
|
213
|
+
valid_splits = [s.strip(" ,;.-•*") for s in refined_sents if s.strip()]
|
|
214
|
+
cleaned_splits = []
|
|
215
|
+
for split in valid_splits:
|
|
216
|
+
split_cleaned = re.sub(r"^(and|or|whether that's|whether|such as|including)\s+", "", split, flags=re.IGNORECASE)
|
|
217
|
+
if len(split_cleaned.split()) >= min_words:
|
|
218
|
+
cleaned_splits.append(split_cleaned)
|
|
219
|
+
|
|
220
|
+
if len(cleaned_splits) > 1:
|
|
221
|
+
replacement_map[(d_idx, s_idx)] = cleaned_splits
|
|
222
|
+
|
|
223
|
+
final_docs: List[List[str]] = []
|
|
224
|
+
for d_idx, sents in enumerate(doc_sentence_groups):
|
|
225
|
+
doc_refined = []
|
|
226
|
+
for s_idx, sent in enumerate(sents):
|
|
227
|
+
key = (d_idx, s_idx)
|
|
228
|
+
candidates = replacement_map.get(key, [sent])
|
|
229
|
+
|
|
230
|
+
for candidate in candidates:
|
|
231
|
+
valid_item = filter_valid_sentence(candidate, min_words, max_words)
|
|
232
|
+
if valid_item is not None:
|
|
233
|
+
doc_refined.append(valid_item)
|
|
234
|
+
|
|
235
|
+
final_docs.append(doc_refined)
|
|
236
|
+
|
|
237
|
+
return final_docs
|
|
238
|
+
|
|
239
|
+
_GLOBAL_SPLITTER: Optional[BatchedSaTSplitter] = None
|
|
240
|
+
|
|
241
|
+
def get_sentence_splitter(device: Optional[str] = None) -> BatchedSaTSplitter:
|
|
242
|
+
global _GLOBAL_SPLITTER
|
|
243
|
+
if _GLOBAL_SPLITTER is None:
|
|
244
|
+
_GLOBAL_SPLITTER = BatchedSaTSplitter(device=device)
|
|
245
|
+
return _GLOBAL_SPLITTER
|
|
246
|
+
|
|
247
|
+
def sent_tokenize(text: str) -> List[str]:
|
|
248
|
+
splitter = get_sentence_splitter()
|
|
249
|
+
results = splitter.split_two_pass([text])
|
|
250
|
+
return results[0] if results else []
|
jaat2/config.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
logger = logging.getLogger("JAATv2")
|
|
5
|
+
|
|
6
|
+
if not logger.handlers:
|
|
7
|
+
handler = logging.StreamHandler()
|
|
8
|
+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
9
|
+
handler.setFormatter(formatter)
|
|
10
|
+
logger.addHandler(handler)
|
|
11
|
+
logger.setLevel(logging.INFO)
|
|
12
|
+
|
|
13
|
+
GLOBAL_SETTINGS = {
|
|
14
|
+
"show_progress": True,
|
|
15
|
+
"single_threaded": False
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
MODEL_CACHE = {}
|
|
19
|
+
|
|
20
|
+
VERSION = "2.0.0beta"
|
|
21
|
+
COPYRIGHT = "© 2026 Stephen Meisenbacher. All rights reserved."
|
|
22
|
+
DESCRIPTION = "An NLP-powered toolkit for job ad text analysis."
|
|
23
|
+
|
|
24
|
+
CYAN = "\033[96m"
|
|
25
|
+
GREEN = "\033[92m"
|
|
26
|
+
RESET = "\033[0m"
|
|
27
|
+
BOLD = "\033[1m"
|
|
28
|
+
DIM = "\033[2m"
|
|
29
|
+
|
|
30
|
+
# Note: 3D Diagonal
|
|
31
|
+
JAAT_ART = r"""
|
|
32
|
+
,----,
|
|
33
|
+
,---._ ,/ .`|
|
|
34
|
+
.-- -.' \ ,---, ,---, ,` .' : ,----,
|
|
35
|
+
| | : ' .' \ ' .' \ ; ; / .' .' \
|
|
36
|
+
: ; | / ; '. / ; '. .'___,/ ,' ,----,' |
|
|
37
|
+
: |: : \ : : \ | : | .---. | : . ;
|
|
38
|
+
| : :: | /\ \ : | /\ \ ; |.'; ; /. ./| ; |.' /
|
|
39
|
+
: | : ' ;. :| : ' ;. :`----' | | .-' . ' | `----'/ ;
|
|
40
|
+
| ; || | ;/ \ \ | ;/ \ \ ' : ;/___/ \: | / ; /
|
|
41
|
+
___ l ' : | \ \ ,' : | \ \ ,' | | '. \ ' . ; / /-,
|
|
42
|
+
/ /\ J :| | ' '--' | | ' '--' ' : | \ \ ' / / /.`|
|
|
43
|
+
/ ../ `..- ,| : : | : : ; |.' \ \ ./__; :
|
|
44
|
+
\ \ ; | | ,' | | ,' '---' \ \ | : .'
|
|
45
|
+
\ \ ,' `--'' `--'' '---"; | .'
|
|
46
|
+
"---....--' `---'
|
|
47
|
+
|
|
48
|
+
"""
|
|
49
|
+
SHOW_ART = os.isatty(1)
|