streamlit-octostar-utils 2.11a5__tar.gz → 2.11a6__tar.gz

{streamlit_octostar_utils-2.11a5 → streamlit_octostar_utils-2.11a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 2.11a5
+Version: 2.11a6
 Summary: 
 License: MIT
 License-File: LICENSE

{streamlit_octostar_utils-2.11a5 → streamlit_octostar_utils-2.11a6}/pyproject.toml

@@ -5,7 +5,7 @@ include = '\.pyi?$'
 
 [tool.poetry]
 name = "streamlit-octostar-utils"
-version = "2.11a5"
+version = "2.11a6"
 description = ""
 license = "MIT"
 authors = ["Octostar"]

streamlit_octostar_utils-2.11a6/streamlit_octostar_utils/nlp/language.py

@@ -0,0 +1,64 @@
+import re
+import py3langid as langid
+
+from iso639 import Lang
+
+FLAIR_MODELS = {
+    "en": "flair/ner-english-large",
+    "es": "flair/ner-spanish-large",
+    "de": "flair/ner-german-large",
+    "nl": "flair/ner-dutch-large",
+    "multi": "flair/ner-multi",                     # English, German, French, Spanish
+    "multi-fast": "flair/ner-multi-fast",           # English, German, Dutch, Spanish
+}
+
+SPACY_MODELS = {
+    "en": "en_core_web_sm",
+    "es": "es_core_news_sm",
+    "fr": "fr_core_news_sm",
+    "de": "de_core_news_sm",
+    "it": "it_core_news_sm"
+}
+
+
+def alpha2_to_language(alpha2: str) -> str:
+    if not alpha2:
+        raise ValueError("Language code must be a non-empty string.")
+    return Lang(alpha2).name
+
+
+def language_to_alpha2(language_name: str) -> str:
+    if not language_name:
+        raise ValueError("Language name must be a non-empty string.")
+
+    name = re.sub(r'\b\w+', lambda m: m.group(0).capitalize(), name)
+    return Lang(name).pt1
+
+
+def detect_language(text, min_confidence=None):
+    detector = langid.langid.LanguageIdentifier.from_pickled_model(
+        langid.langid.MODEL_FILE, norm_probs=True
+    )
+    detected_lang, confidence = detector.classify(text)
+    if min_confidence and confidence < min_confidence:
+        return None, confidence
+    detected_lang = alpha2_to_language(detected_lang)
+    return detected_lang, confidence
+
+
+def load_language_model(language, type):
+    from flair.models import SequenceTagger
+    from spacy_download import load_spacy
+
+    model = None
+
+    match type:
+        case "spacy":
+            model_name = SPACY_MODELS.get(language_to_alpha2(language), SPACY_MODELS["en"])
+            model = load_spacy(model_name)
+
+        case "flair":
+            model_name = FLAIR_MODELS.get(language, "flair/ner-multi")
+            model = SequenceTagger.load(model_name)
+
+    return model

{streamlit_octostar_utils-2.11a5 → streamlit_octostar_utils-2.11a6}/streamlit_octostar_utils/nlp/ner.py

@@ -20,7 +20,7 @@ from sumy.summarizers.lsa import LsaSummarizer
 from sumy.summarizers.luhn import LuhnSummarizer
 from sumy.utils import get_stop_words
 
-from .language import alpha2_to_language, language_to_alpha2
+from .language import alpha2_to_language
 
 BASE_ALLOWED_LABELS = ["PERSON", "ORG", "LOC", "NORP", "GPE", "PRODUCT", "DATE", "PHONE", "IP_ADDRESS", "EMAIL", "URL",
                        "CRYPTO", "IBAN", "CREDIT_CARD", "US_SSN", "US_DRIVER_LICENSE", "US_PASSPORT", "MEDICAL_LICENSE"]
@@ -67,38 +67,6 @@ BASE_TO_RECOGNIZER_EXPANSIONS = {
 
 BASE_TO_ONTONOTES_LABELMAP = {"PER": "PERSON"}
 
-FLAIR_MODELS = {
-    "en": "flair/ner-english-large",
-    "es": "flair/ner-spanish-large",
-    "de": "flair/ner-german-large",
-    "nl": "flair/ner-dutch-large",
-    "multi": "flair/ner-multi",                     # English, German, French, Spanish
-    "multi-fast": "flair/ner-multi-fast",           # English, German, Dutch, Spanish
-}
-
-SPACY_MODELS = {
-    "en": "en_core_web_sm",
-    "es": "es_core_news_sm",
-    "fr": "fr_core_news_sm",
-    "de": "de_core_news_sm",
-    "it": "it_core_news_sm"
-}
-
-def load_language_model(language, type):
-    from flair.models import SequenceTagger
-
-    model = None
-
-    match type:
-        case "spacy":
-            model = SPACY_MODELS.get(language_to_alpha2(language), SPACY_MODELS["en"])
-
-        case "flair":
-            model_name = FLAIR_MODELS.get(language, "flair/ner-multi")
-            model = SequenceTagger.load(model_name)
-
-    return model
-
 
 class FlairRecognizer(EntityRecognizer):
     ENTITIES = [
@@ -430,7 +398,7 @@ def analyze_dataframe_optimized(df: pd.DataFrame, analyzer: AnalyzerEngine, lang
 
 
 def compute_ner_presidio(
-        text,
+        text_or_df,
         language,
         analyzer,
         entities=None,
@@ -441,15 +409,15 @@ def compute_ner_presidio(
         batch_size=32,
         n_process=4
 ):
-    if isinstance(text, pd.DataFrame):
-        if len(text) >= 100:
-            return analyze_dataframe_optimized(text, analyzer, language, entities, score_threshold)
+    if isinstance(text_or_df, pd.DataFrame):
+        if len(text_or_df) >= 100:
+            return analyze_dataframe_optimized(text_or_df, analyzer, language, entities, score_threshold)
 
         else:
             texts = []
 
-            for col in text.columns:
-                for idx, value in text[col].dropna().items():
+            for col in text_or_df.columns:
+                for idx, value in text_or_df[col].dropna().items():
                     text_value = str(value).strip()
 
                     if text_value:
@@ -457,7 +425,8 @@ def compute_ner_presidio(
 
             text = "\n".join(texts)
 
-    elif isinstance(text, list):
+    elif isinstance(text_or_df, list):
+        text = text_or_df
         batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)
 
         results_generator = batch_analyzer.analyze_iterator(
@@ -566,7 +535,7 @@ def get_extractive_summary(text, language, max_chars, fast=False, with_scores=Fa
 
 
 def ner_pipe(
-        text,
+        text_or_df,
         language,
         model,
         engine_type="spacy",
@@ -589,9 +558,9 @@ def ner_pipe(
         model=model,
     )
 
-    if isinstance(text, pd.DataFrame):
+    if isinstance(text_or_df, pd.DataFrame):
         ner = compute_ner_presidio(
-            text,
+            text_or_df,
             language,
             analyzer,
             entities,
@@ -602,6 +571,8 @@ def ner_pipe(
             n_process=n_process
         )
     else:
+        text = text_or_df
+
         if compression_ratio == "auto":
             compression_ratio = max(1.0, len(text) / 15000) if fast else 1.0
 
@@ -640,8 +611,8 @@ def get_ner_handler(
     except LookupError:
         language = "en"
 
-    return lambda text, compression_ratio="auto", with_scores=False, with_comentions=True, with_context=True: ner_pipe(
-        text,
+    return lambda text_or_df, compression_ratio="auto", with_scores=False, with_comentions=True, with_context=True: ner_pipe(
+        text_or_df,
         language,
         model,
         engine_type,

streamlit_octostar_utils-2.11a5/streamlit_octostar_utils/nlp/language.py

@@ -1,28 +0,0 @@
-import re
-import py3langid as langid
-import iso639 as languages
-
-
-def alpha2_to_language(alpha2: str) -> str:
-    if not alpha2:
-        return None
-    code = alpha2.strip().lower()
-    return languages.to_name(code)
-
-def language_to_alpha2(language_name: str) -> str:
-    if not language_name:
-        return None
-    name = language_name.strip().lower()
-    data = languages.find(name)
-    return data["iso639_1"]
-
-def detect_language(text, min_confidence=None):
-    detector = langid.langid.LanguageIdentifier.from_pickled_model(
-        langid.langid.MODEL_FILE, norm_probs=True
-    )
-    detected_lang, confidence = detector.classify(text)
-    if min_confidence and confidence < min_confidence:
-        return None, confidence
-    detected_lang = re.sub("[^A-Za-z]", "", detected_lang).lower()
-    detected_lang = languages.to_name(detected_lang).lower()
-    return detected_lang, confidence