pyannotators-patterns 0.5.84__tar.gz → 0.5.88__tar.gz

{pyannotators_patterns-0.5.84 → pyannotators_patterns-0.5.88}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyannotators-patterns
-Version: 0.5.84
+Version: 0.5.88
 Summary: Annotator based on Presidio pattern recognizer
 Home-page: https://github.com/oterrier/pyannotators_patterns/
 Keywords: 
@@ -37,6 +37,7 @@ Requires-Dist: pytest ; extra == "test"
 Requires-Dist: pytest-cov ; extra == "test"
 Requires-Dist: pytest-flake8 ; extra == "test"
 Requires-Dist: pytest-black ; extra == "test"
+Requires-Dist: pytest_check ; extra == "test"
 Requires-Dist: flake8==3.9.2 ; extra == "test"
 Requires-Dist: tox ; extra == "test"
 Requires-Dist: dirty-equals ; extra == "test"

{pyannotators_patterns-0.5.84 → pyannotators_patterns-0.5.88}/pyproject.toml

@@ -43,7 +43,7 @@ test = [
     "pytest-cov",
     "pytest-flake8",
     "pytest-black",
-#    "hypothesis",
+    "pytest_check",
     "flake8==3.9.2",
     "tox",
     "dirty-equals"

{pyannotators_patterns-0.5.84 → pyannotators_patterns-0.5.88}/setup.py

@@ -31,6 +31,7 @@ extras_require = \
           'pytest-cov',
           'pytest-flake8',
           'pytest-black',
+          'pytest_check',
           'flake8==3.9.2',
           'tox',
           'dirty-equals']}
@@ -40,7 +41,7 @@ entry_points = \
                           'pyannotators_patterns.patterns:PatternsAnnotator']}
 
 setup(name='pyannotators-patterns',
-      version='0.5.84',
+      version='0.5.88',
       description='Annotator based on Presidio pattern recognizer',
       author='Olivier Terrier',
       author_email='olivier.terrier@kairntech.com',

{pyannotators_patterns-0.5.84 → pyannotators_patterns-0.5.88}/src/pyannotators_patterns/__init__.py

@@ -1,2 +1,2 @@
 """Annotator based on Presidio pattern recognizer"""
-__version__ = "0.5.84"
+__version__ = "0.5.88"

pyannotators_patterns-0.5.88/src/pyannotators_patterns/named_pattern_recognizer.py

@@ -0,0 +1,114 @@
+import re
+from typing import List, Optional
+
+from presidio_analyzer import Pattern, PatternRecognizer, RecognizerResult, EntityRecognizer
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+
+class NamedPatternRecognizer(PatternRecognizer):
+    def __init__(
+            self,
+            supported_entity: str,
+            name: str = None,
+            supported_language: str = "en",
+            patterns: List[Pattern] = None,
+            deny_list: List[str] = None,
+            context: List[str] = None,
+            deny_list_score: float = 1.0,
+            global_regex_flags: Optional[int] = re.DOTALL | re.MULTILINE | re.IGNORECASE,
+            version: str = "0.0.1",
+    ):
+        super().__init__(supported_entity, name, supported_language, patterns, deny_list, context, deny_list_score,
+                         global_regex_flags, version)
+
+    def analyze(
+            self,
+            text: str,
+            entities: List[str],
+            nlp_artifacts: Optional[NlpArtifacts] = None,
+            regex_flags: Optional[int] = None,
+    ) -> List[RecognizerResult]:
+        """
+        Analyzes text to detect PII using regular expressions or deny-lists.
+
+        :param text: Text to be analyzed
+        :param entities: Entities this recognizer can detect
+        :param nlp_artifacts: Output values from the NLP engine
+        :param regex_flags: regex flags to be used in regex matching
+        :return:
+        """
+        results = []
+
+        if self.patterns:
+            pattern_result = self.__analyze_patterns(text, regex_flags)
+            results.extend(pattern_result)
+
+        return results
+
+    def __analyze_patterns(
+            self, text: str, flags: int = None
+    ) -> List[RecognizerResult]:
+        """
+        Evaluate all patterns in the provided text.
+
+        Including words in the provided deny-list
+
+        :param text: text to analyze
+        :param flags: regex flags
+        :return: A list of RecognizerResult
+        """
+        flags = flags if flags else self.global_regex_flags
+        results = []
+        for pattern in self.patterns:
+            # Compile regex if flags differ from flags the regex was compiled with
+            if not pattern.compiled_regex or pattern.compiled_with_flags != flags:
+                pattern.compiled_with_flags = flags
+                pattern.compiled_regex = re.compile(pattern.regex, flags=flags)
+
+            matches = pattern.compiled_regex.finditer(text)
+            for match in matches:
+                start, end = match.span()
+                current_match = text[start:end]
+
+                # Skip empty results
+                if current_match == "":
+                    continue
+
+                score = pattern.score
+
+                validation_result = self.validate_result(current_match)
+                description = self.build_regex_explanation(
+                    self.name,
+                    pattern.name,
+                    pattern.regex,
+                    score,
+                    validation_result,
+                    flags,
+                )
+                pattern_result = RecognizerResult(
+                    entity_type=self.supported_entities[0],
+                    start=start,
+                    end=end,
+                    score=score,
+                    analysis_explanation=description,
+                    recognition_metadata=match.groupdict()
+                )
+
+                if validation_result is not None:
+                    if validation_result:
+                        pattern_result.score = EntityRecognizer.MAX_SCORE
+                    else:
+                        pattern_result.score = EntityRecognizer.MIN_SCORE
+
+                invalidation_result = self.invalidate_result(current_match)
+                if invalidation_result is not None and invalidation_result:
+                    pattern_result.score = EntityRecognizer.MIN_SCORE
+
+                if pattern_result.score > EntityRecognizer.MIN_SCORE:
+                    results.append(pattern_result)
+
+                # Update analysis explanation score following validation or invalidation
+                description.score = pattern_result.score
+
+        results = EntityRecognizer.remove_duplicates(results)
+        return results

{pyannotators_patterns-0.5.84 → pyannotators_patterns-0.5.88}/src/pyannotators_patterns/patterns.py

@@ -5,7 +5,7 @@ from typing import Type, List, cast, Dict
 
 import spacy
 from log_with_context import add_logging_context
-from presidio_analyzer import Pattern, PatternRecognizer, RecognizerRegistry, AnalyzerEngine, LemmaContextAwareEnhancer
+from presidio_analyzer import Pattern, RecognizerRegistry, AnalyzerEngine, LemmaContextAwareEnhancer, RecognizerResult
 from presidio_analyzer.nlp_engine import SpacyNlpEngine, NerModelConfiguration
 from pydantic import BaseModel, Field
 from pymultirole_plugins.util import comma_separated_to_list
@@ -15,6 +15,8 @@ from spacy.cli.download import download_model, get_compatibility, get_version
 from spacy.language import Language
 from wasabi import msg
 
+from .named_pattern_recognizer import NamedPatternRecognizer
+
 logger = logging.getLogger(__name__)
 
 PATTERNS_EXAMPLE = {
@@ -50,7 +52,7 @@ class PatternsParameters(AnnotatorParameters):
     ```""" + PATTERNS_EXAMPLE_STR + "```", extra="key:label,val:json")
 
     score_threshold: float = Field(0.0, description="Minimum confidence value for detected entities to be returned")
-    context_similarity_factor: float = Field(0.0,
+    context_similarity_factor: float = Field(0.35,
                                              description="How much to enhance confidence of match entity, as explained [here](https://microsoft.github.io/presidio/tutorial/06_context/)",
                                              extra="advanced")
     min_score_with_context_similarity: float = Field(0.4,
@@ -111,6 +113,9 @@ class PatternsAnnotator(AnnotatorBase):
                         for result in results:
                             start = s.start + result.start
                             end = s.start + + result.end
+                            props = {k: v for k, v in result.recognition_metadata.items() if
+                                     k not in [RecognizerResult.RECOGNIZER_NAME_KEY,
+                                               RecognizerResult.RECOGNIZER_IDENTIFIER_KEY]}
                             document.annotations.append(
                                 Annotation(
                                     start=start,
@@ -118,7 +123,7 @@ class PatternsAnnotator(AnnotatorBase):
                                     text=document.text[start: end],
                                     labelName=result.entity_type,
                                     score=result.score,
-                                    properties={"analysis_explanation": result.analysis_explanation}
+                                    properties=props
                                 )
                             )
 
@@ -171,7 +176,7 @@ def get_registry(mapping_items, lang):
         except BaseException:
             logger.warning(f"Invalid json: {pvalue}", exc_info=True)
         patterns = [Pattern.from_dict(pat) for pat in pattern_definition['patterns']]
-        recognizer = PatternRecognizer(
+        recognizer = NamedPatternRecognizer(
             supported_entity=pname, supported_language=lang, patterns=patterns, context=pattern_definition.get('context', None)
         )
         recognizers.append(recognizer)

pyannotators_patterns-0.5.88/tests/data/coords-document.json

@@ -0,0 +1,8 @@
+{
+  "text": "    Coordonnées degrés décimaux, référence devant :\n    Exemple : N85,8598654 W150,589654\n\n\n    Coordonnées degrés décimaux, référence derrière  : \n    Exemple 85,8598654N 150,589654W",
+  "metadata": {
+    "language": "fr"
+  },
+  "identifier": "c8ecc5153c542c4f6a07fbe813384842",
+  "title": "unknown test document"
+}

pyannotators_patterns-0.5.88/tests/data/coords.json

@@ -0,0 +1,15 @@
+{
+  "patterns": [
+    {
+      "name": "ref_lat",
+      "regex": "\\b(?P<ref_lat>N|S|n|s)(?P<val_lat>[0-9,]+)\\b",
+      "score": 0.9
+    },
+    {
+      "name": "ref_lat_DEVANT",
+      "regex": "\\b(?P<val_lat>[0-9,]+)(?P<ref_lat>N|S|n|s)\\b",
+      "score": 0.9
+    }
+  ],
+  "context": []
+}

pyannotators_patterns-0.5.88/tests/data/tel-document.json

@@ -0,0 +1,8 @@
+{
+  "text": "Si vous êtes intéressé, veuillez nous appeler au +33.089-658-6494.",
+  "metadata": {
+    "language": "fr"
+  },
+  "identifier": "c9321541549ae539665b8bbf440cb1a0",
+  "title": "unknown test document"
+}

pyannotators_patterns-0.5.88/tests/data/tel.json

@@ -0,0 +1,21 @@
+{
+  "patterns": [
+    {
+      "name": "avec préfixe",
+      "regex": "(00|\\+)( |\\-|\\.)?(?P<prefix>9[976]\\d|8[987530]\\d|6[987]\\d|5[90]\\d|42\\d|3[875]\\d|2[98654321]\\d|9[8543210]|8[6421]|6[6543210]|5[87654321]|4[987654310]|3[9643210]|2[70]|7|1)( |\\-|\\.)?(?P<number>(?:\\d{1,3}( |\\-|\\.)?){5,14})\\b",
+      "score": 0.5
+    },
+    {
+      "name": "sans préfixe",
+      "regex": "\\b(?P<number>(?:\\d{1,3}( |\\-|\\.)?){5,14})\\b",
+      "score": 0.5
+    }
+  ],
+  "context": [
+    "appeler",
+    "appel",
+    "numéro",
+    "téléphone",
+    "téléphoner"
+  ]
+}

pyannotators_patterns-0.5.88/tests/test_coords.py

@@ -0,0 +1,37 @@
+import json
+from pathlib import Path
+from typing import List
+
+from dirty_equals import IsPartialDict
+from pymultirole_plugins.v1.schema import Document
+from pytest_check import check
+
+from pyannotators_patterns.patterns import PatternsAnnotator, PatternsParameters
+
+
+def test_coords(
+):
+    testdir = Path(__file__).parent
+    source = Path(testdir, "data/coords.json")
+    with source.open("r") as fin:
+        pat = json.load(fin)
+        parameters = PatternsParameters(mapping={
+            "coords": json.dumps(pat, indent=2)
+        })
+    source = Path(testdir, "data/coords-document.json")
+    with source.open("r") as fin:
+        jdoc = json.load(fin)
+        doc = Document(**jdoc)
+    annotator = PatternsAnnotator()
+    docs: List[Document] = annotator.annotate([doc], parameters)
+    doc0 = docs[0]
+    lat0 = next(a.dict(exclude_none=True, exclude_unset=True) for a in doc0.annotations if
+                a.text == 'N85,8598654')
+    with check:
+        assert lat0 == IsPartialDict(labelName='coords', text='N85,8598654',
+                                     properties=IsPartialDict(ref_lat='N', val_lat='85,8598654'))
+
+    lat1 = next(a.dict(exclude_none=True, exclude_unset=True) for a in doc0.annotations if
+                a.text == '85,8598654N')
+    with check:
+        assert lat1 == IsPartialDict(labelName='coords', text='85,8598654N', properties=IsPartialDict(ref_lat='N', val_lat='85,8598654'))

pyannotators_patterns-0.5.88/tests/test_tel.py

@@ -0,0 +1,32 @@
+import json
+from pathlib import Path
+from typing import List
+
+from dirty_equals import IsPartialDict
+from pymultirole_plugins.v1.schema import Document
+from pytest_check import check
+
+from pyannotators_patterns.patterns import PatternsAnnotator, PatternsParameters
+
+
+def test_tel(
+):
+    testdir = Path(__file__).parent
+    source = Path(testdir, "data/tel.json")
+    with source.open("r") as fin:
+        pat = json.load(fin)
+        parameters = PatternsParameters(mapping={
+            "telephone": json.dumps(pat, indent=2)
+        })
+    source = Path(testdir, "data/tel-document.json")
+    with source.open("r") as fin:
+        jdoc = json.load(fin)
+        doc = Document(**jdoc)
+    annotator = PatternsAnnotator()
+    docs: List[Document] = annotator.annotate([doc], parameters)
+    doc0 = docs[0]
+    tel = next(a.dict(exclude_none=True, exclude_unset=True) for a in doc0.annotations if
+               a.text == '+33.089-658-6494')
+    with check:
+        assert tel == IsPartialDict(labelName='telephone', text='+33.089-658-6494', score=0.85,
+                                    properties=IsPartialDict(prefix='33', number='089-658-6494'))