protoerror 3.20.2__py3-none-any.whl

protoerror/linter.py

@@ -0,0 +1,428 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""The `Linter` defines an interface to write and separate `Finding`s
+which are produced due the `Checker`s.
+
+There are 2 types of Findings. The first finding type is to deliver
+information to the user. These are findings which are `active` and
+`confident` enough to present them to the user as FAILUREs in there
+document. The other type is to give the devloper more information to
+improve the platform.
+
+Note: This class is thread-safe.
+"""
+
+import collections
+import contextlib
+import dataclasses
+import functools
+import importlib
+import os
+import threading
+
+import iamraw
+import serializeraw
+import utilo
+
+import protoerror.config
+import protoerror.control
+import protoerror.finding
+import protoerror.solution
+import protoerror.utils
+
+USER_FILE = 'user_user.yaml'
+DEVELOPER_FILE = 'developer_developer.yaml'
+
+
+@dataclasses.dataclass
+class DumpedLinterResult:
+    user: str
+    developer: str
+
+    def __getitem__(self, index):
+        """Support tuple-like access.
+
+        Example:
+            user, developer = linter_result
+        """
+        if index == 0:  # pylint:disable=C2001
+            return self.user
+        if index == 1:
+            return self.developer
+        raise IndexError(f'index to high {index}')
+
+
+class Linter:
+    """Hint: Messages are activate in default."""
+
+    def __init__(
+        self,
+        solver: protoerror.solution.Solver = None,
+        active: protoerror.config.MessageStatusList = None,
+        checkers: list = None,
+        document: iamraw.DocInfo = None,
+    ):
+        # TODO: USE CHECKER DIRECTLY TO REDUCE AMOUT OF CODE
+        self.solver = solver
+        self.active = {item.msgid: item for item in active} if active else {}
+        self.checkerlist = list(checkers) if checkers else []
+        self.only, self.skip = only_skip(self.checkerlist)
+        self.findings = []
+        self.document = document
+        self.lock = threading.Lock()  # make class thread safe
+
+    def add_finding(
+        self,
+        location: iamraw.Location = None,
+        msgid: str = None,
+        confidence: float = 1.0,
+        **kwargs,
+    ):
+        """Add Finding to store linted result.
+
+        Args:
+            location: locate linting in document
+            msgid: use msgid to mark this problem and find a solution.
+            confidence: how confident this linting is in range
+                        lowest(0.0) to highest (1.0). Lower confident
+                        findings are not presented to the user to avoid
+                        bad quality lintings.
+            kwargs: use key words args to replace values in solution
+                    template.
+        """
+        if self.document and self.document.sections:
+            only, skip = set(), set()
+            with contextlib.suppress(KeyError):
+                only = self.only[msgid]
+            with contextlib.suppress(KeyError):
+                skip = self.skip[msgid]
+            if not self.document.sections(
+                    location=location,
+                    only=only,
+                    skip=skip,
+            ):
+                utilo.debug(f'skip finding in section: {msgid}, {location}')
+                # do not add this finding
+                return
+        # Determine a possible solution
+        solution = None
+        if self.solver:
+            if self.document == iamraw.Generator.MSWORD:
+                kwargs['MSWORD'] = True
+            if self.document == iamraw.Generator.LATEX:
+                kwargs['LATEX'] = True
+            if self.document == iamraw.Generator.UNDEFINED:
+                kwargs['UNDEFINED'] = True
+            solution = self.solver.solution(msgid=msgid, **kwargs)
+
+        active = self.isactive(msgid, confidence)
+        # create finding
+        finding = iamraw.Finding(
+            confidence=confidence,
+            location=location,
+            msgid=msgid,
+            solution=solution,
+            active=active,
+        )
+        finding.number = protoerror.finding.hash_finding(finding)
+        # store finding
+        with self.lock:
+            self.findings.append(finding)
+
+    def count_findings(self, msgid: str):
+        with self.lock:
+            counted = utilo.counts(self.findings, lambda x: x.msgid == msgid)
+        return counted
+
+    def check_findings(self, check: callable):
+        """Run method to rewrite current `findings`."""
+        with self.lock:
+            self.findings = check(self.findings)
+
+    @property
+    def checkers(self):
+        result = self.checkerlist
+        if self.document:
+            result = protoerror.filter_checkers(result, self.document)
+        return result
+
+    def isactive(self, msgid, confidence):
+        if not self.active:
+            return True
+        active = True
+        with contextlib.suppress(KeyError):
+            msgstatus = self.active[msgid]
+            active = msgstatus.active and msgstatus.confidence >= confidence
+        return active
+
+    def write(self, path: str, unique: bool = False):
+        """Write linter result to `user` and `developer`-file.
+
+        Args:
+            path(str): directory to write both files
+            unique(bool): if unique no duplicated user-message are written
+        """
+        assert os.path.isdir(path), str(path)
+        # create result
+        result = self.result(unique=unique)
+        write_result(result, path, unique=unique)
+
+    def run(self, driver=None):
+        self.findings = []
+        # select document dependend checkers
+        for checker in self.checkers:
+            call = functools.partial(
+                self.add_finding,
+                msgid=checker.msgid,
+            )
+            checker(call, driver)
+
+    def result(self, unique: bool = False):
+        """Return current linter result of `user`, `developer`"""
+        with self.lock:
+            result = self.findings[:]
+        if unique:
+            result = utilo.unique(result)
+        return result
+
+    def register_checker(self, checker):
+        """Required method to auto register this checker."""
+        self.checkers.append(checker)
+
+
+def split_userdeveloper(items: list, checkers: list) -> tuple:
+    if not checkers:
+        checkers = []
+    # move inactive findings to developer
+    user, developer = utilo.partition(items=items, key=lambda item: item.active)
+    perpage_disabled = perpage_disable(user, checkers)
+    # move disabled findings to developer findings, do not show it to the
+    # user.
+    for item in perpage_disabled:
+        user.remove(item)
+        developer.append(item)
+    return user, developer
+
+
+def perpage_disable(findings, checkers):
+    """Determine list of findings which are disabled by
+    @disable-decorator."""
+    findings = [item for item in findings if item.location is not None]
+    grouped = protoerror.bypage(findings)
+    # bypage
+    result = []
+    perpage = protoerror.control.get_perpage(checkers)
+    for pageitem in grouped:
+        paged = protoerror.byid(pageitem.content)
+        for method in perpage:
+            msgid = method.msgid
+            try:
+                findings = paged[msgid]
+            except KeyError:
+                continue
+            if not protoerror.is_disabled_perpage(findings, method):
+                # content is not disabled
+                continue
+            result.extend(findings)
+    return result
+
+
+def only_skip(checkers):
+    only = collections.defaultdict(set)
+    skip = collections.defaultdict(set)
+    for item in checkers:
+        msgid = item.msgid
+        item_only, item_skip = protoerror.control.only_skip(item)
+        only[msgid] |= item_only
+        skip[msgid] |= item_skip
+    only: dict = dict(only)
+    skip: dict = dict(skip)
+    return only, skip
+
+
+def dump_result(
+    items: iamraw.Findings,
+    *,
+    unique: bool = False,
+    checkers: list = None,
+) -> DumpedLinterResult:
+    """Write linter result to `user` and `developer`-file.
+
+    Args:
+        items(list): list of `Finding`s
+        unique(bool): remove duplicated linter findings
+        checkers(methods): list of user linters
+    Returns:
+        Result with dumped user ander developer result in yaml format.
+    """
+    if unique:
+        items = utilo.unique(items)
+
+    user, developer = split_userdeveloper(items, checkers=checkers)
+
+    dumped_user = serializeraw.dump_findings(user)
+    dumped_developer = serializeraw.dump_findings(developer)
+
+    result = DumpedLinterResult(user=dumped_user, developer=dumped_developer)
+    return result
+
+
+def write_result(
+    result: iamraw.Findings,
+    path: str,
+    *,
+    unique: bool = False,
+    user_file=USER_FILE,
+    dev_file=DEVELOPER_FILE,
+    private: bool = False,
+):
+    """Write linter result to `user` and `developer`-file.
+
+    Args:
+        result(list): list of `Finding`s
+        path(str): directory to write both files unique(bool): if unique
+                   no duplicated user-messages are written
+        unique(bool): remove duplication out of result
+        user_file(str): filename of user linting file. If None, write
+                        nothing
+        dev_file(str): filename of developer linting file. If None,
+                        write nothing
+        private(bool): use encryption
+    """
+    assert os.path.isdir(path), str(path)
+    dumped_user, dumped_developer = dump_result(result, unique=unique)
+    if user_file:
+        user_outpath = os.path.join(path, user_file)
+        utilo.file_replace(user_outpath, dumped_user, private=private)
+    if dev_file:
+        developer_outpath = os.path.join(path, dev_file)
+        utilo.file_replace(developer_outpath, dumped_developer, private=private)
+
+
+def from_file(path: str, document: iamraw.DocInfo = None) -> Linter:
+    filename = os.path.basename(path)
+    spec = importlib.util.spec_from_file_location(
+        filename,
+        os.path.join(path),
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    try:
+        solution = module.SOLUTION
+    except AttributeError as error:
+        msg = f'could not create solver, no SOLUTION: {path}'
+        raise ValueError(msg) from error
+    try:
+        status = module.STATUS
+    except AttributeError:
+        utilo.debug(f'no `STATUS` provided in {path}')
+        status = []
+    result = from_solution(
+        solution,
+        status,
+        document=document,
+    )
+    return result
+
+
+def from_solution(
+    solutions: iamraw.Solutions,
+    statuses: protoerror.config.MessageStatusList,
+    checkers: list = None,
+    document: iamraw.DocInfo = None,
+) -> Linter:
+    solver = protoerror.solution.Solver.fromlist(solutions)
+    result = Linter(
+        solver,
+        active=statuses,
+        checkers=checkers,
+        document=document,
+    )
+    return result
+
+
+def from_module(
+    name: str,
+    tests: set = None,
+    skips: set = None,
+    document: iamraw.DocInfo = None,
+) -> Linter:
+    result = from_modules(
+        [name],
+        tests=tests,
+        skips=skips,
+        document=document,
+    )
+    return result
+
+
+def from_modules(
+    modules: utilo.Strings,
+    tests: set = None,
+    skips: set = None,
+    document: iamraw.DocInfo = None,
+) -> Linter:
+    modules = module_list(modules)
+    status = []
+    checkers = []
+    solutions = []
+    for name in modules:
+        with contextlib.suppress(AttributeError):
+            # support module type, ensure that module name is str
+            name = name.__name__
+        module = protoerror.utils.module_fromname(name)
+        solutions.extend(
+            protoerror.solution.parse_solutions(
+                module,
+                tests=tests,
+                skips=skips,
+            ))
+        status.extend(parse_active(module))
+        checkers.extend(
+            protoerror.parse_checkers(
+                module,
+                tests=tests,
+                skips=skips,
+            ))
+    result = protoerror.from_solution(
+        solutions,
+        status,
+        checkers=checkers,
+        document=document,
+    )
+    return result
+
+
+def module_list(modulename: list) -> list:
+    r"""\
+    >>> import protoerror.simple; protoerror.simple.run(protoerror.simple)
+    ('[]\n', '[]\n')
+    """
+    if isinstance(modulename, str):
+        return [modulename]
+    if type(modulename).__name__ == 'module':
+        return [modulename]
+    return modulename
+
+
+def parse_active(module):
+    checkers = protoerror.solution.parse_checkers(module)
+    result = [
+        protoerror.MessageStatus(
+            msgid=protoerror.solution.parse_msgid(item.__name__),
+            active=item.confidence > 0.0,
+            confidence=item.confidence,
+        ) for item in checkers
+    ]
+    return result
+
+
+# def register(linter):
+#     """required method to auto register this checker """
+#     linter.register_checker(MisdesignChecker(linter))

protoerror/merger.py

@@ -0,0 +1,69 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+
+import protoerror
+
+
+def merge_findings(findings):
+    """Merge lintings which are neighbors and lint the same error.
+
+    1. Group by page
+    2. Group by message id
+    3. Merge equal neighbors
+    """
+    ranged, notranged = utilo.partition(
+        key=lambda x: x.location.shortcut == 'r',
+        items=findings,
+    )
+    result = list(notranged)
+    paged = protoerror.bypage(ranged)
+    for page in paged:
+        msgid = protoerror.byid(page)
+        for messagegroup in msgid.values():
+            merged = merge_equal_findingid(messagegroup)
+            result.extend(merged)
+    return result
+
+
+def merge_equal_findingid(findings):
+    if not findings:
+        return []
+    # remove findings without lines, cause we carn't merge them
+    locations, nolines = utilo.partition(
+        lambda x: x.location.line is not None,
+        findings,
+    )
+    if not locations:
+        # no findings with lines
+        return findings
+    findings = sorted(locations, key=lambda x: x.location.line)
+    result = [findings[0]]
+    for item in findings[1:]:
+        before = result[-1]
+        linebefore = before.location.line
+        if before.location.line_end is not None:
+            linebefore = before.location.line_end
+        if item.msgid != before.msgid:
+            result.append(item)
+            continue
+        if item.solution.description != before.solution.description:
+            result.append(item)
+            continue
+        if item.location.line != linebefore + 1:
+            result.append(item)
+            continue
+        before.location.line_end = linebefore + 1
+    # update finding number
+    for finding in result:
+        finding.number = protoerror.finding.hash_finding(finding)
+    # append no lines
+    findings.extend(nolines)
+    return result

protoerror/messages.py

@@ -0,0 +1,95 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2019-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+"""The `messages` defines an interface to group problems and sort them by
+weight.
+
+MSG_TYPES description:
+
+    Info: document statistics, wordcount, information about using style,
+          document equality to famous writers.
+
+    Convention: page order - table of content at the end of the
+                document, font style, font size, font spacing.
+
+    Refactor: writing style of paragraph with rewriting hint
+
+    Warning: Umgangsprache, Black and Write-printing warning, to high/low
+             image resolution.
+
+    Error: Write text over page border, text formatting problem
+          "Hurensohn", broken table, wrong citatation, different
+          citation styles, missing reference, table of content
+          order/level definition.
+
+    Fatal: An error which blocks further analysis auf the document. In
+           general this is an program error which is triggered by pdf state.
+
+MSG definition:
+
+    'ERROR_CODE': (
+        'REASON',
+        'INTERNAL_SHORTCUT',
+        'DESCRIPTION_OF_THE_PROBLEM',
+    ),
+
+"""
+
+import contextlib
+
+import utilo
+
+# pylint: disable=consider-using-namedtuple-or-dataclass
+MSGS = {
+    'F0000': (
+        'Fehler beim Lesen der PDF Datei.',
+        'pdf-read-error',
+        'Used when pdf-miner is not able to read pdf file.',
+    ),
+    'F0001': (
+        'Fehler beim Extrahieren der PDF Datei.',
+        'pdf-extract-error',
+        'Used when environment is not able to read given pdf file.',
+    ),
+}
+
+MSG_TYPES = {
+    "I": "info",
+    "C": "convention",  # page order
+    "R": "refactor",  # style of paragraph, page
+    "W": "warning",  # umgangssprache, image black and white problem
+    "E": "error",  # writing over border
+    "F": "fatal",  # pdf analyzing error
+}
+
+TYPE_DEFAULT = 'W'
+
+
+def parse_msgid(msgid: str, idonly: bool = False) -> tuple[str, int]:
+    """Split `msgid` into `type` and `number`.
+
+    Args:
+        msgid(str): define type and number of used message
+        idonly(bool): do not return msg type
+    Returns:
+        typ(str), number(int) of message
+    """
+    if not msgid:
+        return msgid
+    with contextlib.suppress(ValueError):
+        msgid = int(msgid)
+        if idonly:
+            return msgid
+        return TYPE_DEFAULT, msgid
+    typ, number = msgid[0], int(msgid[1:])
+    typ = typ.upper()
+    assert typ in MSG_TYPES, (f'invalid msg type: {typ}; '
+                              f'use {utilo.from_tuple(MSG_TYPES.keys())}')
+    if idonly:
+        return number
+    return typ, number

protoerror/paged.py

@@ -0,0 +1,109 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2020-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import concurrent.futures
+import contextlib
+import os
+
+import iamraw
+import serializeraw
+import utilo
+
+import protoerror
+
+
+def write_grouped(
+    findings: iamraw.Findings,
+    dest: str,
+    overwrite: bool = True,
+    private: bool = False,
+) -> list:
+    result = []
+    grouped = protoerror.bypage(findings)
+    writer = utilo.file_replace if overwrite else utilo.file_create
+    for item in grouped:
+        page = fname(item.page)
+        outpath = os.path.join(dest, page)
+        dumped = serializeraw.dump_findings(item.content)
+        writer(outpath, dumped, private=private)
+        result.append(outpath)
+    return result
+
+
+def load_grouped(
+    source: str,
+    pages: tuple = None,
+    sort: bool = True,
+    worker=10,
+) -> iamraw.PageFindings:
+    if isinstance(pages, int):
+        pages = (pages,)
+    if pages is None:
+        # load all findings
+        pages = [
+            pagenumber(item, none=True) for item in utilo.file_list(source)
+        ]
+        # remove invalid file names
+        pages = utilo.notnone(pages)
+    # yaml parsing is cpu bound, therefore we need a process pool instead
+    # of thread pool.
+    executor = utilo.select_executor()
+    result = []
+    with executor(max_workers=worker) as executor:
+        todo = {
+            executor.submit(load_findings, source, page): page for page in pages
+        }
+        for job in concurrent.futures.as_completed(todo):
+            data = job.result()
+            if not data:
+                continue
+            result.append(data)
+    if sort:
+        result.sort(key=lambda x: x.page)
+    return result
+
+
+def load_findings(source: str, page: int) -> iamraw.PageFinding:
+    name = fname(page)
+    source = os.path.join(source, name)
+    if not os.path.exists(source):
+        return None
+    findings = serializeraw.load_findings(source)
+    return iamraw.PageFinding(page=page, content=findings)
+
+
+def fname(page: int) -> str:
+    """\
+    >>> fname(-5)
+    '_05'
+    >>> fname(1)
+    '001'
+    >>> fname(333)
+    '333'
+    """
+    if page < 0:
+        return '_' + f'{page*-1}'.zfill(2)
+    return f'{page}'.zfill(3)
+
+
+def pagenumber(page: str, none: bool = True) -> int:
+    """\
+    >>> pagenumber('010')
+    10
+    >>> pagenumber(fname(-10))
+    -10
+    """
+    if page[0] == '_':
+        page = '-' + page[1:]
+    with contextlib.suppress(ValueError):
+        return int(page)
+    if none:
+        # handle error case
+        return None
+    raise ValueError(f'could not convert to int: {page}')