omdev 0.0.0.dev313__py3-none-any.whl → 0.0.0.dev314__py3-none-any.whl

omdev/py/docstrings/numpydoc.py

@@ -0,0 +1,403 @@
+"""
+Numpydoc-style docstring parsing.
+
+:see: https://numpydoc.readthedocs.io/en/latest/format.html
+"""
+import abc
+import inspect
+import itertools
+import re
+import textwrap
+import typing as ta
+
+from .common import Docstring
+from .common import DocstringDeprecated
+from .common import DocstringExample
+from .common import DocstringMeta
+from .common import DocstringParam
+from .common import DocstringRaises
+from .common import DocstringReturns
+from .common import DocstringStyle
+from .common import ParseError
+
+
+##
+
+
+def _pairwise(iterable: ta.Iterable, end=None) -> ta.Iterable:
+    left, right = itertools.tee(iterable)
+    next(right, None)
+    return itertools.zip_longest(left, right, fillvalue=end)
+
+
+def _clean_str(string: str) -> str | None:
+    string = string.strip()
+    if len(string) > 0:
+        return string
+    return None
+
+
+KV_PAT = re.compile(r'^[^\s].*$', flags=re.MULTILINE)
+PARAM_KEY_PAT = re.compile(r'^(?P<name>.*?)(?:\s*:\s*(?P<type>.*?))?$')
+PARAM_OPTIONAL_PAT = re.compile(r'(?P<type>.*?)(?:, optional|\(optional\))$')
+
+# numpydoc format has no formal grammar for this, but we can make some educated guesses...
+PARAM_DEFAULT_PAT = re.compile(r'(?<!\S)[Dd]efault(?: is | = |: |s to |)\s*(?P<value>[\w\-\.]*\w)')
+
+RETURN_KEY_PAT = re.compile(r'^(?:(?P<name>.*?)\s*:\s*)?(?P<type>.*?)$')
+
+
+class Section:
+    """
+    Numpydoc section parser.
+
+    :param title: section title. For most sections, this is a heading like "Parameters" which appears on its own line,
+        underlined by en-dashes ('-') on the following line.
+    :param key: meta key string. In the parsed ``DocstringMeta`` instance this will be the first element of the ``args``
+        attribute list.
+    """
+
+    def __init__(self, title: str, key: str) -> None:
+        super().__init__()
+
+        self.title = title
+        self.key = key
+
+    @property
+    def title_pattern(self) -> str:
+        """
+        Regular expression pattern matching this section's header.
+
+        This pattern will match this instance's ``title`` attribute in an anonymous group.
+        """
+
+        dashes = '-' * len(self.title)
+        return rf'^({self.title})\s*?\n{dashes}\s*$'
+
+    def parse(self, text: str) -> ta.Iterable[DocstringMeta]:
+        """
+        Parse ``DocstringMeta`` objects from the body of this section.
+
+        :param text: section body text. Should be cleaned with ``inspect.cleandoc`` before parsing.
+        """
+
+        yield DocstringMeta([self.key], description=_clean_str(text))
+
+
+class _KVSection(Section, abc.ABC):
+    """
+    Base parser for numpydoc sections with key-value syntax.
+
+    E.g. sections that look like this:
+        key
+            value
+        key2 : type
+            values can also span...
+            ... multiple lines
+    """
+
+    @abc.abstractmethod
+    def _parse_item(self, key: str, value: str) -> DocstringMeta:
+        raise NotImplementedError
+
+    def parse(self, text: str) -> ta.Iterable[DocstringMeta]:
+        for match, next_match in _pairwise(KV_PAT.finditer(text)):
+            start = match.end()
+            end = next_match.start() if next_match is not None else None
+            value = text[start:end]
+            yield self._parse_item(
+                key=match.group(),
+                value=inspect.cleandoc(value),
+            )
+
+
+class _SphinxSection(Section):
+    """
+    Base parser for numpydoc sections with sphinx-style syntax.
+
+    E.g. sections that look like this:
+        .. title:: something
+            possibly over multiple lines
+    """
+
+    @property
+    def title_pattern(self) -> str:
+        return rf'^\.\.\s*({self.title})\s*::'
+
+
+class ParamSection(_KVSection):
+    """
+    Parser for numpydoc parameter sections.
+
+    E.g. any section that looks like this:
+        arg_name
+            arg_description
+        arg_2 : type, optional
+            descriptions can also span...
+            ... multiple lines
+    """
+
+    def _parse_item(self, key: str, value: str) -> DocstringParam:
+        match = PARAM_KEY_PAT.match(key)
+        arg_name = type_name = is_optional = None
+        if match is not None:
+            arg_name = match.group('name')
+            type_name = match.group('type')
+            if type_name is not None:
+                optional_match = PARAM_OPTIONAL_PAT.match(type_name)
+                if optional_match is not None:
+                    type_name = optional_match.group('type')
+                    is_optional = True
+                else:
+                    is_optional = False
+
+        default = None
+        if len(value) > 0:
+            default_match = PARAM_DEFAULT_PAT.search(value)
+            if default_match is not None:
+                default = default_match.group('value')
+
+        if arg_name is None:
+            raise ParseError
+
+        return DocstringParam(
+            args=[self.key, arg_name],
+            description=_clean_str(value),
+            arg_name=arg_name,
+            type_name=type_name,
+            is_optional=is_optional,
+            default=default,
+        )
+
+
+class RaisesSection(_KVSection):
+    """
+    Parser for numpydoc raises sections.
+
+    E.g. any section that looks like this:
+        ValueError
+            A description of what might raise ValueError
+    """
+
+    def _parse_item(self, key: str, value: str) -> DocstringRaises:
+        return DocstringRaises(
+            args=[self.key, key],
+            description=_clean_str(value),
+            type_name=key if len(key) > 0 else None,
+        )
+
+
+class ReturnsSection(_KVSection):
+    """
+    Parser for numpydoc returns sections.
+
+    E.g. any section that looks like this:
+        return_name : type
+            A description of this returned value
+        another_type
+            Return names are optional, types are required
+    """
+
+    is_generator = False
+
+    def _parse_item(self, key: str, value: str) -> DocstringReturns:
+        match = RETURN_KEY_PAT.match(key)
+        if match is not None:
+            return_name = match.group('name')
+            type_name = match.group('type')
+        else:
+            return_name = None
+            type_name = None
+
+        return DocstringReturns(
+            args=[self.key],
+            description=_clean_str(value),
+            type_name=type_name,
+            is_generator=self.is_generator,
+            return_name=return_name,
+        )
+
+
+class YieldsSection(ReturnsSection):
+    """Parser for numpydoc generator "yields" sections."""
+
+    is_generator = True
+
+
+class DeprecationSection(_SphinxSection):
+    """Parser for numpydoc "deprecation warning" sections."""
+
+    def parse(self, text: str) -> ta.Iterable[DocstringDeprecated]:
+        version, desc, *_ = [*text.split(sep='\n', maxsplit=1), None, None]
+
+        if desc is not None:
+            desc = _clean_str(inspect.cleandoc(desc))  # type: ignore[unreachable]
+
+        yield DocstringDeprecated(
+            args=[self.key],
+            description=desc,
+            version=_clean_str(version),
+        )
+
+
+class ExamplesSection(Section):
+    """
+    Parser for numpydoc examples sections.
+
+    E.g. any section that looks like this:
+        >>> import numpy.matlib
+        >>> np.matlib.empty((2, 2))    # filled with random data
+        matrix([[  6.76425276e-320,   9.79033856e-307], # random
+                [  7.39337286e-309,   3.22135945e-309]])
+        >>> np.matlib.empty((2, 2), dtype=int)
+        matrix([[ 6600475,        0], # random
+                [ 6586976, 22740995]])
+    """
+
+    def parse(self, text: str) -> ta.Iterable[DocstringMeta]:
+        """
+        Parse ``DocstringExample`` objects from the body of this section.
+
+        :param text: section body text. Should be cleaned with ``inspect.cleandoc`` before parsing.
+        """
+
+        lines = textwrap.dedent(text).strip().splitlines()
+        while lines:
+            snippet_lines = []
+            description_lines = []
+            while lines:
+                if not lines[0].startswith('>>>'):
+                    break
+                snippet_lines.append(lines.pop(0))
+            while lines:
+                if lines[0].startswith('>>>'):
+                    break
+                description_lines.append(lines.pop(0))
+            yield DocstringExample(
+                [self.key],
+                snippet='\n'.join(snippet_lines) if snippet_lines else None,
+                description='\n'.join(description_lines),
+            )
+
+
+DEFAULT_SECTIONS = [
+    ParamSection('Parameters', 'param'),
+    ParamSection('Params', 'param'),
+    ParamSection('Arguments', 'param'),
+    ParamSection('Args', 'param'),
+    ParamSection('Other Parameters', 'other_param'),
+    ParamSection('Other Params', 'other_param'),
+    ParamSection('Other Arguments', 'other_param'),
+    ParamSection('Other Args', 'other_param'),
+    ParamSection('Receives', 'receives'),
+    ParamSection('Receive', 'receives'),
+    RaisesSection('Raises', 'raises'),
+    RaisesSection('Raise', 'raises'),
+    RaisesSection('Warns', 'warns'),
+    RaisesSection('Warn', 'warns'),
+    ParamSection('Attributes', 'attribute'),
+    ParamSection('Attribute', 'attribute'),
+    ReturnsSection('Returns', 'returns'),
+    ReturnsSection('Return', 'returns'),
+    YieldsSection('Yields', 'yields'),
+    YieldsSection('Yield', 'yields'),
+    ExamplesSection('Examples', 'examples'),
+    ExamplesSection('Example', 'examples'),
+    Section('Warnings', 'warnings'),
+    Section('Warning', 'warnings'),
+    Section('See Also', 'see_also'),
+    Section('Related', 'see_also'),
+    Section('Notes', 'notes'),
+    Section('Note', 'notes'),
+    Section('References', 'references'),
+    Section('Reference', 'references'),
+    DeprecationSection('deprecated', 'deprecation'),
+]
+
+
+class NumpydocParser:
+    """Parser for numpydoc-style docstrings."""
+
+    def __init__(self, sections: ta.Iterable[Section] | None = None) -> None:
+        """
+        Setup sections.
+
+        :param sections: Recognized sections or None to defaults.
+        """
+
+        super().__init__()
+
+        sections = sections or DEFAULT_SECTIONS
+        self.sections = {s.title: s for s in sections}
+        self._setup()
+
+    def _setup(self):
+        self.titles_re = re.compile(
+            r'|'.join(s.title_pattern for s in self.sections.values()),
+            flags=re.MULTILINE,
+        )
+
+    def add_section(self, section: Section) -> None:
+        """
+        Add or replace a section.
+
+        :param section: The new section.
+        """
+
+        self.sections[section.title] = section
+        self._setup()
+
+    def parse(self, text: str) -> Docstring:
+        """
+        Parse the numpy-style docstring into its components.
+
+        :returns: parsed docstring
+        """
+
+        ret = Docstring(style=DocstringStyle.NUMPYDOC)
+        if not text:
+            return ret
+
+        # Clean according to PEP-0257
+        text = inspect.cleandoc(text)
+
+        # Find first title and split on its position
+        match = self.titles_re.search(text)
+        if match:
+            desc_chunk = text[: match.start()]
+            meta_chunk = text[match.start():]
+        else:
+            desc_chunk = text
+            meta_chunk = ''
+
+        # Break description into short and long parts
+        parts = desc_chunk.split('\n', 1)
+        ret.short_description = parts[0] or None
+        if len(parts) > 1:
+            long_desc_chunk = parts[1] or ''
+            ret.blank_after_short_description = long_desc_chunk.startswith('\n')
+            ret.blank_after_long_description = long_desc_chunk.endswith('\n\n')
+            ret.long_description = long_desc_chunk.strip() or None
+
+        for match, nextmatch in _pairwise(self.titles_re.finditer(meta_chunk)):
+            if match is None:
+                raise ParseError
+            title = next(g for g in match.groups() if g is not None)
+            factory = self.sections[title]
+
+            # section chunk starts after the header, ends at the start of the next header
+            start = match.end()
+            end = nextmatch.start() if nextmatch is not None else None
+            ret.meta.extend(factory.parse(meta_chunk[start:end]))
+
+        return ret
+
+
+def parse(text: str) -> Docstring:
+    """
+    Parse the numpy-style docstring into its components.
+
+    :returns: parsed docstring
+    """
+
+    return NumpydocParser().parse(text)

omdev/py/docstrings/parser.py

@@ -0,0 +1,84 @@
+"""The main parsing routine."""
+import inspect
+import typing as ta
+
+from . import epydoc
+from . import google
+from . import numpydoc
+from . import rest
+from .attrdoc import add_attribute_docstrings
+from .common import Docstring
+from .common import DocstringStyle
+from .common import ParseError
+
+
+##
+
+
+_STYLE_MAP = {
+    DocstringStyle.REST: rest,
+    DocstringStyle.GOOGLE: google,
+    DocstringStyle.NUMPYDOC: numpydoc,
+    DocstringStyle.EPYDOC: epydoc,
+}
+
+
+def parse(
+        text: str,
+        style: DocstringStyle = DocstringStyle.AUTO,
+) -> Docstring:
+    """
+    Parse the docstring into its components.
+
+    :param text: docstring text to parse
+    :param style: docstring style
+    :returns: parsed docstring representation
+    """
+
+    if style != DocstringStyle.AUTO:
+        return _STYLE_MAP[style].parse(text)
+
+    exc: Exception | type[Exception] = ParseError
+    rets = []
+    for module in _STYLE_MAP.values():
+        try:
+            ret = module.parse(text)
+        except ParseError as ex:
+            exc = ex
+        else:
+            rets.append(ret)
+
+    if not rets:
+        raise exc
+
+    return sorted(rets, key=lambda d: len(d.meta), reverse=True)[0]
+
+
+def parse_from_object(
+        obj: ta.Any,
+        style: DocstringStyle = DocstringStyle.AUTO,
+) -> Docstring:
+    """
+    Parse the object's docstring(s) into its components.
+
+    The object can be anything that has a ``__doc__`` attribute. In contrast to the ``parse`` function,
+    ``parse_from_object`` is able to parse attribute docstrings which are defined in the source code instead of
+    ``__doc__``.
+
+    Currently only attribute docstrings defined at class and module levels are supported. Attribute docstrings defined
+    in ``__init__`` methods are not supported.
+
+    When given a class, only the attribute docstrings of that class are parsed, not its inherited classes. This is a
+    design decision. Separate calls to this function should be performed to get attribute docstrings of parent classes.
+
+    :param obj: object from which to parse the docstring(s)
+    :param style: docstring style
+    :returns: parsed docstring representation
+    """
+
+    docstring = parse(obj.__doc__, style=style)
+
+    if inspect.isclass(obj) or inspect.ismodule(obj):
+        add_attribute_docstrings(obj, docstring)
+
+    return docstring

omdev/py/docstrings/rest.py

@@ -0,0 +1,176 @@
+"""ReST-style docstring parsing."""
+import inspect
+import re
+
+from .common import DEPRECATION_KEYWORDS
+from .common import PARAM_KEYWORDS
+from .common import RAISES_KEYWORDS
+from .common import RETURNS_KEYWORDS
+from .common import YIELDS_KEYWORDS
+from .common import Docstring
+from .common import DocstringDeprecated
+from .common import DocstringMeta
+from .common import DocstringParam
+from .common import DocstringRaises
+from .common import DocstringReturns
+from .common import DocstringStyle
+from .common import ParseError
+
+
+##
+
+
+def _build_meta(args: list[str], desc: str) -> DocstringMeta:
+    key = args[0]
+
+    if key in PARAM_KEYWORDS:
+        if len(args) == 3:
+            key, type_name, arg_name = args
+            if type_name.endswith('?'):
+                is_optional = True
+                type_name = type_name[:-1]
+            else:
+                is_optional = False
+
+        elif len(args) == 2:
+            key, arg_name = args
+            type_name = None
+            is_optional = None
+
+        else:
+            raise ParseError(f'Expected one or two arguments for a {key} keyword.')
+
+        match = re.match(r'.*defaults to (.+)', desc, flags=re.DOTALL)
+        default = match.group(1).rstrip('.') if match else None
+
+        return DocstringParam(
+            args=args,
+            description=desc,
+            arg_name=arg_name,
+            type_name=type_name,
+            is_optional=is_optional,
+            default=default,
+        )
+
+    if key in RETURNS_KEYWORDS | YIELDS_KEYWORDS:
+        if len(args) == 2:
+            type_name = args[1]
+        elif len(args) == 1:
+            type_name = None
+        else:
+            raise ParseError(f'Expected one or no arguments for a {key} keyword.')
+
+        return DocstringReturns(
+            args=args,
+            description=desc,
+            type_name=type_name,
+            is_generator=key in YIELDS_KEYWORDS,
+        )
+
+    if key in DEPRECATION_KEYWORDS:
+        match = re.search(
+            r'^(?P<version>v?((?:\d+)(?:\.[0-9a-z\.]+))) (?P<desc>.+)',
+            desc,
+            flags=re.IGNORECASE,
+        )
+
+        return DocstringDeprecated(
+            args=args,
+            version=match.group('version') if match else None,
+            description=match.group('desc') if match else desc,
+        )
+
+    if key in RAISES_KEYWORDS:
+        if len(args) == 2:
+            type_name = args[1]
+        elif len(args) == 1:
+            type_name = None
+        else:
+            raise ParseError(f'Expected one or no arguments for a {key} keyword.')
+
+        return DocstringRaises(
+            args=args,
+            description=desc,
+            type_name=type_name,
+        )
+
+    return DocstringMeta(args=args, description=desc)
+
+
+def parse(text: str) -> Docstring:
+    """
+    Parse the ReST-style docstring into its components.
+
+    :returns: parsed docstring
+    """
+
+    ret = Docstring(style=DocstringStyle.REST)
+    if not text:
+        return ret
+
+    text = inspect.cleandoc(text)
+    match = re.search('^:', text, flags=re.MULTILINE)
+    if match:
+        desc_chunk = text[: match.start()]
+        meta_chunk = text[match.start():]
+    else:
+        desc_chunk = text
+        meta_chunk = ''
+
+    parts = desc_chunk.split('\n', 1)
+    ret.short_description = parts[0] or None
+    if len(parts) > 1:
+        long_desc_chunk = parts[1] or ''
+        ret.blank_after_short_description = long_desc_chunk.startswith('\n')
+        ret.blank_after_long_description = long_desc_chunk.endswith('\n\n')
+        ret.long_description = long_desc_chunk.strip() or None
+
+    types = {}
+    rtypes = {}
+    for match in re.finditer(r'(^:.*?)(?=^:|\Z)', meta_chunk, flags=re.DOTALL | re.MULTILINE):
+        chunk = match.group(0)
+        if not chunk:
+            continue
+
+        try:
+            args_chunk, desc_chunk = chunk.lstrip(':').split(':', 1)
+        except ValueError as ex:
+            raise ParseError(f'Error parsing meta information near "{chunk}".') from ex
+
+        args = args_chunk.split()
+        desc = desc_chunk.strip()
+
+        if '\n' in desc:
+            first_line, rest = desc.split('\n', 1)
+            desc = first_line + '\n' + inspect.cleandoc(rest)
+
+        # Add special handling for :type a: typename
+        if len(args) == 2 and args[0] == 'type':
+            types[args[1]] = desc
+
+        elif len(args) in [1, 2] and args[0] == 'rtype':
+            rtypes[None if len(args) == 1 else args[1]] = desc
+
+        else:
+            ret.meta.append(_build_meta(args, desc))
+
+    for meta in ret.meta:
+        if isinstance(meta, DocstringParam):
+            meta.type_name = meta.type_name or types.get(meta.arg_name)
+
+        elif isinstance(meta, DocstringReturns):
+            meta.type_name = meta.type_name or rtypes.get(meta.return_name)
+
+    if not any(isinstance(m, DocstringReturns) for m in ret.meta) and rtypes:
+        for return_name, type_name in rtypes.items():
+            ret.meta.append(
+                DocstringReturns(
+                    args=[],
+                    type_name=type_name,
+                    description=None,
+                    is_generator=False,
+                    return_name=return_name,
+                ),
+            )
+
+    return ret

omdev/scripts/pyproject.py

@@ -205,14 +205,21 @@ class MagicStyle:
 
 PY_MAGIC_STYLE = MagicStyle(
     name='py',
-    exts=frozenset(['py']),
+    exts=frozenset([
+        'py',
+    ]),
     line_prefix='# ',
 )
 
 
 C_MAGIC_STYLE = MagicStyle(
     name='c',
-    exts=frozenset(['c', 'cc', 'cpp', 'cu']),
+    exts=frozenset([
+        'c',
+        'cc',
+        'cpp',
+        'cu',
+    ]),
     line_prefix='// ',
     block_prefix_suffix=('/* ', '*/'),
 )

omdev/tools/antlr/gen.py

@@ -133,6 +133,7 @@ class GenPy:
             '# type: ignore\n',
             '# ruff: noqa\n',
             '# flake8: noqa\n',
+            '# @omlish-generated\n',
         ]
 
         for l in in_lines: