Cython 3.1.0__py3-none-any.whl

Cython/Compiler/AnalysedTreeTransforms.py

@@ -0,0 +1,97 @@
+from .Visitor import ScopeTrackingTransform
+from .Nodes import StatListNode, SingleAssignmentNode, CFuncDefNode, DefNode
+from .ExprNodes import DictNode, DictItemNode, NameNode, UnicodeNode
+from .PyrexTypes import py_object_type
+from .StringEncoding import EncodedString
+from . import Symtab
+
+class AutoTestDictTransform(ScopeTrackingTransform):
+    # Handles autotestdict directive
+
+    excludelist = ['__cinit__', '__dealloc__', '__richcmp__',
+                   '__nonzero__', '__bool__',
+                   '__len__', '__contains__']
+
+    def visit_ModuleNode(self, node):
+        if node.is_pxd:
+            return node
+        self.scope_type = 'module'
+        self.scope_node = node
+
+        if not self.current_directives['autotestdict']:
+            return node
+        self.all_docstrings = self.current_directives['autotestdict.all']
+        self.cdef_docstrings = self.all_docstrings or self.current_directives['autotestdict.cdef']
+
+        assert isinstance(node.body, StatListNode)
+
+        # First see if __test__ is already created
+        if '__test__' in node.scope.entries:
+            # Do nothing
+            return node
+
+        pos = node.pos
+
+        self.tests = []
+        self.testspos = node.pos
+
+        test_dict_entry = node.scope.declare_var(EncodedString('__test__'),
+                                                 py_object_type,
+                                                 pos,
+                                                 visibility='public')
+        create_test_dict_assignment = SingleAssignmentNode(pos,
+            lhs=NameNode(pos, name=EncodedString('__test__'),
+                         entry=test_dict_entry),
+            rhs=DictNode(pos, key_value_pairs=self.tests))
+        self.visitchildren(node)
+        node.body.stats.append(create_test_dict_assignment)
+        return node
+
+    def add_test(self, testpos, path, doctest):
+        pos = self.testspos
+        keystr = EncodedString(f'{path} (line {testpos[1]:d})')
+        key = UnicodeNode(pos, value=keystr)
+        value = UnicodeNode(pos, value=doctest)
+        self.tests.append(DictItemNode(pos, key=key, value=value))
+
+    def visit_ExprNode(self, node):
+        # expressions cannot contain functions and lambda expressions
+        # do not have a docstring
+        return node
+
+    def visit_FuncDefNode(self, node):
+        if not node.doc or (isinstance(node, DefNode) and node.fused_py_func):
+            return node
+        if not self.cdef_docstrings:
+            if isinstance(node, CFuncDefNode) and not node.py_func:
+                return node
+        if not self.all_docstrings and '>>>' not in node.doc:
+            return node
+
+        pos = self.testspos
+        if self.scope_type == 'module':
+            path = node.entry.name
+        elif self.scope_type in ('pyclass', 'cclass'):
+            if isinstance(node, CFuncDefNode):
+                if node.py_func is not None:
+                    name = node.py_func.name
+                else:
+                    name = node.entry.name
+            else:
+                name = node.name
+            if self.scope_type == 'cclass' and name in self.excludelist:
+                return node
+            if self.scope_type == 'pyclass':
+                class_name = self.scope_node.name
+            else:
+                class_name = self.scope_node.class_name
+            if isinstance(node.entry.scope, Symtab.PropertyScope):
+                property_method_name = node.entry.scope.name
+                path = "%s.%s.%s" % (class_name, node.entry.scope.name,
+                                     node.entry.name)
+            else:
+                path = "%s.%s" % (class_name, node.entry.name)
+        else:
+            assert False
+        self.add_test(node.pos, path, node.doc)
+        return node

Cython/Compiler/Annotate.py

@@ -0,0 +1,326 @@
+# Note: Work in progress
+
+
+import os
+import os.path
+import re
+import codecs
+import textwrap
+from datetime import datetime
+from functools import partial
+from collections import defaultdict
+from xml.sax.saxutils import escape as html_escape
+from io import StringIO
+
+from . import Version
+from .Code import CCodeWriter
+from .. import Utils
+
+
+class AnnotationCCodeWriter(CCodeWriter):
+
+    # also used as marker for detection of complete code emission in tests
+    COMPLETE_CODE_TITLE = "Complete cythonized code"
+
+    def __init__(self, create_from=None, buffer=None, copy_formatting=True, show_entire_c_code=False, source_desc=None):
+        CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting)
+        self.show_entire_c_code = show_entire_c_code
+        if create_from is None:
+            self.annotation_buffer = StringIO()
+            self.last_annotated_pos = None
+            # annotations[filename][line] -> [(column, AnnotationItem)*]
+            self.annotations = defaultdict(partial(defaultdict, list))
+            # code[filename][line] -> str
+            self.code = defaultdict(partial(defaultdict, str))
+            # scopes[filename][line] -> set(scopes)
+            self.scopes = defaultdict(partial(defaultdict, set))
+        else:
+            # When creating an insertion point, keep references to the same database
+            self.annotation_buffer = create_from.annotation_buffer
+            self.annotations = create_from.annotations
+            self.code = create_from.code
+            self.scopes = create_from.scopes
+            self.last_annotated_pos = create_from.last_annotated_pos
+
+    def create_new(self, create_from, buffer, copy_formatting):
+        return AnnotationCCodeWriter(create_from, buffer, copy_formatting)
+
+    def _write_to_buffer(self, s):
+        self.buffer.write(s)
+        self.annotation_buffer.write(s)
+
+    def mark_pos(self, pos, trace=True):
+        if pos is not None:
+            CCodeWriter.mark_pos(self, pos, trace)
+            if self.funcstate and self.funcstate.scope:
+                # lambdas and genexprs can result in multiple scopes per line => keep them in a set
+                self.scopes[pos[0].filename][pos[1]].add(self.funcstate.scope)
+        if self.last_annotated_pos:
+            source_desc, line, _ = self.last_annotated_pos
+            pos_code = self.code[source_desc.filename]
+            pos_code[line] += self.annotation_buffer.getvalue()
+        self.annotation_buffer = StringIO()
+        self.last_annotated_pos = pos
+
+    def annotate(self, pos, item):
+        self.annotations[pos[0].filename][pos[1]].append((pos[2], item))
+
+    def _css(self):
+        """css template will later allow to choose a colormap"""
+        css = [self._css_template]
+        for i in range(255):
+            color_shade = int(255.0 // (1.0 + i/10.0))
+            css.append(f'.cython.score-{i:d} {{background-color: #FFFF{color_shade:02x};}}')
+        try:
+            from pygments.formatters import HtmlFormatter
+        except ImportError:
+            pass
+        else:
+            css.append(HtmlFormatter().get_style_defs('.cython'))
+        return '\n'.join(css)
+
+    _css_template = textwrap.dedent("""
+        body.cython { font-family: courier; font-size: 12; }
+
+        .cython.tag  {  }
+        .cython.line { color: #000000; margin: 0em }
+        .cython.code { font-size: 9; color: #444444; display: none; margin: 0px 0px 0px 8px; border-left: 8px none; }
+
+        .cython.line .run { background-color: #B0FFB0; }
+        .cython.line .mis { background-color: #FFB0B0; }
+        .cython.code.run  { border-left: 8px solid #B0FFB0; }
+        .cython.code.mis  { border-left: 8px solid #FFB0B0; }
+
+        .cython.code .py_c_api  { color: red; }
+        .cython.code .py_macro_api  { color: #FF7000; }
+        .cython.code .pyx_c_api  { color: #FF3000; }
+        .cython.code .pyx_macro_api  { color: #FF7000; }
+        .cython.code .refnanny  { color: #FFA000; }
+        .cython.code .trace  { color: #FFA000; }
+        .cython.code .error_goto  { color: #FFA000; }
+
+        .cython.code .coerce  { color: #008000; border: 1px dotted #008000 }
+        .cython.code .py_attr { color: #FF0000; font-weight: bold; }
+        .cython.code .c_attr  { color: #0000FF; }
+        .cython.code .py_call { color: #FF0000; font-weight: bold; }
+        .cython.code .c_call  { color: #0000FF; }
+    """)
+
+    # on-click toggle function to show/hide C source code
+    _onclick_attr = ' onclick="{}"'.format((
+        "(function(s){"
+        "    s.display =  s.display === 'block' ? 'none' : 'block'"
+        "})(this.nextElementSibling.style)"
+        ).replace(' ', '')  # poor dev's JS minification
+    )
+
+    def save_annotation(self, source_filename, target_filename, coverage_xml=None):
+        with Utils.open_source_file(source_filename) as f:
+            code = f.read()
+        generated_code = self.code.get(source_filename, {})
+        c_file = Utils.decode_filename(os.path.basename(target_filename))
+        html_filename = os.path.splitext(target_filename)[0] + ".html"
+
+        with codecs.open(html_filename, "w", encoding="UTF-8") as out_buffer:
+            out_buffer.write(self._save_annotation(code, generated_code, c_file, source_filename, coverage_xml))
+
+    def _save_annotation_header(self, c_file, source_filename, coverage_timestamp=None):
+        coverage_info = ''
+        if coverage_timestamp:
+            coverage_info = ' with coverage data from {timestamp}'.format(
+                timestamp=datetime.fromtimestamp(int(coverage_timestamp) // 1000))
+
+        outlist = [
+            textwrap.dedent('''\
+            <!DOCTYPE html>
+            <!-- Generated by Cython {watermark} -->
+            <html>
+            <head>
+                <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+                <title>Cython: {filename}</title>
+                <style type="text/css">
+                {css}
+                </style>
+            </head>
+            <body class="cython">
+            <p><span style="border-bottom: solid 1px grey;">Generated by Cython {watermark}</span>{more_info}</p>
+            <p>
+                <span style="background-color: #FFFF00">Yellow lines</span> hint at Python interaction.<br />
+                Click on a line that starts with a "<code>+</code>" to see the C code that Cython generated for it.
+            </p>
+            ''').format(css=self._css(), watermark=Version.watermark,
+                        filename=os.path.basename(source_filename) if source_filename else '',
+                        more_info=coverage_info)
+        ]
+        if c_file:
+            outlist.append('<p>Raw output: <a href="%s">%s</a></p>\n' % (c_file, c_file))
+        return outlist
+
+    def _save_annotation_footer(self):
+        return ('</body></html>\n',)
+
+    def _save_annotation(self, code, generated_code, c_file=None, source_filename=None, coverage_xml=None):
+        """
+        lines : original cython source code split by lines
+        generated_code : generated c code keyed by line number in original file
+        target filename : name of the file in which to store the generated html
+        c_file : filename in which the c_code has been written
+        """
+        if coverage_xml is not None and source_filename:
+            coverage_timestamp = coverage_xml.get('timestamp', '').strip()
+            covered_lines = self._get_line_coverage(coverage_xml, source_filename)
+        else:
+            coverage_timestamp = covered_lines = None
+        annotation_items = dict(self.annotations[source_filename])
+        scopes = dict(self.scopes[source_filename])
+
+        outlist = []
+        outlist.extend(self._save_annotation_header(c_file, source_filename, coverage_timestamp))
+        outlist.extend(self._save_annotation_body(code, generated_code, annotation_items, scopes, covered_lines))
+        outlist.extend(self._save_annotation_footer())
+        return ''.join(outlist)
+
+    def _get_line_coverage(self, coverage_xml, source_filename):
+        coverage_data = None
+        for entry in coverage_xml.iterfind('.//class'):
+            if not entry.get('filename'):
+                continue
+            if (entry.get('filename') == source_filename or
+                    os.path.abspath(entry.get('filename')) == source_filename):
+                coverage_data = entry
+                break
+            elif source_filename.endswith(entry.get('filename')):
+                coverage_data = entry  # but we might still find a better match...
+        if coverage_data is None:
+            return None
+        return {
+            int(line.get('number')): int(line.get('hits'))
+            for line in coverage_data.iterfind('lines/line')
+        }
+
+    def _htmlify_code(self, code, language):
+        try:
+            from pygments import highlight
+            from pygments.lexers import CythonLexer, CppLexer
+            from pygments.formatters import HtmlFormatter
+        except ImportError:
+            # no Pygments, just escape the code
+            return html_escape(code)
+
+        if language == "cython":
+            lexer = CythonLexer(stripnl=False, stripall=False)
+        elif language == "c/cpp":
+            lexer = CppLexer(stripnl=False, stripall=False)
+        else:
+            # unknown language, use fallback
+            return html_escape(code)
+        html_code = highlight(
+            code, lexer,
+            HtmlFormatter(nowrap=True))
+        return html_code
+
+    def _save_annotation_body(self, cython_code, generated_code, annotation_items, scopes, covered_lines=None):
+        outlist = ['<div class="cython">']
+        pos_comment_marker = '/* \N{HORIZONTAL ELLIPSIS} */\n'
+        new_calls_map = {
+            name: 0 for name in
+            'refnanny trace py_macro_api py_c_api pyx_macro_api pyx_c_api error_goto'.split()
+        }.copy
+
+        self.mark_pos(None)
+
+        def annotate(match):
+            group_name = match.lastgroup
+            calls[group_name] += 1
+            return f"<span class='{group_name}'>{match.group(group_name)}</span>"
+
+        lines = self._htmlify_code(cython_code, "cython").splitlines()
+        lineno_width = len(str(len(lines)))
+        if not covered_lines:
+            covered_lines = None
+
+        for k, line in enumerate(lines, 1):
+            try:
+                c_code = generated_code[k]
+            except KeyError:
+                c_code = ''
+            else:
+                c_code = _replace_pos_comment(pos_comment_marker, c_code)
+                if c_code.startswith(pos_comment_marker):
+                    c_code = c_code[len(pos_comment_marker):]
+                c_code = html_escape(c_code)
+
+            calls = new_calls_map()
+            c_code = _parse_code(annotate, c_code)
+            score = (5 * calls['py_c_api'] + 2 * calls['pyx_c_api'] +
+                     calls['py_macro_api'] + calls['pyx_macro_api'])
+
+            if c_code:
+                onclick = self._onclick_attr
+                expandsymbol = '+'
+            else:
+                onclick = ''
+                expandsymbol = '&#xA0;'
+
+            covered = ''
+            if covered_lines is not None and k in covered_lines:
+                hits = covered_lines[k]
+                if hits is not None:
+                    covered = 'run' if hits else 'mis'
+
+            outlist.append(
+                f'<pre class="cython line score-{score}"{onclick}>'
+                # generate line number with expand symbol in front,
+                # and the right  number of digit
+                f'{expandsymbol}<span class="{covered}">{k:0{lineno_width}d}</span>: {line.rstrip()}</pre>\n'
+            )
+            if c_code:
+                outlist.append(f"<pre class='cython code score-{score} {covered}'>{c_code}</pre>")
+        outlist.append("</div>")
+
+        # now the whole c-code if needed:
+        if self.show_entire_c_code:
+            complete_code_as_html = self._htmlify_code(self.buffer.getvalue(), "c/cpp")
+            outlist.append(
+                '<p><div class="cython">'
+                f"<pre class='cython line'{self._onclick_attr}>+ {AnnotationCCodeWriter.COMPLETE_CODE_TITLE}</pre>\n"
+                f"<pre class='cython code'>{complete_code_as_html}</pre>"
+                "</div></p>"
+            )
+
+        return outlist
+
+
+_parse_code = re.compile((
+    br'(?P<refnanny>__Pyx_X?(?:GOT|GIVE)REF|__Pyx_RefNanny[A-Za-z]+)|'
+    br'(?P<trace>__Pyx_Trace[A-Za-z]+)|'
+    br'(?:'
+    br'(?P<pyx_macro_api>__Pyx_[A-Z][A-Z_]+)|'
+    br'(?P<pyx_c_api>(?:__Pyx_[A-Z][a-z_][A-Za-z_]*)|__pyx_convert_[A-Za-z_]*)|'
+    br'(?P<py_macro_api>Py[A-Z][a-z]+_[A-Z][A-Z_]+)|'
+    br'(?P<py_c_api>Py[A-Z][a-z]+_[A-Z][a-z][A-Za-z_]*)'
+    br')(?=\()|'       # look-ahead to exclude subsequent '(' from replacement
+    br'(?P<error_goto>(?:(?<=;) *if [^;]* +)?__PYX_ERR\([^)]+\))'
+).decode('ascii')).sub
+
+
+_replace_pos_comment = re.compile(
+    # this matches what Cython generates as code line marker comment
+    br'^\s*/\*(?:(?:[^*]|\*[^/])*\n)+\s*\*/\s*\n'.decode('ascii'),
+    re.M
+).sub
+
+
+class AnnotationItem:
+
+    def __init__(self, style, text, tag="", size=0):
+        self.style = style
+        self.text = text
+        self.tag = tag
+        self.size = size
+
+    def start(self):
+        return "<span class='cython tag %s' title='%s'>%s" % (self.style, self.text, self.tag)
+
+    def end(self):
+        return self.size, "</span>"