Cython 3.1.0a1__py3-none-any.whl

Cython/Compiler/__init__.py

@@ -0,0 +1 @@
+# empty file

Cython/Coverage.py

@@ -0,0 +1,443 @@
+"""
+A Cython plugin for coverage.py
+
+Requires the coverage package at least in version 4.0 (which added the plugin API).
+
+This plugin requires the generated C sources to be available, next to the extension module.
+It parses the C file and reads the original source files from it, which are stored in C comments.
+It then reports a source file to coverage.py when it hits one of its lines during line tracing.
+
+Basically, Cython can (on request) emit explicit trace calls into the C code that it generates,
+and as a general human debugging helper, it always copies the current source code line
+(and its surrounding context) into the C files before it generates code for that line, e.g.
+
+::
+
+      /* "line_trace.pyx":147
+       * def cy_add_with_nogil(a,b):
+       *     cdef int z, x=a, y=b         # 1
+       *     with nogil:                  # 2             # <<<<<<<<<<<<<<
+       *         z = 0                    # 3
+       *         z += cy_add_nogil(x, y)  # 4
+       */
+       __Pyx_TraceLine(147,6,1,__PYX_ERR(0, 147, __pyx_L4_error))
+      [C code generated for file line_trace.pyx, line 147, follows here]
+
+The crux is that multiple source files can contribute code to a single C (or C++) file
+(and thus, to a single extension module) besides the main module source file (.py/.pyx),
+usually shared declaration files (.pxd) but also literally included files (.pxi).
+
+Therefore, the coverage plugin doesn't actually try to look at the file that happened
+to contribute the current source line for the trace call, but simply looks up the single
+.c file from which the extension was compiled (which usually lies right next to it after
+the build, having the same name), and parses the code copy comments from that .c file
+to recover the original source files and their code as a line-to-file mapping.
+
+That mapping is then used to report the ``__Pyx_TraceLine()`` calls to the coverage tool.
+The plugin also reports the line of source code that it found in the C file to the coverage
+tool to support annotated source representations.  For this, again, it does not look at the
+actual source files but only reports the source code that it found in the C code comments.
+
+Apart from simplicity (read one file instead of finding and parsing many), part of the
+reasoning here is that any line in the original sources for which there is no comment line
+(and trace call) in the generated C code cannot count as executed, really, so the C code
+comments are a very good source for coverage reporting.  They already filter out purely
+declarative code lines that do not contribute executable code, and such (missing) lines
+can then be marked as excluded from coverage analysis.
+"""
+
+
+import re
+import os.path
+import sys
+from collections import defaultdict
+
+from coverage.plugin import CoveragePlugin, FileTracer, FileReporter  # requires coverage.py 4.0+
+from coverage.files import canonical_filename
+
+from .Utils import find_root_package_dir, is_package_dir, is_cython_generated_file, open_source_file
+
+
+from . import __version__
+
+
+C_FILE_EXTENSIONS = ['.c', '.cpp', '.cc', '.cxx']
+MODULE_FILE_EXTENSIONS = set(['.py', '.pyx', '.pxd'] + C_FILE_EXTENSIONS)
+
+
+def _find_c_source(base_path):
+    file_exists = os.path.exists
+    for ext in C_FILE_EXTENSIONS:
+        file_name = base_path + ext
+        if file_exists(file_name):
+            return file_name
+    return None
+
+
+def _find_dep_file_path(main_file, file_path, relative_path_search=False):
+    abs_path = os.path.abspath(file_path)
+    if not os.path.exists(abs_path) and (file_path.endswith('.pxi') or
+                                         relative_path_search):
+        # files are looked up relative to the main source file
+        rel_file_path = os.path.join(os.path.dirname(main_file), file_path)
+        if os.path.exists(rel_file_path):
+            abs_path = os.path.abspath(rel_file_path)
+
+        abs_no_ext = os.path.splitext(abs_path)[0]
+        file_no_ext, extension = os.path.splitext(file_path)
+        # We check if the paths match by matching the directories in reverse order.
+        # pkg/module.pyx /long/absolute_path/bla/bla/site-packages/pkg/module.c should match.
+        # this will match the pairs: module-module and pkg-pkg. After which there is nothing left to zip.
+        abs_no_ext = os.path.normpath(abs_no_ext)
+        file_no_ext = os.path.normpath(file_no_ext)
+        matching_paths = zip(reversed(abs_no_ext.split(os.sep)), reversed(file_no_ext.split(os.sep)))
+        for one, other in matching_paths:
+            if one != other:
+                break
+        else:  # No mismatches detected
+            matching_abs_path = os.path.splitext(main_file)[0] + extension
+            if os.path.exists(matching_abs_path):
+                return canonical_filename(matching_abs_path)
+
+    # search sys.path for external locations if a valid file hasn't been found
+    if not os.path.exists(abs_path):
+        for sys_path in sys.path:
+            test_path = os.path.realpath(os.path.join(sys_path, file_path))
+            if os.path.exists(test_path):
+                return canonical_filename(test_path)
+    return canonical_filename(abs_path)
+
+
+def _offset_to_line(offset):
+    return offset >> 9
+
+
+class Plugin(CoveragePlugin):
+    # map from traced file paths to absolute file paths
+    _file_path_map = None
+    # map from traced file paths to corresponding C files
+    _c_files_map = None
+    # map from parsed C files to their content
+    _parsed_c_files = None
+    # map from traced files to lines that are excluded from coverage
+    _excluded_lines_map = None
+    # list of regex patterns for lines to exclude
+    _excluded_line_patterns = ()
+
+    def sys_info(self):
+        return [('Cython version', __version__)]
+
+    def configure(self, config):
+        # Entry point for coverage "configurer".
+        # Read the regular expressions from the coverage config that match lines to be excluded from coverage.
+        self._excluded_line_patterns = config.get_option("report:exclude_lines")
+
+    def file_tracer(self, filename):
+        """
+        Try to find a C source file for a file path found by the tracer.
+        """
+        if filename.startswith('<') or filename.startswith('memory:'):
+            return None
+        c_file = py_file = None
+        filename = canonical_filename(os.path.abspath(filename))
+        if self._c_files_map and filename in self._c_files_map:
+            c_file = self._c_files_map[filename][0]
+
+        if c_file is None:
+            c_file, py_file = self._find_source_files(filename)
+            if not c_file:
+                return None  # unknown file
+
+            # parse all source file paths and lines from C file
+            # to learn about all relevant source files right away (pyx/pxi/pxd)
+            # FIXME: this might already be too late if the first executed line
+            #        is not from the main .pyx file but a file with a different
+            #        name than the .c file (which prevents us from finding the
+            #        .c file)
+            _, code = self._read_source_lines(c_file, filename)
+            if code is None:
+                return None  # no source found
+
+        if self._file_path_map is None:
+            self._file_path_map = {}
+        return CythonModuleTracer(filename, py_file, c_file, self._c_files_map, self._file_path_map)
+
+    def file_reporter(self, filename):
+        # TODO: let coverage.py handle .py files itself
+        #ext = os.path.splitext(filename)[1].lower()
+        #if ext == '.py':
+        #    from coverage.python import PythonFileReporter
+        #    return PythonFileReporter(filename)
+
+        filename = canonical_filename(os.path.abspath(filename))
+        if self._c_files_map and filename in self._c_files_map:
+            c_file, rel_file_path, code = self._c_files_map[filename]
+        else:
+            c_file, _ = self._find_source_files(filename)
+            if not c_file:
+                return None  # unknown file
+            rel_file_path, code = self._read_source_lines(c_file, filename)
+            if code is None:
+                return None  # no source found
+        return CythonModuleReporter(
+            c_file,
+            filename,
+            rel_file_path,
+            code,
+            self._excluded_lines_map.get(rel_file_path, frozenset())
+        )
+
+    def _find_source_files(self, filename):
+        basename, ext = os.path.splitext(filename)
+        ext = ext.lower()
+        if ext in MODULE_FILE_EXTENSIONS:
+            pass
+        elif ext == '.pyd':
+            # Windows extension module
+            platform_suffix = re.search(r'[.]cp[0-9]+-win[_a-z0-9]*$', basename, re.I)
+            if platform_suffix:
+                basename = basename[:platform_suffix.start()]
+        elif ext == '.so':
+            # Linux/Unix/Mac extension module
+            platform_suffix = re.search(r'[.](?:cpython|pypy)-[0-9]+[-_a-z0-9]*$', basename, re.I)
+            if platform_suffix:
+                basename = basename[:platform_suffix.start()]
+        elif ext == '.pxi':
+            # if we get here, it means that the first traced line of a Cython module was
+            # not in the main module but in an include file, so try a little harder to
+            # find the main source file
+            self._find_c_source_files(os.path.dirname(filename), filename)
+            if filename in self._c_files_map:
+                return self._c_files_map[filename][0], None
+        else:
+            # none of our business
+            return None, None
+
+        c_file = filename if ext in C_FILE_EXTENSIONS else _find_c_source(basename)
+        if c_file is None:
+            # a module "pkg/mod.so" can have a source file "pkg/pkg.mod.c"
+            package_root = find_root_package_dir.uncached(filename)
+            package_path = os.path.relpath(basename, package_root).split(os.path.sep)
+            if len(package_path) > 1:
+                test_basepath = os.path.join(os.path.dirname(filename), '.'.join(package_path))
+                c_file = _find_c_source(test_basepath)
+
+        py_source_file = None
+        if c_file:
+            py_source_file = os.path.splitext(c_file)[0] + '.py'
+            if not os.path.exists(py_source_file):
+                py_source_file = None
+            if not is_cython_generated_file(c_file, if_not_found=False):
+                if py_source_file and os.path.exists(c_file):
+                    # if we did not generate the C file,
+                    # then we probably also shouldn't care about the .py file.
+                    py_source_file = None
+                c_file = None
+
+        return c_file, py_source_file
+
+    def _find_c_source_files(self, dir_path, source_file):
+        """
+        Desperately parse all C files in the directory or its package parents
+        (not re-descending) to find the (included) source file in one of them.
+        """
+        if not os.path.isdir(dir_path):
+            return
+        splitext = os.path.splitext
+        for filename in os.listdir(dir_path):
+            ext = splitext(filename)[1].lower()
+            if ext in C_FILE_EXTENSIONS:
+                self._read_source_lines(os.path.join(dir_path, filename), source_file)
+                if source_file in self._c_files_map:
+                    return
+        # not found? then try one package up
+        if is_package_dir(dir_path):
+            self._find_c_source_files(os.path.dirname(dir_path), source_file)
+
+    def _read_source_lines(self, c_file, sourcefile):
+        """
+        Parse a Cython generated C/C++ source file and find the executable lines.
+        Each executable line starts with a comment header that states source file
+        and line number, as well as the surrounding range of source code lines.
+        """
+        if self._parsed_c_files is None:
+            self._parsed_c_files = {}
+        if c_file in self._parsed_c_files:
+            code_lines = self._parsed_c_files[c_file]
+        else:
+            code_lines = self._parse_cfile_lines(c_file)
+            self._parsed_c_files[c_file] = code_lines
+
+        if self._c_files_map is None:
+            self._c_files_map = {}
+
+        for filename, code in code_lines.items():
+            abs_path = _find_dep_file_path(c_file, filename,
+                                           relative_path_search=True)
+            self._c_files_map[abs_path] = (c_file, filename, code)
+
+        if sourcefile not in self._c_files_map:
+            return (None,) * 2  # e.g. shared library file
+        return self._c_files_map[sourcefile][1:]
+
+    def _parse_cfile_lines(self, c_file):
+        """
+        Parse a C file and extract all source file lines that generated executable code.
+        """
+        match_source_path_line = re.compile(r' */[*] +"(.*)":([0-9]+)$').match
+        match_current_code_line = re.compile(r' *[*] (.*) # <<<<<<+$').match
+        match_comment_end = re.compile(r' *[*]/$').match
+        match_trace_line = re.compile(r' *__Pyx_TraceLine\(([0-9]+),').match
+        not_executable = re.compile(
+            r'\s*c(?:type)?def\s+'
+            r'(?:(?:public|external)\s+)?'
+            r'(?:struct|union|enum|class)'
+            r'(\s+[^:]+|)\s*:'
+        ).match
+        if self._excluded_line_patterns:
+            line_is_excluded = re.compile("|".join(["(?:%s)" % regex for regex in self._excluded_line_patterns])).search
+        else:
+            line_is_excluded = lambda line: False
+
+        code_lines = defaultdict(dict)
+        executable_lines = defaultdict(set)
+        current_filename = None
+        if self._excluded_lines_map is None:
+            self._excluded_lines_map = defaultdict(set)
+
+        with open(c_file, encoding='utf8') as lines:
+            lines = iter(lines)
+            for line in lines:
+                match = match_source_path_line(line)
+                if not match:
+                    if '__Pyx_TraceLine(' in line and current_filename is not None:
+                        trace_line = match_trace_line(line)
+                        if trace_line:
+                            lineno = int(trace_line.group(1))
+                            executable_lines[current_filename].add(lineno)
+                    continue
+                filename, lineno = match.groups()
+                current_filename = filename
+                lineno = int(lineno)
+                for comment_line in lines:
+                    match = match_current_code_line(comment_line)
+                    if match:
+                        code_line = match.group(1).rstrip()
+                        if not_executable(code_line):
+                            break
+                        if line_is_excluded(code_line):
+                            self._excluded_lines_map[filename].add(lineno)
+                            break
+                        code_lines[filename][lineno] = code_line
+                        break
+                    elif match_comment_end(comment_line):
+                        # unexpected comment format - false positive?
+                        break
+
+        # Remove lines that generated code but are not traceable.
+        for filename, lines in code_lines.items():
+            dead_lines = set(lines).difference(executable_lines.get(filename, ()))
+            for lineno in dead_lines:
+                del lines[lineno]
+        return code_lines
+
+
+class CythonModuleTracer(FileTracer):
+    """
+    Find the Python/Cython source file for a Cython module.
+    """
+    def __init__(self, module_file, py_file, c_file, c_files_map, file_path_map):
+        super().__init__()
+        self.module_file = module_file
+        self.py_file = py_file
+        self.c_file = c_file
+        self._c_files_map = c_files_map
+        self._file_path_map = file_path_map
+
+    def has_dynamic_source_filename(self):
+        return True
+
+    def dynamic_source_filename(self, filename, frame):
+        """
+        Determine source file path.  Called by the function call tracer.
+        """
+        source_file = frame.f_code.co_filename
+        try:
+            return self._file_path_map[source_file]
+        except KeyError:
+            pass
+        abs_path = _find_dep_file_path(filename, source_file)
+
+        if self.py_file and source_file[-3:].lower() == '.py':
+            # always let coverage.py handle this case itself
+            self._file_path_map[source_file] = self.py_file
+            return self.py_file
+
+        assert self._c_files_map is not None
+        if abs_path not in self._c_files_map:
+            self._c_files_map[abs_path] = (self.c_file, source_file, None)
+        self._file_path_map[source_file] = abs_path
+        return abs_path
+
+
+class CythonModuleReporter(FileReporter):
+    """
+    Provide detailed trace information for one source file to coverage.py.
+    """
+    def __init__(self, c_file, source_file, rel_file_path, code, excluded_lines):
+        super().__init__(source_file)
+        self.name = rel_file_path
+        self.c_file = c_file
+        self._code = code
+        self._excluded_lines = excluded_lines
+
+    def lines(self):
+        """
+        Return set of line numbers that are possibly executable.
+        """
+        return set(self._code)
+
+    def excluded_lines(self):
+        """
+        Return set of line numbers that are excluded from coverage.
+        """
+        return self._excluded_lines
+
+    def _iter_source_tokens(self):
+        current_line = 1
+        for line_no, code_line in sorted(self._code.items()):
+            while line_no > current_line:
+                yield []
+                current_line += 1
+            yield [('txt', code_line)]
+            current_line += 1
+
+    def source(self):
+        """
+        Return the source code of the file as a string.
+        """
+        if os.path.exists(self.filename):
+            with open_source_file(self.filename) as f:
+                return f.read()
+        else:
+            return '\n'.join(
+                (tokens[0][1] if tokens else '')
+                for tokens in self._iter_source_tokens())
+
+    def source_token_lines(self):
+        """
+        Iterate over the source code tokens.
+        """
+        if os.path.exists(self.filename):
+            with open_source_file(self.filename) as f:
+                for line in f:
+                    yield [('txt', line.rstrip('\n'))]
+        else:
+            for line in self._iter_source_tokens():
+                yield [('txt', line)]
+
+
+def coverage_init(reg, options):
+    plugin = Plugin()
+    reg.add_configurer(plugin)
+    reg.add_file_tracer(plugin)

Cython/Debugger/Cygdb.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+
+"""
+The Cython debugger
+
+The current directory should contain a directory named 'cython_debug', or a
+path to the cython project directory should be given (the parent directory of
+cython_debug).
+
+Additional gdb args can be provided only if a path to the project directory is
+given.
+"""
+
+import os
+import sys
+import glob
+import tempfile
+import textwrap
+import subprocess
+import optparse
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def make_command_file(path_to_debug_info, prefix_code='',
+                      no_import=False, skip_interpreter=False):
+    if not no_import:
+        pattern = os.path.join(path_to_debug_info,
+                               'cython_debug',
+                               'cython_debug_info_*')
+        debug_files = glob.glob(pattern)
+
+        if not debug_files:
+            sys.exit('%s.\nNo debug files were found in %s. Aborting.' % (
+                                   usage, os.path.abspath(path_to_debug_info)))
+
+    fd, tempfilename = tempfile.mkstemp()
+    f = os.fdopen(fd, 'w')
+    try:
+        f.write(prefix_code)
+        f.write(textwrap.dedent('''\
+            # This is a gdb command file
+            # See https://sourceware.org/gdb/onlinedocs/gdb/Command-Files.html
+
+            set breakpoint pending on
+            set print pretty on
+
+            python
+            try:
+                # Activate virtualenv, if we were launched from one
+                import os
+                virtualenv = os.getenv('VIRTUAL_ENV')
+                if virtualenv:
+                    path_to_activate_this_py = os.path.join(virtualenv, 'bin', 'activate_this.py')
+                    print("gdb command file: Activating virtualenv: %s; path_to_activate_this_py: %s" % (
+                        virtualenv, path_to_activate_this_py))
+                    with open(path_to_activate_this_py) as f:
+                        exec(f.read(), dict(__file__=path_to_activate_this_py))
+                from Cython.Debugger import libcython, libpython
+            except Exception as ex:
+                from traceback import print_exc
+                print("There was an error in Python code originating from the file ''' + str(__file__) + '''")
+                print("It used the Python interpreter " + str(sys.executable))
+                print_exc()
+                exit(1)
+            end
+            '''))
+
+        if no_import:
+            # don't do this, this overrides file command in .gdbinit
+            # f.write("file %s\n" % sys.executable)
+            pass
+        else:
+            if not skip_interpreter:
+                # Point Cygdb to the interpreter that was used to generate
+                # the debugging information.
+                path = os.path.join(path_to_debug_info, "cython_debug", "interpreter")
+                interpreter_file = open(path)
+                try:
+                    interpreter = interpreter_file.read()
+                finally:
+                    interpreter_file.close()
+                f.write("file %s\n" % interpreter)
+
+            f.write('\n'.join('cy import %s\n' % fn for fn in debug_files))
+
+            if not skip_interpreter:
+                f.write(textwrap.dedent('''\
+                    python
+                    import sys
+                    # Check if the Python executable provides a symbol table.
+                    if not hasattr(gdb.selected_inferior().progspace, "symbol_file"):
+                        sys.stderr.write(
+                            "''' + interpreter + ''' was not compiled with debug symbols (or it was "
+                            "stripped). Some functionality may not work (properly).\\n")
+                    end
+                '''))
+
+            f.write("source .cygdbinit\n")
+    finally:
+        f.close()
+
+    return tempfilename
+
+usage = "Usage: cygdb [options] [PATH [-- GDB_ARGUMENTS]]"
+
+def main(path_to_debug_info=None, gdb_argv=None, no_import=False):
+    """
+    Start the Cython debugger. This tells gdb to import the Cython and Python
+    extensions (libcython.py and libpython.py) and it enables gdb's pending
+    breakpoints.
+
+    path_to_debug_info is the path to the Cython build directory
+    gdb_argv is the list of options to gdb
+    no_import tells cygdb whether it should import debug information
+    """
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option("--gdb-executable",
+        dest="gdb", default='gdb',
+        help="gdb executable to use [default: gdb]")
+    parser.add_option("--verbose", "-v",
+        dest="verbosity", action="count", default=0,
+        help="Verbose mode. Multiple -v options increase the verbosity")
+    parser.add_option("--skip-interpreter",
+                      dest="skip_interpreter", default=False, action="store_true",
+                      help="Do not automatically point GDB to the same interpreter "
+                           "used to generate debugging information")
+
+    (options, args) = parser.parse_args()
+    if path_to_debug_info is None:
+        if len(args) > 1:
+            path_to_debug_info = args[0]
+        else:
+            path_to_debug_info = os.curdir
+
+    if gdb_argv is None:
+        gdb_argv = args[1:]
+
+    if path_to_debug_info == '--':
+        no_import = True
+
+    logging_level = logging.WARN
+    if options.verbosity == 1:
+        logging_level = logging.INFO
+    if options.verbosity >= 2:
+        logging_level = logging.DEBUG
+    logging.basicConfig(level=logging_level)
+
+    skip_interpreter = options.skip_interpreter
+
+    logger.info("verbosity = %r", options.verbosity)
+    logger.debug("options = %r; args = %r", options, args)
+    logger.debug("Done parsing command-line options. path_to_debug_info = %r, gdb_argv = %r",
+        path_to_debug_info, gdb_argv)
+
+    tempfilename = make_command_file(path_to_debug_info,
+                                     no_import=no_import,
+                                     skip_interpreter=skip_interpreter)
+    logger.info("Launching %s with command file: %s and gdb_argv: %s",
+        options.gdb, tempfilename, gdb_argv)
+    with open(tempfilename) as tempfile:
+        logger.debug('Command file (%s) contains: """\n%s"""', tempfilename, tempfile.read())
+        logger.info("Spawning %s...", options.gdb)
+        p = subprocess.Popen([options.gdb, '-command', tempfilename] + gdb_argv)
+        logger.info("Spawned %s (pid %d)", options.gdb, p.pid)
+        while True:
+            try:
+                logger.debug("Waiting for gdb (pid %d) to exit...", p.pid)
+                ret = p.wait()
+                logger.debug("Wait for gdb (pid %d) to exit is done. Returned: %r", p.pid, ret)
+            except KeyboardInterrupt:
+                pass
+            else:
+                break
+        logger.debug("Closing temp command file with fd: %s", tempfile.fileno())
+    logger.debug("Removing temp command file: %s", tempfilename)
+    os.remove(tempfilename)
+    logger.debug("Removed temp command file: %s", tempfilename)