Cython 3.3.0__cp315-cp315-win_amd64.whl

Cython/Build/SharedModule.py

@@ -0,0 +1,165 @@
+import os
+
+from Cython.Compiler import (
+    MemoryView, Code, Options, Pipeline, Errors, Main, Symtab
+)
+from Cython.Compiler.StringEncoding import EncodedString
+from Cython.Compiler.Scanning import SharedUtilitySourceDescriptor
+from Cython.Compiler.Errors import warning
+
+
+def list_of_features(included=None, excluded=None, _list_of_features=[]):
+    if _list_of_features:
+        feature_names = _list_of_features[:]
+    else:
+        feature_names = sorted({feature_name for feature_name, _, _ in _iter_exports()})
+        _list_of_features[:] = feature_names
+
+    return _select_features(feature_names, included, excluded)
+
+
+def _select_features(all_names, included, excluded):
+    names = all_names
+    if included:
+        included = set(included)
+        unknown = included.difference(all_names)
+        if unknown:
+            warning(None, f"Unknown shared module features selected: {', '.join(sorted(unknown))}")
+
+        names = [name for name in names if name in included]
+
+    if excluded:
+        excluded = set(excluded)
+        unknown = excluded.difference(all_names)
+        if unknown:
+            warning(None, f"Unknown shared module features excluded: {', '.join(sorted(unknown))}")
+        unused = excluded.difference(names)
+        if unused:
+            warning(None, f"Shared module features excluded that was not included: {', '.join(sorted(unused))}")
+
+        names = [name for name in names if name not in excluded]
+
+    return names
+
+
+def _iter_exports(selected_features=None):
+    UtilityCode = Code.UtilityCode
+    match_special = UtilityCode.get_special_comment_matcher('/')
+
+    for c_utility_file in os.listdir(Code.get_utility_dir()):
+        if not c_utility_file.endswith('.c'):
+            continue
+
+        file_base_name = os.path.splitext(os.path.basename(c_utility_file))[0]
+        current_feature_name = file_base_name.partition('_')[0]
+
+        current_section = None
+        for line in Code.read_utilities_hook(c_utility_file):
+            if '//' not in line or not line.startswith('//'):
+                continue
+            match = match_special(line)
+            if match is None:
+                continue
+
+            name = match.group('name')
+            if name:
+                if current_section:
+                    if selected_features is None or current_section[0] in selected_features:
+                        yield current_section
+                current_section = None
+
+                section_title = UtilityCode.match_section_title(name) if '.export' in name else None
+                if section_title:
+                    name, section_type = section_title.groups()
+                    if section_type == 'export':
+                        current_section = [current_feature_name, name, c_utility_file]
+
+            elif (tag := match.group('tag')) and tag.startswith('feature') and current_section:
+                explicit_feature_name = tag.split(':', 1)[-1].strip()
+                assert current_section[0] == current_feature_name, (
+                    "Conflicting feature names given to the same utility code section: "
+                    f"{current_section[0]!r} and {explicit_feature_name!r}")
+                current_section[0] = explicit_feature_name
+
+        if current_section:
+            if selected_features is None or current_section[0] in selected_features:
+                yield current_section
+
+
+def create_shared_library_pipeline(context, scope, options, result, selected_features=None):
+
+    parse = Pipeline.parse_stage_factory(context)
+
+    def generate_tree_factory(context):
+        def generate_tree(compsrc):
+            tree = parse(compsrc)
+
+            if selected_features is None or 'MemoryView' in selected_features:
+                tree.scope.use_utility_code(
+                    MemoryView.get_view_utility_code(options.shared_utility_qualified_name))
+
+                tree.scope.use_utility_code(MemoryView._get_memviewslice_declare_code())
+                tree.scope.use_utility_code(MemoryView._get_typeinfo_to_format_code())
+
+            context.include_directories.append(Code.get_utility_dir())
+            return tree
+
+        return generate_tree
+
+    def generate_c_utilities(module_node):
+        UtilityCode = Code.UtilityCode
+        for _, name, c_utility_file in _iter_exports(selected_features):
+            module_node.scope.use_utility_code(UtilityCode.load_cached(name, c_utility_file))
+        return module_node
+
+    orig_cimport_from_pyx = Options.cimport_from_pyx
+
+    def set_cimport_from_pyx(cimport_from_pyx):
+        def inner(node):
+            Options.cimport_from_pyx = cimport_from_pyx
+            return node
+        return inner
+
+    return [
+        # "cimport_from_pyx=True" to force generating __Pyx_ExportFunction
+        set_cimport_from_pyx(True),
+        generate_tree_factory(context),
+        *Pipeline.create_pipeline(context, 'pyx', exclude_classes=()),
+        generate_c_utilities,
+        Pipeline.inject_pxd_code_stage_factory(context),
+        Pipeline.inject_utility_code_stage_factory(context, internalise_c_class_entries=False),
+        Pipeline.inject_utility_pxd_code_stage_factory(context),
+        Pipeline.abort_on_errors,
+        Pipeline.generate_pyx_code_stage_factory(options, result),
+        set_cimport_from_pyx(orig_cimport_from_pyx),
+    ]
+
+
+def generate_shared_module(options):
+    Errors.reset()
+
+    dest_c_file = options.shared_c_file_path
+    pyx_file = os.path.splitext(dest_c_file)[0] + '.pyx'
+    module_name = os.path.splitext(os.path.basename(dest_c_file))[0]
+
+    selected_features = None
+    if options.shared_utility_features_enabled or options.shared_utility_features_disabled:
+        selected_features = list_of_features(
+            options.shared_utility_features_enabled,
+            options.shared_utility_features_disabled,
+        )
+
+    context = Main.Context.from_options(options)
+    scope = Symtab.ModuleScope('MemoryView', parent_module = None, context = context, is_package=False)
+
+    source_desc = SharedUtilitySourceDescriptor(pyx_file)
+    comp_src = Main.CompilationSource(source_desc, EncodedString(module_name), os.getcwd())
+    result = Main.create_default_resultobj(comp_src, options)
+
+    pipeline = create_shared_library_pipeline(
+        context, scope, options, result,
+        selected_features=selected_features,
+    )
+    err, enddata = Pipeline.run_pipeline(pipeline, comp_src)
+
+    return err, enddata

Cython/Build/Tests/TestCyCache.py

@@ -0,0 +1,195 @@
+import difflib
+import glob
+import gzip
+import os
+import sys
+import tempfile
+import unittest
+
+import Cython.Build.Dependencies
+import Cython.Compiler.Main
+import Cython.Utils
+from Cython.TestUtils import CythonTest
+
+
+class TestCyCache(CythonTest):
+
+    def setUp(self):
+        CythonTest.setUp(self)
+        self.temp_dir = tempfile.mkdtemp(
+            prefix='cycache-test',
+            dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None)
+        self.src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir)
+        self.cache_dir = tempfile.mkdtemp(prefix='cache', dir=self.temp_dir)
+
+    def cache_files(self, file_glob):
+        return glob.glob(os.path.join(self.cache_dir, file_glob))
+
+    def fresh_cythonize(self, *args, **kwargs):
+        Cython.Utils.clear_function_caches()
+        Cython.Build.Dependencies._dep_tree = None  # discard method caches
+        Cython.Build.Dependencies.cythonize(*args, **kwargs)
+
+    def fresh_compile(self, *args, **kwargs):
+        Cython.Utils.clear_function_caches()
+        Cython.Compiler.Main.compile(*args, **kwargs)
+
+
+    def _test_cycache_switch(self, compilation_method):
+        content1 = 'value = 1\n'
+        content2 = 'value = 2\n'
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+
+        with open(a_pyx, 'w') as f:
+            f.write(content1)
+
+        compilation_method(a_pyx, cache=self.cache_dir)
+        compilation_method(a_pyx, cache=self.cache_dir)
+
+        self.assertEqual(1, len(self.cache_files('a.c*')))
+        with open(a_c) as f:
+            a_contents1 = f.read()
+        os.unlink(a_c)
+
+        with open(a_pyx, 'w') as f:
+            f.write(content2)
+
+        compilation_method(a_pyx, cache=self.cache_dir)
+
+        with open(a_c) as f:
+            a_contents2 = f.read()
+        os.unlink(a_c)
+
+        self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!')
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+
+        with open(a_pyx, 'w') as f:
+            f.write(content1)
+
+        compilation_method(a_pyx, cache=self.cache_dir)
+
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+        with open(a_c) as f:
+            a_contents = f.read()
+        self.assertEqual(
+            a_contents, a_contents1,
+            msg='\n'.join(list(difflib.unified_diff(
+                a_contents.split('\n'), a_contents1.split('\n')))[:10]))
+
+    def test_cycache_switch_cythonize(self):
+        self._test_cycache_switch(self.fresh_cythonize)
+
+    def test_cycache_switch_compile(self):
+        self._test_cycache_switch(self.fresh_compile)
+
+    def _test_cycache_uses_cache(self, compilation_method):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        with open(a_pyx, 'w') as f:
+            f.write('pass')
+
+        compilation_method(a_pyx, cache=self.cache_dir)
+
+        a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0])
+        with gzip.GzipFile(a_cache, 'wb') as gzipfile:
+            gzipfile.write(b'fake stuff')
+        os.unlink(a_c)
+
+        compilation_method(a_pyx, cache=self.cache_dir)
+
+        with open(a_c) as f:
+            a_contents = f.read()
+        self.assertEqual(a_contents, 'fake stuff',
+                         'Unexpected contents: %s...' % a_contents[:100])
+
+
+    def test_cycache_uses_cache_cythonize(self):
+        self._test_cycache_uses_cache(self.fresh_cythonize)
+
+    def test_cycache_uses_cache_compile(self):
+        self._test_cycache_uses_cache(self.fresh_compile)
+
+    def _test_cycache_annotation(self, compilation_method):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        a_html = a_pyx[:-4] + '.html'
+        with open(a_pyx, 'w') as f:
+            f.write('pass')
+
+        compilation_method(a_pyx, cache=self.cache_dir, annotate='default')
+        self.assertTrue(os.path.exists(a_html), a_html)
+        os.unlink(a_html)
+        os.unlink(a_c)
+        compilation_method(a_pyx, cache=self.cache_dir, annotate='default')
+        self.assertTrue(os.path.exists(a_html), a_html)
+
+    def test_cycache_annotation_cythonize(self):
+        self._test_cycache_annotation(self.fresh_cythonize)
+
+    def test_cycache_annotation_compile(self):
+        self._test_cycache_annotation(self.fresh_compile)
+
+    def _test_multi_file_output(self, compilation_method):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        a_h = a_pyx[:-4] + '.h'
+        a_api_h = a_pyx[:-4] + '_api.h'
+        with open(a_pyx, 'w') as f:
+            f.write('cdef public api int foo(int x): return x\n')
+
+        compilation_method(a_pyx, cache=self.cache_dir)
+
+        expected = [a_c, a_h, a_api_h]
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+            os.unlink(output)
+
+        compilation_method(a_pyx, cache=self.cache_dir)
+
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+            self.assertTrue(os.path.isfile(output), output)
+
+    def test_multi_file_output_cythonize(self):
+        self._test_multi_file_output(self.fresh_cythonize)
+
+    def test_multi_file_output_compile(self):
+        self._test_multi_file_output(self.fresh_compile)
+
+    def _test_options_invalidation(self, compilation_method):
+        hash_pyx = os.path.join(self.src_dir, 'options.pyx')
+        hash_c = hash_pyx[:-len('.pyx')] + '.c'
+        hash_cpp = hash_pyx[:-len('.pyx')] + '.cpp'
+
+        with open(hash_pyx, 'w') as f:
+            f.write('pass')
+        compilation_method(hash_pyx, cache=self.cache_dir, cplus=False)
+        self.assertEqual(1, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+
+        compilation_method(hash_pyx, cache=self.cache_dir, cplus=True)
+
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        try:
+            os.unlink(hash_c)
+        except FileNotFoundError:
+            # fresh_cythonize() produces .c file, fresh_compile produces .cpp file
+            os.unlink(hash_cpp)
+
+        compilation_method(hash_pyx, cache=self.cache_dir, cplus=False, show_version=False)
+
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+
+        compilation_method(hash_pyx, cache=self.cache_dir, cplus=False, show_version=True)
+
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+    def test_options_invalidation_cythonize(self):
+        self._test_options_invalidation(self.fresh_cythonize)
+
+    def test_options_invalidation_compile(self):
+        self._test_options_invalidation(self.fresh_compile)