omdev 0.0.0.dev23__py3-none-any.whl → 0.0.0.dev25__py3-none-any.whl

omdev/scripts/interp.py

@@ -1234,13 +1234,134 @@ class StandardLogFormatter(logging.Formatter):
 ##
 
 
+class ProxyLogFilterer(logging.Filterer):
+    def __init__(self, underlying: logging.Filterer) -> None:  # noqa
+        self._underlying = underlying
+
+    @property
+    def underlying(self) -> logging.Filterer:
+        return self._underlying
+
+    @property
+    def filters(self):
+        return self._underlying.filters
+
+    @filters.setter
+    def filters(self, filters):
+        self._underlying.filters = filters
+
+    def addFilter(self, filter):  # noqa
+        self._underlying.addFilter(filter)
+
+    def removeFilter(self, filter):  # noqa
+        self._underlying.removeFilter(filter)
+
+    def filter(self, record):
+        return self._underlying.filter(record)
+
+
+class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
+    def __init__(self, underlying: logging.Handler) -> None:  # noqa
+        ProxyLogFilterer.__init__(self, underlying)
+
+    _underlying: logging.Handler
+
+    @property
+    def underlying(self) -> logging.Handler:
+        return self._underlying
+
+    def get_name(self):
+        return self._underlying.get_name()
+
+    def set_name(self, name):
+        self._underlying.set_name(name)
+
+    @property
+    def name(self):
+        return self._underlying.name
+
+    @property
+    def level(self):
+        return self._underlying.level
+
+    @level.setter
+    def level(self, level):
+        self._underlying.level = level
+
+    @property
+    def formatter(self):
+        return self._underlying.formatter
+
+    @formatter.setter
+    def formatter(self, formatter):
+        self._underlying.formatter = formatter
+
+    def createLock(self):
+        self._underlying.createLock()
+
+    def acquire(self):
+        self._underlying.acquire()
+
+    def release(self):
+        self._underlying.release()
+
+    def setLevel(self, level):
+        self._underlying.setLevel(level)
+
+    def format(self, record):
+        return self._underlying.format(record)
+
+    def emit(self, record):
+        self._underlying.emit(record)
+
+    def handle(self, record):
+        return self._underlying.handle(record)
+
+    def setFormatter(self, fmt):
+        self._underlying.setFormatter(fmt)
+
+    def flush(self):
+        self._underlying.flush()
+
+    def close(self):
+        self._underlying.close()
+
+    def handleError(self, record):
+        self._underlying.handleError(record)
+
+
+##
+
+
+class StandardLogHandler(ProxyLogHandler):
+    pass
+
+
+##
+
+
 def configure_standard_logging(
         level: ta.Union[int, str] = logging.INFO,
         *,
         json: bool = False,
-) -> logging.Handler:
+        target: ta.Optional[logging.Logger] = None,
+        no_check: bool = False,
+) -> ta.Optional[StandardLogHandler]:
+    if target is None:
+        target = logging.root
+
+    #
+
+    if not no_check:
+        if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+            return None
+
+    #
+
     handler = logging.StreamHandler()
 
+    #
+
     formatter: logging.Formatter
     if json:
         formatter = JsonLogFormatter()
@@ -1248,14 +1369,22 @@ def configure_standard_logging(
         formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
     handler.setFormatter(formatter)
 
+    #
+
     handler.addFilter(TidLogFilter())
 
-    logging.root.addHandler(handler)
+    #
+
+    target.addHandler(handler)
+
+    #
 
     if level is not None:
-        logging.root.setLevel(level)
+        target.setLevel(level)
+
+    #
 
-    return handler
+    return StandardLogHandler(handler)
 
 
 ########################################
@@ -1705,6 +1834,14 @@ class Pyenv:
             ret.append(l)
         return ret
 
+    def update(self) -> bool:
+        if (root := self.root()) is None:
+            return False
+        if not os.path.isdir(os.path.join(root, '.git')):
+            return False
+        subprocess_check_call('git', 'pull', cwd=root)
+        return True
+
 
 ##
 
@@ -1905,6 +2042,10 @@ class PyenvInterpProvider(InterpProvider):
 
             inspect: bool = False,
             inspector: InterpInspector = INTERP_INSPECTOR,
+
+            *,
+
+            try_update: bool = False,
     ) -> None:
         super().__init__()
 
@@ -1913,6 +2054,8 @@ class PyenvInterpProvider(InterpProvider):
         self._inspect = inspect
         self._inspector = inspector
 
+        self._try_update = try_update
+
     #
 
     @staticmethod
@@ -1977,8 +2120,9 @@ class PyenvInterpProvider(InterpProvider):
 
     #
 
-    def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
+    def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
         lst = []
+
         for vs in self._pyenv.installable_versions():
             if (iv := self.guess_version(vs)) is None:
                 continue
@@ -1986,6 +2130,16 @@ class PyenvInterpProvider(InterpProvider):
                 raise Exception('Pyenv installable versions not expected to have debug suffix')
             for d in [False, True]:
                 lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
+
+        return lst
+
+    def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
+        lst = self._get_installable_versions(spec)
+
+        if self._try_update and not any(v in spec for v in lst):
+            if self._pyenv.update():
+                lst = self._get_installable_versions(spec)
+
         return lst
 
     def install_version(self, version: InterpVersion) -> Interp:
@@ -2208,7 +2362,7 @@ class InterpResolver:
 
 DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
     # pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
-    PyenvInterpProvider(),
+    PyenvInterpProvider(try_update=True),
 
     RunningInterpProvider(),
 

omdev/scripts/pyproject.py

@@ -115,6 +115,13 @@ def find_magic(
         *,
         py: bool = False,
 ) -> ta.Iterator[str]:
+    if isinstance(roots, str):
+        raise TypeError(roots)
+    if isinstance(magics, str):
+        raise TypeError(magics)
+    if isinstance(exts, str):
+        raise TypeError(exts)
+
     if not magics:
         raise Exception('Must specify magics')
     if not exts:
@@ -2580,13 +2587,134 @@ class StandardLogFormatter(logging.Formatter):
 ##
 
 
+class ProxyLogFilterer(logging.Filterer):
+    def __init__(self, underlying: logging.Filterer) -> None:  # noqa
+        self._underlying = underlying
+
+    @property
+    def underlying(self) -> logging.Filterer:
+        return self._underlying
+
+    @property
+    def filters(self):
+        return self._underlying.filters
+
+    @filters.setter
+    def filters(self, filters):
+        self._underlying.filters = filters
+
+    def addFilter(self, filter):  # noqa
+        self._underlying.addFilter(filter)
+
+    def removeFilter(self, filter):  # noqa
+        self._underlying.removeFilter(filter)
+
+    def filter(self, record):
+        return self._underlying.filter(record)
+
+
+class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
+    def __init__(self, underlying: logging.Handler) -> None:  # noqa
+        ProxyLogFilterer.__init__(self, underlying)
+
+    _underlying: logging.Handler
+
+    @property
+    def underlying(self) -> logging.Handler:
+        return self._underlying
+
+    def get_name(self):
+        return self._underlying.get_name()
+
+    def set_name(self, name):
+        self._underlying.set_name(name)
+
+    @property
+    def name(self):
+        return self._underlying.name
+
+    @property
+    def level(self):
+        return self._underlying.level
+
+    @level.setter
+    def level(self, level):
+        self._underlying.level = level
+
+    @property
+    def formatter(self):
+        return self._underlying.formatter
+
+    @formatter.setter
+    def formatter(self, formatter):
+        self._underlying.formatter = formatter
+
+    def createLock(self):
+        self._underlying.createLock()
+
+    def acquire(self):
+        self._underlying.acquire()
+
+    def release(self):
+        self._underlying.release()
+
+    def setLevel(self, level):
+        self._underlying.setLevel(level)
+
+    def format(self, record):
+        return self._underlying.format(record)
+
+    def emit(self, record):
+        self._underlying.emit(record)
+
+    def handle(self, record):
+        return self._underlying.handle(record)
+
+    def setFormatter(self, fmt):
+        self._underlying.setFormatter(fmt)
+
+    def flush(self):
+        self._underlying.flush()
+
+    def close(self):
+        self._underlying.close()
+
+    def handleError(self, record):
+        self._underlying.handleError(record)
+
+
+##
+
+
+class StandardLogHandler(ProxyLogHandler):
+    pass
+
+
+##
+
+
 def configure_standard_logging(
         level: ta.Union[int, str] = logging.INFO,
         *,
         json: bool = False,
-) -> logging.Handler:
+        target: ta.Optional[logging.Logger] = None,
+        no_check: bool = False,
+) -> ta.Optional[StandardLogHandler]:
+    if target is None:
+        target = logging.root
+
+    #
+
+    if not no_check:
+        if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
+            return None
+
+    #
+
     handler = logging.StreamHandler()
 
+    #
+
     formatter: logging.Formatter
     if json:
         formatter = JsonLogFormatter()
@@ -2594,14 +2722,22 @@ def configure_standard_logging(
         formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
     handler.setFormatter(formatter)
 
+    #
+
     handler.addFilter(TidLogFilter())
 
-    logging.root.addHandler(handler)
+    #
+
+    target.addHandler(handler)
+
+    #
 
     if level is not None:
-        logging.root.setLevel(level)
+        target.setLevel(level)
 
-    return handler
+    #
+
+    return StandardLogHandler(handler)
 
 
 ########################################
@@ -3707,6 +3843,7 @@ class PyprojectPackageGenerator(BasePyprojectPackageGenerator):
         st.pop('cexts', None)
 
         self._move_dict_key(st, 'find_packages', pyp_dct, 'tool.setuptools.packages.find')
+        self._move_dict_key(st, 'package_data', pyp_dct, 'tool.setuptools.package-data')
 
         mani_in = st.pop('manifest_in', None)
 
@@ -3789,9 +3926,14 @@ class _PyprojectCextPackageGenerator(BasePyprojectPackageGenerator):
         st = specs.setuptools
         pyp_dct['tool.setuptools'] = st
 
-        st.pop('cexts', None)
-        st.pop('find_packages', None)
-        st.pop('manifest_in', None)
+        for k in [
+            'cexts',
+
+            'find_packages',
+            'package_data',
+            'manifest_in',
+        ]:
+            st.pop(k, None)
 
         pyp_dct['tool.setuptools.packages.find'] = {
             'include': [],
@@ -3973,6 +4115,14 @@ class Pyenv:
             ret.append(l)
         return ret
 
+    def update(self) -> bool:
+        if (root := self.root()) is None:
+            return False
+        if not os.path.isdir(os.path.join(root, '.git')):
+            return False
+        subprocess_check_call('git', 'pull', cwd=root)
+        return True
+
 
 ##
 
@@ -4173,6 +4323,10 @@ class PyenvInterpProvider(InterpProvider):
 
             inspect: bool = False,
             inspector: InterpInspector = INTERP_INSPECTOR,
+
+            *,
+
+            try_update: bool = False,
     ) -> None:
         super().__init__()
 
@@ -4181,6 +4335,8 @@ class PyenvInterpProvider(InterpProvider):
         self._inspect = inspect
         self._inspector = inspector
 
+        self._try_update = try_update
+
     #
 
     @staticmethod
@@ -4245,8 +4401,9 @@ class PyenvInterpProvider(InterpProvider):
 
     #
 
-    def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
+    def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
         lst = []
+
         for vs in self._pyenv.installable_versions():
             if (iv := self.guess_version(vs)) is None:
                 continue
@@ -4254,6 +4411,16 @@ class PyenvInterpProvider(InterpProvider):
                 raise Exception('Pyenv installable versions not expected to have debug suffix')
             for d in [False, True]:
                 lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
+
+        return lst
+
+    def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
+        lst = self._get_installable_versions(spec)
+
+        if self._try_update and not any(v in spec for v in lst):
+            if self._pyenv.update():
+                lst = self._get_installable_versions(spec)
+
         return lst
 
     def install_version(self, version: InterpVersion) -> Interp:
@@ -4476,7 +4643,7 @@ class InterpResolver:
 
 DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
     # pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
-    PyenvInterpProvider(),
+    PyenvInterpProvider(try_update=True),
 
     RunningInterpProvider(),
 
@@ -4679,18 +4846,30 @@ def _venv_cmd(args) -> None:
             f'--_docker_container={shlex.quote(sd)}',
             *map(shlex.quote, sys.argv[1:]),
         ])
+
+        docker_env = {
+            'DOCKER_HOST_PLATFORM': os.environ.get('DOCKER_HOST_PLATFORM', sys.platform),
+        }
+        for e in args.docker_env or []:
+            if '=' in e:
+                k, _, v = e.split('=')
+                docker_env[k] = v
+            else:
+                docker_env[e] = os.environ.get(e, '')
+
         subprocess_check_call(
             'docker',
             'compose',
             '-f', 'docker/compose.yml',
             'exec',
             *itertools.chain.from_iterable(
-                ('-e', f'{e}={os.environ.get(e, "")}' if '=' not in e else e)
-                for e in (args.docker_env or [])
+                ('-e', f'{k}={v}')
+                for k, v in docker_env.items()
             ),
             '-it', sd,
             'bash', '--login', '-c', script,
         )
+
         return
 
     cmd = args.cmd

omdev/tools/importscan.py

@@ -0,0 +1,187 @@
+"""
+TODO:
+ - dump agg stats
+ - graphviz
+"""
+import argparse
+import dataclasses as dc
+import inspect
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+import typing as ta
+
+from omlish import concurrent as cu
+from omlish import lang
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Item:
+    spec: str
+
+    time: float
+    rss: int
+    imported: frozenset[str]
+
+
+##
+
+
+def _payload(specs) -> None:
+    import resource
+    import sys
+    import time
+
+    def get_rss() -> int:
+        return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+
+    start_modules = frozenset(sys.modules)
+    start_rss = get_rss()
+    start_time = time.time()
+
+    for spec in specs:
+        exec(f'import {spec}')
+
+    end_time = time.time()
+    end_rss = get_rss()
+    end_modules = frozenset(sys.modules)
+
+    import json
+
+    def json_dumps(obj):
+        return json.dumps(obj, indent=None, separators=(',', ':'))
+
+    print(json_dumps({
+        'time': end_time - start_time,
+        'rss': end_rss - start_rss,
+        'imported': sorted(end_modules - start_modules),
+    }))
+
+
+@lang.cached_function
+def payload_src() -> str:
+    return inspect.getsource(_payload)
+
+
+def run_one(
+        spec: str,
+        *,
+        shell_wrap: bool = True,
+) -> Item:
+    spec_payload_src = '\n\n'.join([
+        payload_src(),
+        f'_payload([{spec!r}])',
+    ])
+
+    args = [
+        sys.executable,
+        '-c',
+        spec_payload_src,
+    ]
+    if shell_wrap:
+        args = ['sh', '-c', ' '.join(map(shlex.quote, args))]
+
+    output = subprocess.check_output(args)
+
+    output_lines = output.decode().strip().splitlines()
+    if not output_lines:
+        raise Exception(f'no output: {spec}')
+    if len(output_lines) > 1:
+        print(f'warning: unexpected output: {spec}')
+
+    dct = json.loads(output_lines[-1])
+    return Item(
+        spec=spec,
+        **dct,
+    )
+
+
+##
+
+
+def _find_specs(
+        *roots: str,
+        filters: ta.Iterable[str] | None = None,
+) -> ta.Sequence[str]:
+    filter_pats = [re.compile(f) for f in filters or []]
+
+    out: list[str] = []
+    stk: list[str] = list(reversed(roots))
+    while stk:
+        cur = stk.pop()
+        if os.sep in cur:
+            if os.path.isdir(cur):
+                stk.extend(reversed([
+                    os.path.join(cur, c)
+                    for c in os.listdir(cur)
+                ]))
+                continue
+
+            if not cur.endswith('.py'):
+                continue
+
+            spec = cur.rpartition('.')[0].replace(os.sep, '.')
+
+        else:
+            spec = cur
+
+        if any(p.fullmatch(spec) for p in filter_pats):
+            continue
+
+        out.append(spec)
+
+    return out
+
+
+def run(
+        *roots: str,
+        filters: ta.Iterable[str] | None = None,
+        num_threads: int | None = 0,
+        **kwargs: ta.Any,
+) -> ta.Mapping[str, Item]:
+    specs = _find_specs(*roots, filters=filters)
+
+    out: dict[str, Item] = {}
+    with cu.new_executor(num_threads) as ex:
+        futs = [ex.submit(run_one, spec, **kwargs) for spec in specs]
+        for fut in futs:
+            item = fut.result()
+            out[item.spec] = item
+
+    return out
+
+
+##
+
+
+def _main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-j', '--jobs', type=int)
+    parser.add_argument('-f', '--filter', action='append')
+    parser.add_argument('-t', '--filter-tests', action='store_true')
+    parser.add_argument('root', nargs='+')
+    args = parser.parse_args()
+
+    filters = [*(args.filter or [])]
+    if args.filter_tests:
+        filters.extend([
+            r'.*\.conftest',
+            r'.*\.tests\..*',
+        ])
+
+    for item in run(
+            *args.root,
+            filters=filters,
+            num_threads=args.jobs,
+    ).values():
+        print(json.dumps(dc.asdict(item)))
+
+
+if __name__ == '__main__':
+    _main()