scitex 2.14.0__py3-none-any.whl → 2.15.2__py3-none-any.whl

scitex/introspect/_core.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-01-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/introspect/_core.py
+
+"""Core introspection module - re-exports all utilities."""
+
+from ._call_graph import get_call_graph, get_function_calls
+from ._class_hierarchy import get_class_hierarchy, get_mro
+from ._docstring import get_docstring
+from ._examples import find_examples
+from ._imports import get_dependencies, get_imports
+from ._members import dir, get_exports
+from ._resolve import get_type_info, resolve_object
+from ._signature import q
+from ._source import qq
+from ._type_hints import get_class_annotations, get_type_hints_detailed
+
+__all__ = [
+    # IPython-style names
+    "q",  # signature (like func?)
+    "qq",  # source (like func??)
+    "dir",  # members (like dir())
+    # Basic
+    "get_docstring",
+    "get_exports",
+    "find_examples",
+    "resolve_object",
+    "get_type_info",
+    # Advanced - Class hierarchy
+    "get_class_hierarchy",
+    "get_mro",
+    # Advanced - Type hints
+    "get_type_hints_detailed",
+    "get_class_annotations",
+    # Advanced - Imports
+    "get_imports",
+    "get_dependencies",
+    # Advanced - Call graph
+    "get_call_graph",
+    "get_function_calls",
+]

scitex/introspect/_docstring.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-01-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/introspect/_docstring.py
+
+"""Docstring extraction and parsing utilities."""
+
+from __future__ import annotations
+
+import inspect
+import re
+from typing import Literal
+
+from ._resolve import get_type_info, resolve_object
+
+
+def _parse_docstring(docstring: str) -> dict:
+    """Parse numpy/google style docstring into sections."""
+    sections = {
+        "summary": "",
+        "description": "",
+        "parameters": "",
+        "returns": "",
+        "examples": "",
+        "notes": "",
+    }
+
+    if not docstring:
+        return sections
+
+    section_patterns = [
+        (r"Parameters?\s*\n[-=]+", "parameters"),
+        (r"Returns?\s*\n[-=]+", "returns"),
+        (r"Examples?\s*\n[-=]+", "examples"),
+        (r"Notes?\s*\n[-=]+", "notes"),
+        (r"Raises?\s*\n[-=]+", "raises"),
+        (r"See Also\s*\n[-=]+", "see_also"),
+    ]
+
+    lines = docstring.split("\n")
+
+    i = 0
+    summary_lines = []
+    while i < len(lines):
+        line = lines[i].strip()
+        if not line:
+            if summary_lines:
+                break
+        elif any(re.match(p, line, re.IGNORECASE) for p, _ in section_patterns):
+            break
+        else:
+            summary_lines.append(line)
+        i += 1
+
+    sections["summary"] = " ".join(summary_lines)
+
+    current_section = "description"
+    current_content = []
+
+    for line in lines[i:]:
+        matched = False
+        for pattern, section_name in section_patterns:
+            if re.match(pattern, line, re.IGNORECASE):
+                if current_content:
+                    sections[current_section] = "\n".join(current_content).strip()
+                current_section = section_name
+                current_content = []
+                matched = True
+                break
+
+        if not matched:
+            current_content.append(line)
+
+    if current_content:
+        sections[current_section] = "\n".join(current_content).strip()
+
+    return sections
+
+
+def get_docstring(
+    dotted_path: str,
+    format: Literal["raw", "parsed", "summary"] = "raw",
+) -> dict:
+    """
+    Get the docstring of a Python object.
+
+    Parameters
+    ----------
+    dotted_path : str
+        Dotted path to the object
+    format : str
+        'raw' - Return full docstring as-is
+        'parsed' - Parse numpy/google style into sections
+        'summary' - Return only first line/paragraph
+
+    Returns
+    -------
+    dict
+        docstring: str
+        sections: dict (if format='parsed')
+        type_info: dict
+    """
+    obj, error = resolve_object(dotted_path)
+    if error:
+        return {"success": False, "error": error}
+
+    type_info = get_type_info(obj)
+    docstring = inspect.getdoc(obj) or ""
+
+    if format == "summary":
+        lines = docstring.split("\n\n")
+        summary = lines[0].strip() if lines else ""
+        return {
+            "success": True,
+            "docstring": summary,
+            "type_info": type_info,
+        }
+
+    if format == "parsed":
+        sections = _parse_docstring(docstring)
+        return {
+            "success": True,
+            "docstring": docstring,
+            "sections": sections,
+            "type_info": type_info,
+        }
+
+    return {
+        "success": True,
+        "docstring": docstring,
+        "type_info": type_info,
+    }

scitex/introspect/_examples.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-01-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/introspect/_examples.py
+
+"""Usage example finding utilities."""
+
+from __future__ import annotations
+
+import importlib
+from pathlib import Path
+
+from ._resolve import resolve_object
+
+
+def find_examples(
+    dotted_path: str,
+    search_paths: list[str] | None = None,
+    max_results: int = 10,
+) -> dict:
+    """
+    Find usage examples of a function/class in tests and examples.
+
+    Parameters
+    ----------
+    dotted_path : str
+        Dotted path to search for (e.g., 'scitex.plt.plot')
+    search_paths : list[str] | None
+        Paths to search. If None, auto-detect from module location
+    max_results : int
+        Maximum number of examples to return
+
+    Returns
+    -------
+    dict
+        examples: list[dict] - Each with file, line, context
+        count: int
+    """
+    obj, error = resolve_object(dotted_path)
+    if error:
+        return {"success": False, "error": error}
+
+    name = getattr(obj, "__name__", dotted_path.split(".")[-1])
+
+    if search_paths is None:
+        search_paths = []
+
+        module = dotted_path.split(".")[0]
+        try:
+            mod = importlib.import_module(module)
+            if hasattr(mod, "__file__") and mod.__file__:
+                mod_dir = Path(mod.__file__).parent
+                project_root = mod_dir.parent
+                for subdir in ["tests", "test", "examples", "example"]:
+                    test_dir = project_root / subdir
+                    if test_dir.exists():
+                        search_paths.append(str(test_dir))
+        except ImportError:
+            pass
+
+    if not search_paths:
+        return {
+            "success": True,
+            "examples": [],
+            "count": 0,
+            "message": "No test/example directories found",
+        }
+
+    examples = []
+
+    for search_path in search_paths:
+        path = Path(search_path)
+        if not path.exists():
+            continue
+
+        for py_file in path.rglob("*.py"):
+            try:
+                content = py_file.read_text()
+            except Exception:
+                continue
+
+            if name not in content:
+                continue
+
+            lines = content.split("\n")
+            for i, line in enumerate(lines):
+                if name in line:
+                    start = max(0, i - 2)
+                    end = min(len(lines), i + 3)
+                    context = "\n".join(lines[start:end])
+
+                    examples.append(
+                        {
+                            "file": str(py_file),
+                            "line": i + 1,
+                            "context": context,
+                        }
+                    )
+
+                    if len(examples) >= max_results:
+                        break
+
+            if len(examples) >= max_results:
+                break
+
+        if len(examples) >= max_results:
+            break
+
+    return {
+        "success": True,
+        "examples": examples,
+        "count": len(examples),
+        "search_paths": search_paths,
+    }

scitex/introspect/_imports.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+# Timestamp: 2025-01-20
+# File: /home/ywatanabe/proj/scitex-code/src/scitex/introspect/_imports.py
+
+"""Import analysis utilities using AST."""
+
+from __future__ import annotations
+
+import ast
+import inspect
+from pathlib import Path
+
+from ._resolve import get_type_info, resolve_object
+
+
+def get_imports(
+    dotted_path: str,
+    categorize: bool = True,
+) -> dict:
+    """
+    Get all imports from a module's source code using AST.
+
+    Parameters
+    ----------
+    dotted_path : str
+        Dotted path to the module
+    categorize : bool
+        Group imports by category (stdlib, third-party, local)
+
+    Returns
+    -------
+    dict
+        imports: list[dict] - All imports with details
+        categories: dict - Grouped by category (if categorize=True)
+
+    Examples
+    --------
+    >>> get_imports("scitex.audio")
+    """
+    obj, error = resolve_object(dotted_path)
+    if error:
+        return {"success": False, "error": error}
+
+    type_info = get_type_info(obj)
+
+    if not inspect.ismodule(obj):
+        return {
+            "success": False,
+            "error": f"'{dotted_path}' is not a module",
+            "type_info": type_info,
+        }
+
+    # Get source file
+    try:
+        source_file = inspect.getfile(obj)
+    except TypeError:
+        return {
+            "success": False,
+            "error": "Cannot get source file (builtin module?)",
+            "type_info": type_info,
+        }
+
+    # Read and parse source
+    try:
+        source = Path(source_file).read_text()
+        tree = ast.parse(source)
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Cannot parse source: {e}",
+            "type_info": type_info,
+        }
+
+    imports = _extract_imports(tree)
+
+    result = {
+        "success": True,
+        "module": dotted_path,
+        "source_file": source_file,
+        "imports": imports,
+        "import_count": len(imports),
+        "type_info": type_info,
+    }
+
+    if categorize:
+        result["categories"] = _categorize_imports(imports)
+
+    return result
+
+
+def _extract_imports(tree: ast.AST) -> list[dict]:
+    """Extract all imports from an AST."""
+    imports = []
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                imports.append(
+                    {
+                        "type": "import",
+                        "module": alias.name,
+                        "alias": alias.asname,
+                        "line": node.lineno,
+                    }
+                )
+        elif isinstance(node, ast.ImportFrom):
+            module = node.module or ""
+            level = node.level  # Relative import level
+
+            for alias in node.names:
+                imports.append(
+                    {
+                        "type": "from",
+                        "module": module,
+                        "name": alias.name,
+                        "alias": alias.asname,
+                        "level": level,
+                        "line": node.lineno,
+                    }
+                )
+
+    return imports
+
+
+def _categorize_imports(imports: list[dict]) -> dict:
+    """Categorize imports into stdlib, third-party, local."""
+    import sys
+
+    stdlib_modules = (
+        set(sys.stdlib_module_names)
+        if hasattr(sys, "stdlib_module_names")
+        else _get_stdlib_modules()
+    )
+
+    categories = {
+        "stdlib": [],
+        "third_party": [],
+        "local": [],
+    }
+
+    for imp in imports:
+        module = imp["module"]
+        top_level = module.split(".")[0] if module else ""
+
+        # Relative imports are local
+        if imp.get("level", 0) > 0:
+            categories["local"].append(imp)
+        elif top_level in stdlib_modules:
+            categories["stdlib"].append(imp)
+        else:
+            categories["third_party"].append(imp)
+
+    return categories
+
+
+def _get_stdlib_modules() -> set:
+    """Get stdlib module names for Python < 3.10."""
+    import pkgutil
+
+    stdlib = set()
+    for module in pkgutil.iter_modules():
+        if module.name.startswith("_"):
+            continue
+        try:
+            spec = __import__(module.name).__spec__
+            if spec and spec.origin:
+                if "site-packages" not in spec.origin:
+                    stdlib.add(module.name)
+        except Exception:
+            pass
+
+    # Add common ones that might be missed
+    stdlib.update(
+        [
+            "abc",
+            "ast",
+            "asyncio",
+            "collections",
+            "contextlib",
+            "dataclasses",
+            "datetime",
+            "functools",
+            "inspect",
+            "io",
+            "itertools",
+            "json",
+            "logging",
+            "os",
+            "pathlib",
+            "re",
+            "sys",
+            "typing",
+            "unittest",
+            "warnings",
+        ]
+    )
+
+    return stdlib
+
+
+def get_dependencies(
+    dotted_path: str,
+    recursive: bool = False,
+    max_depth: int = 3,
+) -> dict:
+    """
+    Get module dependencies (what it imports).
+
+    Parameters
+    ----------
+    dotted_path : str
+        Dotted path to the module
+    recursive : bool
+        Recursively analyze imported modules
+    max_depth : int
+        Maximum recursion depth
+
+    Returns
+    -------
+    dict
+        dependencies: list[str] - Direct dependencies
+        tree: dict - Dependency tree (if recursive)
+    """
+    result = get_imports(dotted_path, categorize=True)
+    if not result.get("success"):
+        return result
+
+    # Get unique module names
+    deps = set()
+    for imp in result["imports"]:
+        module = imp["module"]
+        if module:
+            deps.add(module.split(".")[0])
+
+    result["dependencies"] = sorted(deps)
+    result["dependency_count"] = len(deps)
+
+    if recursive:
+        result["tree"] = _build_dep_tree(dotted_path, max_depth, set())
+
+    return result
+
+
+def _build_dep_tree(
+    module_path: str,
+    max_depth: int,
+    visited: set,
+    current_depth: int = 0,
+) -> dict:
+    """Build dependency tree recursively."""
+    if current_depth >= max_depth or module_path in visited:
+        return {"module": module_path, "truncated": True}
+
+    visited.add(module_path)
+
+    result = {"module": module_path, "imports": []}
+
+    imports_result = get_imports(module_path, categorize=False)
+    if not imports_result.get("success"):
+        return result
+
+    for imp in imports_result.get("imports", []):
+        module = imp["module"]
+        if module and module not in visited:
+            top_level = module.split(".")[0]
+            # Only recurse into non-stdlib
+            if top_level not in _get_stdlib_modules():
+                child = _build_dep_tree(module, max_depth, visited, current_depth + 1)
+                result["imports"].append(child)
+
+    return result

scitex/{gen/_inspect_module.py → introspect/_list_api.py}

@@ -1,18 +1,18 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-11-07 18:58:55 (ywatanabe)"
-# File: ./scitex_repo/src/scitex/gen/_inspect_module.py
+# Timestamp: 2025-01-27
+# File: /home/ywatanabe/proj/scitex-python/src/scitex/introspect/_list_api.py
+
+"""Module API listing utilities."""
 
 import inspect
 import sys
 import warnings
 from typing import Any, List, Optional, Set, Union
 
-import scitex
 import pandas as pd
 
 
-def inspect_module(
+def list_api(
     module: Union[str, Any],
     columns: List[str] = ["Type", "Name", "Docstring", "Depth"],
     prefix: str = "",
@@ -26,41 +26,14 @@ def inspect_module(
     drop_duplicates: bool = True,
     root_only: bool = False,
 ) -> pd.DataFrame:
-    return _inspect_module(
-        module=module,
-        prefix=prefix,
-        max_depth=max_depth,
-        visited=visited,
-        docstring=docstring,
-        tree=tree,
-        current_depth=current_depth,
-        print_output=print_output,
-        skip_depwarnings=skip_depwarnings,
-        drop_duplicates=drop_duplicates,
-        root_only=root_only,
-    )[columns]
-
+    """
+    List the API of a module recursively and return as a DataFrame.
 
-def _inspect_module(
-    module: Union[str, Any],
-    columns: List[str] = ["Type", "Name", "Docstring", "Depth"],
-    prefix: str = "",
-    max_depth: int = 5,
-    visited: Optional[Set[str]] = None,
-    docstring: bool = False,
-    tree: bool = True,
-    current_depth: int = 0,
-    print_output: bool = False,
-    skip_depwarnings: bool = True,
-    drop_duplicates: bool = True,
-    root_only: bool = False,
-) -> pd.DataFrame:
-    """List the contents of a module recursively and return as a DataFrame.
+    Like a recursive `dir()` that shows the entire module tree.
 
     Example
     -------
-    >>>
-    >>> df = inspect_module(scitex)
+    >>> df = list_api(scitex)
     >>> print(df)
        Type           Name                    Docstring  Depth
     0    M            scitex  Module description              0
@@ -100,6 +73,36 @@ def _inspect_module(
     pd.DataFrame
         Module structure with specified columns
     """
+    return _list_api_impl(
+        module=module,
+        prefix=prefix,
+        max_depth=max_depth,
+        visited=visited,
+        docstring=docstring,
+        tree=tree,
+        current_depth=current_depth,
+        print_output=print_output,
+        skip_depwarnings=skip_depwarnings,
+        drop_duplicates=drop_duplicates,
+        root_only=root_only,
+    )[columns]
+
+
+def _list_api_impl(
+    module: Union[str, Any],
+    columns: List[str] = ["Type", "Name", "Docstring", "Depth"],
+    prefix: str = "",
+    max_depth: int = 5,
+    visited: Optional[Set[str]] = None,
+    docstring: bool = False,
+    tree: bool = True,
+    current_depth: int = 0,
+    print_output: bool = False,
+    skip_depwarnings: bool = True,
+    drop_duplicates: bool = True,
+    root_only: bool = False,
+) -> pd.DataFrame:
+    """Internal implementation of list_api."""
     if skip_depwarnings:
         warnings.filterwarnings("ignore", category=DeprecationWarning)
         warnings.filterwarnings("ignore", category=UserWarning)
@@ -150,7 +153,7 @@ def _inspect_module(
                         )
                     )
                     try:
-                        sub_df = _inspect_module(
+                        sub_df = _list_api_impl(
                             obj,
                             columns=columns,
                             prefix=full_path,
@@ -235,22 +238,8 @@ def _print_module_contents(df: pd.DataFrame) -> None:
 
 
 if __name__ == "__main__":
+    import scitex
+
     sys.setrecursionlimit(10_000)
-    df = inspect_module(scitex, docstring=True, print_output=False, columns=["Name"])
+    df = list_api(scitex, docstring=True, print_output=False, columns=["Name"])
     print(scitex.pd.round(df))
-    #                                 Name
-    # 0                               scitex
-    # 1                            scitex.ai
-    # 3     scitex.ai.ClassificationReporter
-    # 4           scitex.ai.ClassifierServer
-    # 5              scitex.ai.EarlyStopping
-    # ...                              ...
-    # 5373                     scitex.typing
-    # 5375                 scitex.typing.Any
-    # 5376            scitex.typing.Iterable
-    # 5377                        scitex.web
-    # 5379          scitex.web.summarize_url
-
-    # [5361 rows x 1 columns]
-
-# EOF