skylos 1.0.10__tar.gz → 1.0.11__tar.gz

{skylos-1.0.10 → skylos-1.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skylos
-Version: 1.0.10
+Version: 1.0.11
 Summary: A static analysis tool for Python codebases
 Author-email: oha <aaronoh2015@gmail.com>
 Requires-Python: >=3.9

{skylos-1.0.10 → skylos-1.0.11}/README.md

@@ -212,7 +212,7 @@ We welcome contributions! Please read our [Contributing Guidelines](CONTRIBUTING
 5. Open a Pull Request
 
 ## Roadmap
-
+- [ ] Add a production flag, to include dead codes that are used in test but not in the actual execution 
 - [ ] Expand our test cases
 - [ ] Configuration file support 
 - [ ] Custom analysis rules

{skylos-1.0.10 → skylos-1.0.11}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "skylos"
-version = "1.0.10"
+version = "1.0.11"
 requires-python = ">=3.9"
 description = "A static analysis tool for Python codebases"
 authors = [{name = "oha", email = "aaronoh2015@gmail.com"}]

{skylos-1.0.10 → skylos-1.0.11}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="skylos",
-    version="1.0.10",
+    version="1.0.11",
     packages=find_packages(),
     python_requires=">=3.9",
     install_requires=["inquirer>=3.0.0"],

{skylos-1.0.10 → skylos-1.0.11}/skylos/__init__.py

@@ -1,6 +1,6 @@
 from skylos.analyzer import analyze
 
-__version__ = "1.0.10"
+__version__ = "1.0.11"
 
 def debug_test():
     return "debug-ok"

{skylos-1.0.10 → skylos-1.0.11}/skylos/analyzer.py

@@ -97,6 +97,12 @@ class Skylos:
             if d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__")):
                 d.confidence = 0
             
+            if d.type == "parameter" and d.simple_name in ("self", "cls"):
+                d.confidence = 0
+
+            if d.type != "parameter" and (d.simple_name in MAGIC_METHODS or (d.simple_name.startswith("__") and d.simple_name.endswith("__"))):
+                d.confidence = 0
+
             if not d.simple_name.startswith("_") and d.type in ("function", "method", "class"):
                 d.confidence = min(d.confidence, 90)
             
@@ -106,6 +112,9 @@ class Skylos:
             if d.name.split(".")[0] in self.dynamic:
                 d.confidence = min(d.confidence, 60)
             
+            if d.type == "variable" and d.simple_name == "_":
+                d.confidence = 0
+            
             if d.type == "method" and TEST_METHOD_PATTERN.match(d.simple_name):
                 class_name = d.name.rsplit(".", 1)[0]
                 class_simple_name = class_name.split(".")[-1]
@@ -134,7 +143,7 @@ class Skylos:
         self._mark_refs()
         self._apply_heuristics()
         self._mark_exports()
-        
+                
         thr = max(0, thr)
 
         unused = []
@@ -146,7 +155,8 @@ class Skylos:
             "unused_functions": [], 
             "unused_imports": [], 
             "unused_classes": [],
-            "unused_variables": []
+            "unused_variables": [],
+            "unused_parameters": []
         }
         
         for u in unused:
@@ -158,6 +168,8 @@ class Skylos:
                 result["unused_classes"].append(u)
             elif u["type"] == "variable":
                 result["unused_variables"].append(u)
+            elif u["type"] == "parameter":
+                result["unused_parameters"].append(u)
                 
         return json.dumps(result, indent=2)
 

{skylos-1.0.10 → skylos-1.0.11}/skylos/cli.py

@@ -247,6 +247,8 @@ def main() -> None:
 
     unused_functions = result.get("unused_functions", [])
     unused_imports = result.get("unused_imports", [])
+    unused_parameters = result.get("unused_parameters", [])
+    unused_variables = result.get("unused_variables", [])
     
     logger.info(f"{Colors.CYAN}{Colors.BOLD}🔍 Python Static Analysis Results{Colors.RESET}")
     logger.info(f"{Colors.CYAN}{'=' * 35}{Colors.RESET}")
@@ -254,7 +256,10 @@ def main() -> None:
     logger.info(f"\n{Colors.BOLD}Summary:{Colors.RESET}")
     logger.info(f"  • Unreachable functions: {Colors.YELLOW}{len(unused_functions)}{Colors.RESET}")
     logger.info(f"  • Unused imports: {Colors.YELLOW}{len(unused_imports)}{Colors.RESET}")
-    
+    logger.info(f"  • Unused parameters: {Colors.YELLOW}{len(unused_parameters)}{Colors.RESET}")
+    logger.info(f"  • Unused variables: {Colors.YELLOW}{len(unused_variables)}{Colors.RESET}")
+
+
     if args.interactive and (unused_functions or unused_imports):
         logger.info(f"\n{Colors.BOLD}Interactive Mode:{Colors.RESET}")
         selected_functions, selected_imports = interactive_selection(logger, unused_functions, unused_imports)
@@ -324,6 +329,24 @@ def main() -> None:
         else:
             logger.info(f"\n{Colors.GREEN}✓ All imports are being used!{Colors.RESET}")
         
+        if unused_parameters:
+            logger.info(f"\n{Colors.BLUE}{Colors.BOLD}🔧 Unused Parameters{Colors.RESET}")
+            logger.info(f"{Colors.BLUE}{'=' * 18}{Colors.RESET}")
+            for i, item in enumerate(unused_parameters, 1):
+                logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.BLUE}{item['name']}{Colors.RESET}")
+                logger.info(f"    {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
+        else:
+            logger.info(f"\n{Colors.GREEN}✓ All parameters are being used!{Colors.RESET}")
+        
+        if unused_variables:
+            logger.info(f"\n{Colors.YELLOW}{Colors.BOLD}📊 Unused Variables{Colors.RESET}")
+            logger.info(f"{Colors.YELLOW}{'=' * 18}{Colors.RESET}")
+            for i, item in enumerate(unused_variables, 1):
+                logger.info(f"{Colors.GRAY}{i:2d}. {Colors.RESET}{Colors.YELLOW}{item['name']}{Colors.RESET}")
+                logger.info(f"    {Colors.GRAY}└─ {item['file']}:{item['line']}{Colors.RESET}")
+        else:
+            logger.info(f"\n{Colors.GREEN}✓ All variables are being used!{Colors.RESET}")
+
         dead_code_count = len(unused_functions) + len(unused_imports)
         print_badge(dead_code_count, logger)
 

{skylos-1.0.10 → skylos-1.0.11}/skylos/visitor.py

@@ -52,6 +52,7 @@ class Visitor(ast.NodeVisitor):
         self.dyn=set()
         self.exports=set()
         self.current_function_scope = []
+        self.current_function_params = []
 
     def add_def(self,n,t,l):
         if n not in{d.name for d in self.defs}:self.defs.append(Definition(n,t,self.file,l))
@@ -85,17 +86,27 @@ class Visitor(ast.NodeVisitor):
             self.alias[a.asname or a.name.split(".")[-1]]=full
             self.add_def(full,"import",node.lineno)
 
-    def visit_ImportFrom(self,node):
-        if node.module is None:return
+    def visit_ImportFrom(self, node):
+        if node.module is None:
+            return
         for a in node.names:
-            if a.name=="*":continue
-            base=node.module
+            if a.name == "*":
+                continue
+            base = node.module
             if node.level:
-                parts=self.mod.split(".")
-                base=".".join(parts[:-node.level])+(f".{node.module}"if node.module else"")
-            full=f"{base}.{a.name}"
-            self.alias[a.asname or a.name]=full
-            self.add_def(full,"import",node.lineno)
+                parts = self.mod.split(".")
+                base = ".".join(parts[:-node.level]) + (f".{node.module}" if node.module else "")
+            
+            full = f"{base}.{a.name}"
+            
+            if a.asname:
+                alias_full = f"{self.mod}.{a.asname}" if self.mod else a.asname
+                self.add_def(alias_full, "import", node.lineno)
+                self.alias[a.asname] = full
+                self.add_ref(full)
+            else:
+                self.alias[a.name] = full
+                self.add_def(full, "import", node.lineno)
 
     def visit_arguments(self, args):
         for arg in args.args:
@@ -128,15 +139,25 @@ class Visitor(ast.NodeVisitor):
         
         self.current_function_scope.append(node.name)
         
+        old_params = self.current_function_params
+        self.current_function_params = []
+        
         for d_node in node.decorator_list:
             self.visit(d_node)
         
+        for arg in node.args.args:
+            param_name = f"{qualified_name}.{arg.arg}"
+            self.add_def(param_name, "parameter", node.lineno)
+            self.current_function_params.append((arg.arg, param_name))
+        
         self.visit_arguments(node.args)
         self.visit_annotation(node.returns)
         
         for stmt in node.body:
             self.visit(stmt)
+            
         self.current_function_scope.pop()
+        self.current_function_params = old_params
         
     visit_AsyncFunctionDef=visit_FunctionDef
 
@@ -178,6 +199,30 @@ class Visitor(ast.NodeVisitor):
             self.visit(node.step)
 
     def visit_Assign(self, node):
+        def process_target_for_def(target_node):
+            if isinstance(target_node, ast.Name):
+                var_name_simple = target_node.id
+                if var_name_simple == "__all__" and not self.current_function_scope and not self.cls:
+                    return
+
+                scope_parts = [self.mod]
+                if self.cls:
+                    scope_parts.append(self.cls)
+                if self.current_function_scope:
+                    scope_parts.extend(self.current_function_scope)
+                
+                prefix = '.'.join(filter(None, scope_parts))
+                qualified_var_name = f"{prefix}.{var_name_simple}" if prefix else var_name_simple
+
+                self.add_def(qualified_var_name, "variable", target_node.lineno)
+
+            elif isinstance(target_node, (ast.Tuple, ast.List)):
+                for elt in target_node.elts:
+                    process_target_for_def(elt)
+
+        for t in node.targets:
+            process_target_for_def(t)
+        
         for target in node.targets:
             if isinstance(target, ast.Name) and target.id == "__all__":
                 if isinstance(node.value, (ast.List, ast.Tuple)):
@@ -189,9 +234,10 @@ class Visitor(ast.NodeVisitor):
                             value = elt.s
                             
                         if value is not None:
-                            full_name = f"{self.mod}.{value}"
-                            self.add_ref(full_name)
-                            self.add_ref(value)
+                            full_name_export = f"{self.mod}.{value}" if self.mod else value
+                            self.add_ref(full_name_export)
+                            self.add_ref(value) 
+        
         self.generic_visit(node)
 
     def visit_Call(self, node):
@@ -219,12 +265,26 @@ class Visitor(ast.NodeVisitor):
 
     def visit_Name(self,node):
         if isinstance(node.ctx,ast.Load):
-            self.add_ref(self.qual(node.id))
-            if node.id in DYNAMIC_PATTERNS:self.dyn.add(self.mod.split(".")[0])
+            for param_name, param_full_name in self.current_function_params:
+                if node.id == param_name:
+                    self.add_ref(param_full_name)
+                    break
+            else:
+                # not parameter, handle normally
+                self.add_ref(self.qual(node.id))
+                if node.id in DYNAMIC_PATTERNS:
+                    self.dyn.add(self.mod.split(".")[0])
 
-    def visit_Attribute(self,node):
+    def visit_Attribute(self, node):
         self.generic_visit(node)
-        if isinstance(node.ctx,ast.Load)and isinstance(node.value,ast.Name):
+        if isinstance(node.ctx, ast.Load) and isinstance(node.value, ast.Name):
+            if node.value.id in [param_name for param_name, _ in self.current_function_params]:
+                # mark parameter as referenced
+                for param_name, param_full_name in self.current_function_params:
+                    if node.value.id == param_name:
+                        self.add_ref(param_full_name)
+                        break
+
             self.add_ref(f"{self.qual(node.value.id)}.{node.attr}")
 
     def visit_keyword(self, node):

{skylos-1.0.10 → skylos-1.0.11}/skylos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skylos
-Version: 1.0.10
+Version: 1.0.11
 Summary: A static analysis tool for Python codebases
 Author-email: oha <aaronoh2015@gmail.com>
 Requires-Python: >=3.9

{skylos-1.0.10 → skylos-1.0.11}/skylos.egg-info/SOURCES.txt

@@ -16,6 +16,15 @@ test/compare_tools.py
 test/diagnostics.py
 test/test_skylos.py
 test/test_visitor.py
+test/pykomodo/__init__.py
+test/pykomodo/command_line.py
+test/pykomodo/config.py
+test/pykomodo/core.py
+test/pykomodo/dashboard.py
+test/pykomodo/enhanced_chunker.py
+test/pykomodo/multi_dirs_chunker.py
+test/pykomodo/pykomodo_config.py
+test/pykomodo/token_chunker.py
 test/sample_repo/__init__.py
 test/sample_repo/app.py
 test/sample_repo/sample_repo/__init__.py

skylos-1.0.11/test/pykomodo/command_line.py

@@ -0,0 +1,176 @@
+import sys
+import argparse
+import os
+
+KOMODO_VERSION = "0.2.5"
+
+def launch_dashboard():
+    """Launch the dashboard interface."""
+    try:
+        from pykomodo.dashboard import launch_dashboard
+        print("Starting Komodo Dashboard...")
+        demo = launch_dashboard()
+        demo.launch(
+            server_name="0.0.0.0", 
+            server_port=7860,
+            share=False,
+            debug=False
+        )
+    except ImportError as e:
+        print(f"[Error] Dashboard dependencies not available: {e}", file=sys.stderr)
+        print("Please install gradio: pip install gradio", file=sys.stderr)
+        sys.exit(1)
+    except Exception as e:
+        print(f"[Error] Failed to launch dashboard: {e}", file=sys.stderr)
+        sys.exit(1)
+
+def main():
+    """Main entry point for the komodo CLI."""
+    parser = argparse.ArgumentParser(
+        description="Process and chunk codebase content with advanced chunking strategies."
+    )
+
+    parser.add_argument("--version", action="version", version=f"komodo {KOMODO_VERSION}")
+    
+    parser.add_argument("--dashboard", action="store_true",
+                        help="Launch the web-based dashboard interface")
+
+    parser.add_argument("dirs", nargs="*", default=["."],
+                        help="Directories to process (default: current directory)")
+    
+    chunk_group = parser.add_mutually_exclusive_group(required=False)
+    chunk_group.add_argument("--equal-chunks", type=int, 
+                            help="Split into N equal chunks")
+    chunk_group.add_argument("--max-chunk-size", type=int, 
+                            help="Maximum tokens/lines per chunk")
+    chunk_group.add_argument("--max-tokens", type=int,
+                            help="Maximum tokens per chunk (token-based chunking)")
+    
+    parser.add_argument("--output-dir", default="chunks",
+                        help="Output directory for chunks (default: chunks)")
+    
+    parser.add_argument("--ignore", action="append", default=[],
+                        help="Repeatable. Each usage adds one ignore pattern. Example: --ignore '**/node_modules/**' --ignore 'venv'")
+    parser.add_argument("--unignore", action="append", default=[],
+                        help="Repeatable. Each usage adds one unignore pattern. Example: --unignore '*.md'")
+    
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Show which files would be processed, but do not generate any chunks.")
+
+    parser.add_argument("--priority", action="append", default=[],
+                        help="Priority rules in format 'pattern,score' (repeatable). Example: --priority '*.py,10' --priority 'file2.txt,20'")
+    
+    parser.add_argument("--num-threads", type=int, default=4,
+                        help="Number of processing threads (default: 4)")
+
+    parser.add_argument("--enhanced", action="store_true",
+                        help="Enable LLM optimizations")
+    
+    parser.add_argument("--semantic-chunks", action="store_true",
+                        help="Use AST-based chunking for .py files (splits by top-level functions/classes)")
+
+    parser.add_argument("--context-window", type=int, default=4096,
+                        help="Target LLM context window size (default: 4096)")
+    parser.add_argument("--min-relevance", type=float, default=0.3,
+                        help="Minimum relevance score 0.0-1.0 (default: 0.3)")
+    parser.add_argument("--no-metadata", action="store_true",
+                        help="Disable metadata extraction")
+    parser.add_argument("--keep-redundant", action="store_true",
+                        help="Keep redundant content")
+    parser.add_argument("--no-summaries", action="store_true",
+                        help="Disable summary generation")
+
+    parser.add_argument("--file-type", type=str, 
+                        help="Only chunk files of this type (e.g., 'pdf', 'py')")
+                        
+    parser.add_argument("--verbose", action="store_true",
+                        help="Enable verbose output")
+
+    args = parser.parse_args()
+
+    if args.dashboard:
+        launch_dashboard()
+        return
+
+    if not any([args.equal_chunks, args.max_chunk_size, args.max_tokens]):
+        parser.error("One of --equal-chunks, --max-chunk-size, or --max-tokens is required (unless using --dashboard)")
+
+    if args.output_dir:
+        os.makedirs(args.output_dir, exist_ok=True)
+
+    priority_rules = []
+    for rule in args.priority:
+        if not rule:
+            continue
+        try:
+            pattern, score = rule.split(",", 1)
+            priority_rules.append((pattern.strip(), int(score.strip())))
+        except ValueError:
+            print(f"[Error] Priority rule must be 'pattern,score': {rule}", 
+                  file=sys.stderr)
+            sys.exit(1)
+
+    chunker = None
+    try:
+        if args.max_tokens:
+            try:
+                from pykomodo.token_chunker import TokenBasedChunker as ChunkerClass
+                if args.verbose:
+                    print("Using TokenBasedChunker for token-based chunking")
+            except ImportError:
+                print("[Error] TokenBasedChunker not available. Please install tiktoken or update pykomodo.", 
+                      file=sys.stderr)
+                sys.exit(1)
+                
+            chunker_args = {
+                "max_tokens_per_chunk": args.max_tokens,
+                "output_dir": args.output_dir,
+                "user_ignore": args.ignore,
+                "user_unignore": args.unignore,
+                "priority_rules": priority_rules,
+                "num_threads": args.num_threads,
+                "dry_run": args.dry_run,
+                "semantic_chunking": args.semantic_chunks,
+                "file_type": args.file_type,
+                "verbose": args.verbose
+            }
+        else:
+            if args.enhanced:
+                from pykomodo.enhanced_chunker import EnhancedParallelChunker as ChunkerClass
+            else:
+                from pykomodo.multi_dirs_chunker import ParallelChunker as ChunkerClass
+                
+            chunker_args = {
+                "equal_chunks": args.equal_chunks,
+                "max_chunk_size": args.max_chunk_size,
+                "output_dir": args.output_dir,
+                "user_ignore": args.ignore,
+                "user_unignore": args.unignore,
+                "priority_rules": priority_rules,
+                "num_threads": args.num_threads,
+                "dry_run": args.dry_run,
+                "semantic_chunking": args.semantic_chunks,
+                "file_type": args.file_type
+            }
+            
+            if args.enhanced:
+                chunker_args.update({
+                    "extract_metadata": not args.no_metadata,
+                    "add_summaries": not args.no_summaries,
+                    "remove_redundancy": not args.keep_redundant,
+                    "context_window": args.context_window,
+                    "min_relevance_score": args.min_relevance
+                })
+    
+        chunker = ChunkerClass(**chunker_args)
+        chunker.process_directories(args.dirs)
+        
+    except Exception as e:
+        print(f"[Error] Processing failed: {e}", file=sys.stderr)
+        sys.exit(1)
+    finally:
+        if chunker and hasattr(chunker, 'close'):
+            chunker.close()
+
+if __name__ == "__main__":
+    main()

skylos-1.0.11/test/pykomodo/config.py

@@ -0,0 +1,20 @@
+# src/config.py
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+@dataclass
+class PriorityRule:
+    pattern: str
+    score: int
+
+@dataclass
+class KomodoConfig:
+    max_size: int = 10 * 1024 * 1024
+    token_mode: bool = False
+    output_dir: Optional[Path] = None
+    stream: bool = False
+    ignore_patterns: list[str] = None
+    priority_rules: list[PriorityRule] = None
+    binary_extensions: list[str] = None

skylos-1.0.11/test/pykomodo/core.py

@@ -0,0 +1,121 @@
+import os
+import fnmatch
+from typing import List, Optional
+
+class PriorityRule:
+    """
+    Simple Python container for (pattern, score).
+    """
+    def __init__(self, pattern, score):
+        self.pattern: str = pattern
+        self.score: int = score
+
+class PyCConfig:
+    """
+    A pure Python equivalent of the 'PyCConfig' that in Cython
+    wrapped the 'CConfig' struct. This class maintains the same
+    conceptual fields but in Pythonic form (lists, strings, booleans).
+    """
+
+    def __init__(self):
+        self.max_size: int = 0
+        self.token_mode: bool = False
+        self.output_dir: Optional[str] = None
+        self.stream: bool = False
+        
+        self.ignore_patterns: List[str] = []
+        self.unignore_patterns: List[str] = [] 
+        self.priority_rules: List[PriorityRule] = []
+        self.binary_exts: List[str] = []
+
+    def add_ignore_pattern(self, pattern: str) -> None:
+        """
+        Just appends to a Python list.
+        """
+        self.ignore_patterns.append(pattern)
+
+    def add_unignore_pattern(self, pattern: str) -> None:
+        self.unignore_patterns.append(pattern)
+
+    def add_priority_rule(self, pattern: str, score: int) -> None:
+        self.priority_rules.append(PriorityRule(pattern, score))
+
+    def should_ignore(self, path: str) -> bool:
+        """
+        Return True if path matches one of the ignore_patterns,
+        unless it matches unignore_patterns first.
+        """
+        for pat in self.unignore_patterns:
+            if fnmatch.fnmatch(path, pat):
+                return False
+
+        for pat in self.ignore_patterns:
+            if fnmatch.fnmatch(path, pat):
+                return True
+
+        return False
+
+    def calculate_priority(self, path: str) -> int:
+        """
+        Returns the highest score among any matching priority rule.
+        """
+        highest = 0
+        for rule in self.priority_rules:
+            if fnmatch.fnmatch(path, rule.pattern):
+                if rule.score > highest:
+                    highest = rule.score
+        return highest
+
+    def is_binary_file(self, path: str) -> bool:
+        """
+        1) If extension is in self.binary_exts -> True
+        2) Else read up to 512 bytes, if it has a null byte -> True
+        3) If can't open -> True
+        """
+        _, ext = os.path.splitext(path)
+        ext = ext.lstrip(".").lower()
+        if ext in (b.lower() for b in self.binary_exts):
+            return True
+
+        try:
+            with open(path, "rb") as f:
+                chunk = f.read(512)
+        except OSError:
+            return True
+
+        if b"\0" in chunk:
+            return True
+
+        return False
+
+    def read_file_contents(self, path: str) -> str:
+        """
+        Reads the entire file as text, returns it.
+        If can't open, return "<NULL>" or handle differently.
+        """
+        try:
+            with open(path, "rb") as f:
+                data = f.read()
+            return data.decode("utf-8", errors="replace")
+        except OSError:
+            return "<NULL>"
+
+    def count_tokens(self, text: str) -> int:
+        """
+        Replicates py_count_tokens:
+        Simple whitespace-based token counting in pure Python.
+        """
+        return len(text.split())
+
+    def make_c_string(self, text: Optional[str]) -> str:
+        if text is None:
+            return "<NULL>"
+        return text
+
+    def __repr__(self) -> str:
+        return (f"PyCConfig(max_size={self.max_size}, token_mode={self.token_mode}, "
+                f"output_dir={self.output_dir!r}, stream={self.stream}, "
+                f"ignore_patterns={self.ignore_patterns}, "
+                f"unignore_patterns={self.unignore_patterns}, "
+                f"priority_rules={[ (r.pattern, r.score) for r in self.priority_rules ]}, "
+                f"binary_exts={self.binary_exts})")