ommlds 0.0.0.dev448__py3-none-any.whl → 0.0.0.dev450__py3-none-any.whl

ommlds/minichain/lib/fs/binfiles.py

@@ -0,0 +1,108 @@
+import os.path
+import typing as ta
+
+
+##
+
+
+IMAGE_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'jpg',
+    'jpeg',
+    'png',
+    'gif',
+    'bmp',
+    'webp',
+])
+
+
+ARCHIVE_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'zip',
+    'tar',
+    'gz',
+    'xz',
+    '7z',
+])
+
+
+DOC_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'doc',
+    'docx',
+    'xls',
+    'xlsx',
+    'ppt',
+    'pptx',
+    'odt',
+    'ods',
+    'odp',
+])
+
+
+EXECUTABLE_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'exe',
+    'dll',
+    'so',
+
+    'obj',
+    'o',
+    'a',
+    'lib',
+
+    'class',
+    'jar',
+    'war',
+
+    'wasm',
+
+    'pyc',
+    'pyo',
+])
+
+
+BLOB_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'bin',
+    'dat',
+])
+
+
+BINARY_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    *IMAGE_FILE_EXTENSIONS,
+    *ARCHIVE_FILE_EXTENSIONS,
+    *DOC_FILE_EXTENSIONS,
+    *EXECUTABLE_FILE_EXTENSIONS,
+    *BLOB_FILE_EXTENSIONS,
+])
+
+
+def has_binary_file_extension(file_path: str) -> bool:
+    return os.path.basename(file_path).partition('.')[-1] in BINARY_FILE_EXTENSIONS
+
+
+##
+
+
+def is_binary_file(
+        file_path: str,
+        *,
+        chunk_size: int = 0x1000,
+        non_printable_cutoff: float = .3,
+
+        st: os.stat_result | None = None,
+) -> bool:
+    if st is None:
+        try:
+            st = os.stat(file_path)
+        except OSError:
+            return False
+
+    if not st.st_size:
+        return False
+
+    with open(file_path, 'rb') as f:
+        chunk = f.read(chunk_size)
+
+    if 0 in chunk:
+        return True
+
+    # Count "non-printable" ASCII-ish control chars (excluding TAB/LF/CR)
+    np = sum(1 for b in chunk if b < 9 or (13 < b < 32))
+    return (np / len(chunk)) > non_printable_cutoff

ommlds/minichain/lib/fs/context.py

@@ -0,0 +1,112 @@
+import os.path
+import stat
+
+from omlish import check
+
+from ...tools.execution.context import tool_context
+from .binfiles import has_binary_file_extension
+from .binfiles import is_binary_file
+from .errors import RequestedPathDoesNotExistError
+from .errors import RequestedPathOutsideRootDirError
+from .errors import RequestedPathWrongTypeError
+from .suggestions import get_path_suggestions
+
+
+##
+
+
+class FsContext:
+    def __init__(
+            self,
+            *,
+            root_dir: str | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._root_dir = root_dir
+        self._abs_root_dir = os.path.abspath(root_dir) if root_dir is not None else None
+
+    #
+
+    def check_requested_path(self, req_path: str) -> None:
+        abs_req_path = os.path.abspath(req_path)
+
+        if (
+            self._abs_root_dir is None or
+            not (
+                abs_req_path == self._abs_root_dir or
+                abs_req_path.startswith(self._abs_root_dir + os.path.sep)
+            )
+        ):
+            raise RequestedPathOutsideRootDirError(
+                req_path,
+                root_dir=check.not_none(self._root_dir),
+            )
+
+    #
+
+    def check_stat_dir(
+            self,
+            req_path: str,
+    ) -> os.stat_result:
+        self.check_requested_path(req_path)
+
+        try:
+            st = os.stat(req_path)
+        except FileNotFoundError:
+            raise RequestedPathDoesNotExistError(
+                req_path,
+                suggested_paths=get_path_suggestions(
+                    req_path,
+                    filter=lambda e: e.is_dir(),
+                ),
+            ) from None
+
+        if not stat.S_ISDIR(st.st_mode):
+            raise RequestedPathWrongTypeError(
+                req_path,
+                expected_type='dir',
+                **(dict(actual_type='file') if stat.S_ISREG(st.st_mode) else {}),
+            )
+
+        return st
+
+    def check_stat_file(
+            self,
+            req_path: str,
+            *,
+            text: bool = False,
+    ) -> os.stat_result:
+        self.check_requested_path(req_path)
+
+        try:
+            st = os.stat(req_path)
+        except FileNotFoundError:
+            raise RequestedPathDoesNotExistError(
+                req_path,
+                suggested_paths=get_path_suggestions(
+                    req_path,
+                    filter=lambda e: (e.is_file() and not (text and has_binary_file_extension(e.name))),
+                ),
+            ) from None
+
+        if not stat.S_ISREG(st.st_mode):
+            is_dir = stat.S_ISDIR(st.st_mode)
+            raise RequestedPathWrongTypeError(
+                req_path,
+                expected_type='file',
+                **(dict(actual_type='dir') if is_dir else {}),
+            )
+
+        if text and is_binary_file(req_path, st=st):
+            raise RequestedPathWrongTypeError(
+                req_path,
+                expected_type='text file',
+                actual_type='binary file',
+            )
+
+        return st
+
+
+def fs_tool_context() -> FsContext:
+    return tool_context()[FsContext]

ommlds/minichain/lib/fs/errors.py

@@ -0,0 +1,95 @@
+import typing as ta
+
+from omlish import lang
+
+from ...tools.execution.errors import ToolExecutionError
+
+
+##
+
+
+class FsToolExecutionError(ToolExecutionError, lang.Abstract):
+    pass
+
+
+##
+
+
+class RequestedPathError(FsToolExecutionError, lang.Abstract):
+    def __init__(self, requested_path: str, *args: ta.Any) -> None:
+        super().__init__(requested_path, *args)
+
+        self.requested_path = requested_path
+
+
+class RequestedPathMustBeAbsoluteError(RequestedPathError):
+    @property
+    def content(self) -> str:
+        return f'Requested path {self.requested_path!r} must be absolute.'
+
+
+class RequestedPathOutsideRootDirError(RequestedPathError):
+    def __init__(
+            self,
+            requested_path: str,
+            *,
+            root_dir: str,
+    ) -> None:
+        super().__init__(
+            requested_path,
+            root_dir,
+        )
+
+        self.root_dir = root_dir
+
+    @property
+    def content(self) -> str:
+        return f'Requested path {self.requested_path!r} was outside of permitted root directory {self.root_dir!r}.'
+
+
+class RequestedPathWrongTypeError(RequestedPathError):
+    def __init__(
+            self,
+            requested_path: str,
+            *,
+            expected_type: str,
+            actual_type: str | None = None,
+    ) -> None:
+        super().__init__(
+            requested_path,
+            expected_type,
+            actual_type,
+        )
+
+        self.expected_type = expected_type
+        self.actual_type = actual_type
+
+    @property
+    def content(self) -> str:
+        return ''.join([
+            f'Requested path {self.requested_path!r} must be of type {self.expected_type!r}',
+            *([f', but it is actually of type {self.actual_type!r}'] if self.actual_type is not None else []),
+            '.',
+        ])
+
+
+class RequestedPathDoesNotExistError(RequestedPathError):
+    def __init__(
+            self,
+            requested_path: str,
+            *,
+            suggested_paths: ta.Sequence[str] | None = None,
+    ) -> None:
+        super().__init__(
+            requested_path,
+            suggested_paths,
+        )
+
+        self.suggested_paths = suggested_paths
+
+    @property
+    def content(self) -> str:
+        return ''.join([
+            f'Requested path {self.requested_path!r} does not exist.',
+            *([f' Did you mean one of these valid paths: {self.suggested_paths!r}?'] if self.suggested_paths else []),
+        ])

ommlds/minichain/lib/fs/suggestions.py

@@ -0,0 +1,36 @@
+import difflib
+import os.path
+import typing as ta
+
+
+##
+
+
+def get_path_suggestions(
+        bad_path: str,
+        n: int = 3,
+        *,
+        filter: ta.Callable[[os.DirEntry], bool] | None = None,  # noqa
+        cutoff: float = .6,
+) -> ta.Sequence[str] | None:
+    dn = os.path.dirname(bad_path)
+    try:
+        sdi = os.scandir(dn)
+    except FileNotFoundError:
+        return None
+
+    fl = [
+        e.name
+        for e in sdi
+        if (filter is None or filter(e))
+    ]
+
+    return [
+        os.path.join(dn, sn)
+        for sn in difflib.get_close_matches(
+            os.path.basename(bad_path),
+            fl,
+            n,
+            cutoff=cutoff,
+        )
+    ]

ommlds/minichain/lib/fs/tools/ls.py

@@ -0,0 +1,38 @@
+import io
+import os
+
+from omlish import lang
+
+from ....tools.execution.catalog import ToolCatalogEntry
+from ....tools.execution.reflect import reflect_tool_catalog_entry
+from ..context import fs_tool_context
+
+
+##
+
+
+def execute_ls_tool(
+        dir_path: str,
+) -> str:
+    """
+    Lists the contents of the specified dir.
+
+    Args:
+        dir_path: The dir to list the contents of. Must be an absolute path.
+    """
+
+    ctx = fs_tool_context()
+    ctx.check_stat_dir(dir_path)
+
+    out = io.StringIO()
+    out.write('<dir>\n')
+    for e in sorted(os.scandir(dir_path), key=lambda e: e.name):  # noqa
+        out.write(f'{e.name}{"/" if e.is_dir() else ""}\n')
+    out.write('</dir>\n')
+
+    return out.getvalue()
+
+
+@lang.cached_function
+def ls_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_ls_tool)

ommlds/minichain/lib/fs/tools/read.py

@@ -0,0 +1,115 @@
+"""
+TODO:
+ - better bad unicode handling
+ - read whole file if < some size, report filesize / num lines / mtime inline
+ - fs cache
+ - track changes
+"""
+import io
+import itertools
+
+from omlish import lang
+
+from ....tools.execution.catalog import ToolCatalogEntry
+from ....tools.execution.reflect import reflect_tool_catalog_entry
+from ....tools.reflect import tool_spec_override
+from ....tools.types import ToolParam
+from ..context import fs_tool_context
+
+
+##
+
+
+DEFAULT_MAX_NUM_LINES = 2_000
+
+MAX_LINE_LENGTH = 2_000
+
+
+@tool_spec_override(
+    desc=f"""
+         Reads a file from the local filesystem. You can access any file directly by using this tool.
+
+        Assume this tool is able to read all files on the machine. If the User provides a path to a file assume that
+        path is valid. It is okay to read a file that does not exist; an error will be returned.
+
+        Usage:
+        - The file_path parameter must be an absolute path, not a relative path.
+        - By default, it reads up to {DEFAULT_MAX_NUM_LINES} lines starting from the beginning of the file.
+        - You can optionally specify a line offset and limit (especially handy for long files), but it's recommended to
+          read the whole file by not providing these parameters.
+        - Any lines longer than {MAX_LINE_LENGTH} characters will be truncated with "...".
+        - Invalid unicode characters will be replaced with the unicode replacement character "\\ufffd".
+        - Results are returned using cat -n format, with line numbers starting at 1 and suffixed with a pipe character
+          "|".
+        - This tool cannot read binary files, including images.
+    """,
+    params=[
+        ToolParam(
+            'file_path',
+            desc='The absolute path to the file to read.',
+        ),
+        ToolParam(
+            'line_offset',
+            desc='The line number to start reading from (0-based).',
+        ),
+        ToolParam(
+            'num_lines',
+            desc=f'The number of lines to read (defaults to {DEFAULT_MAX_NUM_LINES}).',
+        ),
+    ],
+)
+def execute_read_tool(
+        file_path: str,
+        *,
+        line_offset: int = 0,
+        num_lines: int = DEFAULT_MAX_NUM_LINES,
+) -> str:
+    ctx = fs_tool_context()
+    ctx.check_stat_file(file_path, text=True)
+
+    out = io.StringIO()
+    out.write('<file>\n')
+
+    zp = len(str(line_offset + num_lines))
+    n = line_offset
+    has_trunc = False  # noqa
+    with open(file_path, errors='replace') as f:
+        fi = iter(f)
+
+        for line in itertools.islice(fi, line_offset, line_offset + num_lines):
+            out.write(f'{str(n + 1).zfill(zp):}|')
+            line = line.removesuffix('\n')
+            if len(line) > MAX_LINE_LENGTH:
+                has_trunc = True  # noqa
+                out.write(line[:MAX_LINE_LENGTH])
+                out.write('...')
+            else:
+                out.write(line)
+            out.write('\n')
+            n += 1
+
+        # tl = n
+        # if (ml := lang.ilen(fi)):
+        #     check.state(n == num_lines)
+        #     tl += ml
+
+        try:
+            next(fi)
+        except StopIteration:
+            has_more = False
+        else:
+            has_more = True
+
+    out.write(f'</file>\n')
+
+    if has_more:
+        out.write(
+            f'\n(File has more lines. Use "line_offset" parameter to read beyond line {line_offset + num_lines}.)\n',
+        )
+
+    return out.getvalue()
+
+
+@lang.cached_function
+def read_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_read_tool)

ommlds/minichain/lib/fs/tools/recursivels/execution.py

@@ -0,0 +1,40 @@
+from omlish import lang
+
+from .....tools.execution.catalog import ToolCatalogEntry
+from .....tools.execution.reflect import reflect_tool_catalog_entry
+from ...context import fs_tool_context
+from .rendering import LsLinesRenderer
+from .running import LsRunner
+
+
+##
+
+
+def execute_recursive_ls_tool(
+        base_path: str,
+) -> str:
+    """
+    Recursively lists the directory contents of the given base path.
+
+    Args:
+        base_path: The path of the directory to list the contents of. Must be absolute.
+
+    Returns:
+        A formatted string of the recursive directory contents.
+    """
+
+    ft_ctx = fs_tool_context()
+    ft_ctx.check_requested_path(base_path)
+
+    root = LsRunner().run(base_path)
+    lines = LsLinesRenderer().render(root)
+    return '\n'.join([
+        '<dir>',
+        *lines.lines,
+        '</dir>',
+    ])
+
+
+@lang.cached_function
+def recursive_ls_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_recursive_ls_tool)

ommlds/minichain/lib/todo/context.py

@@ -0,0 +1,27 @@
+import typing as ta
+
+from ...tools.execution.context import tool_context
+from .types import TodoItem
+
+
+##
+
+
+class TodoContext:
+    def __init__(
+            self,
+            items: ta.Sequence[TodoItem] | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._items = items
+
+    def get_items(self) -> ta.Sequence[TodoItem] | None:
+        return self._items
+
+    def set_items(self, items: ta.Sequence[TodoItem] | None) -> None:
+        self._items = list(items) if items is not None else None
+
+
+def todo_tool_context() -> TodoContext:
+    return tool_context()[TodoContext]

ommlds/minichain/lib/todo/tools/read.py

@@ -0,0 +1,39 @@
+from omlish import lang
+from omlish.formats import json
+
+from ....tools.execution.catalog import ToolCatalogEntry
+from ....tools.execution.reflect import reflect_tool_catalog_entry
+from ....tools.reflect import tool_spec_override
+from ..context import todo_tool_context
+
+
+##
+
+
+@tool_spec_override(
+    desc="""
+        Use this tool to read the current todo list for your current session. This tool should be used proactively and
+        frequently to ensure that you are aware of the status of the current subtask list.
+
+        You should make use of this tool often, especially in the following situations:
+        - At the beginning of conversations to see what's pending.
+        - Before starting new tasks to prioritize work.
+        - When the user asks about previous tasks or plans.
+        - Whenever you're uncertain about what to do next.
+        - After completing tasks to update your understanding of remaining work.
+        - After every few messages to ensure you're on track.
+
+        Usage:
+        - Returns a list of todo items in json format with their id, status, priority, and content.
+        - Use this information to track progress and plan next steps.
+    """,
+)
+def execute_todo_read_tool() -> str:
+    ctx = todo_tool_context()
+
+    return json.dumps_compact(ctx.get_items() or [])
+
+
+@lang.cached_function
+def todo_read_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_todo_read_tool)