ommlds 0.0.0.dev436__py3-none-any.whl → 0.0.0.dev480__py3-none-any.whl

ommlds/minichain/lib/fs/binfiles.py

@@ -0,0 +1,108 @@
+import os.path
+import typing as ta
+
+
+##
+
+
+IMAGE_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'jpg',
+    'jpeg',
+    'png',
+    'gif',
+    'bmp',
+    'webp',
+])
+
+
+ARCHIVE_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'zip',
+    'tar',
+    'gz',
+    'xz',
+    '7z',
+])
+
+
+DOC_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'doc',
+    'docx',
+    'xls',
+    'xlsx',
+    'ppt',
+    'pptx',
+    'odt',
+    'ods',
+    'odp',
+])
+
+
+EXECUTABLE_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'exe',
+    'dll',
+    'so',
+
+    'obj',
+    'o',
+    'a',
+    'lib',
+
+    'class',
+    'jar',
+    'war',
+
+    'wasm',
+
+    'pyc',
+    'pyo',
+])
+
+
+BLOB_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    'bin',
+    'dat',
+])
+
+
+BINARY_FILE_EXTENSIONS: ta.AbstractSet[str] = frozenset([
+    *IMAGE_FILE_EXTENSIONS,
+    *ARCHIVE_FILE_EXTENSIONS,
+    *DOC_FILE_EXTENSIONS,
+    *EXECUTABLE_FILE_EXTENSIONS,
+    *BLOB_FILE_EXTENSIONS,
+])
+
+
+def has_binary_file_extension(file_path: str) -> bool:
+    return os.path.basename(file_path).partition('.')[-1] in BINARY_FILE_EXTENSIONS
+
+
+##
+
+
+def is_binary_file(
+        file_path: str,
+        *,
+        chunk_size: int = 0x1000,
+        non_printable_cutoff: float = .3,
+
+        st: os.stat_result | None = None,
+) -> bool:
+    if st is None:
+        try:
+            st = os.stat(file_path)
+        except OSError:
+            return False
+
+    if not st.st_size:
+        return False
+
+    with open(file_path, 'rb') as f:
+        chunk = f.read(chunk_size)
+
+    if 0 in chunk:
+        return True
+
+    # Count "non-printable" ASCII-ish control chars (excluding TAB/LF/CR)
+    np = sum(1 for b in chunk if b < 9 or (13 < b < 32))
+    return (np / len(chunk)) > non_printable_cutoff

ommlds/minichain/lib/fs/context.py

@@ -0,0 +1,126 @@
+import os.path
+import stat
+
+from omlish import check
+
+from ...tools.execution.context import tool_context
+from .binfiles import has_binary_file_extension
+from .binfiles import is_binary_file
+from .errors import RequestedPathDoesNotExistError
+from .errors import RequestedPathOutsideRootDirError
+from .errors import RequestedPathWriteNotPermittedError
+from .errors import RequestedPathWrongTypeError
+from .suggestions import get_path_suggestions
+
+
+##
+
+
+class FsContext:
+    def __init__(
+            self,
+            *,
+            root_dir: str | None = None,
+            writes_permitted: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self._root_dir = root_dir
+        self._writes_permitted = writes_permitted
+
+        self._abs_root_dir = os.path.abspath(root_dir) if root_dir is not None else None
+
+    #
+
+    @property
+    def writes_permitted(self) -> bool:
+        return self._writes_permitted
+
+    #
+
+    def check_requested_path(self, req_path: str) -> None:
+        abs_req_path = os.path.abspath(req_path)
+
+        if (
+            self._abs_root_dir is None or
+            not (
+                abs_req_path == self._abs_root_dir or
+                abs_req_path.startswith(self._abs_root_dir + os.path.sep)
+            )
+        ):
+            raise RequestedPathOutsideRootDirError(
+                req_path,
+                root_dir=check.not_none(self._root_dir),
+            )
+
+    #
+
+    def check_stat_dir(
+            self,
+            req_path: str,
+    ) -> os.stat_result:
+        self.check_requested_path(req_path)
+
+        try:
+            st = os.stat(req_path)
+        except FileNotFoundError:
+            raise RequestedPathDoesNotExistError(
+                req_path,
+                suggested_paths=get_path_suggestions(
+                    req_path,
+                    filter=lambda e: e.is_dir(),
+                ),
+            ) from None
+
+        if not stat.S_ISDIR(st.st_mode):
+            raise RequestedPathWrongTypeError(
+                req_path,
+                expected_type='dir',
+                **(dict(actual_type='file') if stat.S_ISREG(st.st_mode) else {}),
+            )
+
+        return st
+
+    def check_stat_file(
+            self,
+            req_path: str,
+            *,
+            text: bool = False,
+            write: bool = False,
+    ) -> os.stat_result:
+        self.check_requested_path(req_path)
+
+        try:
+            st = os.stat(req_path)
+        except FileNotFoundError:
+            raise RequestedPathDoesNotExistError(
+                req_path,
+                suggested_paths=get_path_suggestions(
+                    req_path,
+                    filter=lambda e: (e.is_file() and not (text and has_binary_file_extension(e.name))),
+                ),
+            ) from None
+
+        if not stat.S_ISREG(st.st_mode):
+            is_dir = stat.S_ISDIR(st.st_mode)
+            raise RequestedPathWrongTypeError(
+                req_path,
+                expected_type='file',
+                **(dict(actual_type='dir') if is_dir else {}),
+            )
+
+        if text and is_binary_file(req_path, st=st):
+            raise RequestedPathWrongTypeError(
+                req_path,
+                expected_type='text file',
+                actual_type='binary file',
+            )
+
+        if write and not self._writes_permitted:
+            raise RequestedPathWriteNotPermittedError(req_path)
+
+        return st
+
+
+def tool_fs_context() -> FsContext:
+    return tool_context()[FsContext]

ommlds/minichain/lib/fs/errors.py

@@ -0,0 +1,101 @@
+import typing as ta
+
+from omlish import lang
+
+from ...tools.execution.errors import ToolExecutionError
+
+
+##
+
+
+class FsToolExecutionError(ToolExecutionError, lang.Abstract):
+    pass
+
+
+##
+
+
+class RequestedPathError(FsToolExecutionError, lang.Abstract):
+    def __init__(self, requested_path: str, *args: ta.Any) -> None:
+        super().__init__(requested_path, *args)
+
+        self.requested_path = requested_path
+
+
+class RequestedPathMustBeAbsoluteError(RequestedPathError):
+    @property
+    def content(self) -> str:
+        return f'Requested path {self.requested_path!r} must be absolute.'
+
+
+class RequestedPathOutsideRootDirError(RequestedPathError):
+    def __init__(
+            self,
+            requested_path: str,
+            *,
+            root_dir: str,
+    ) -> None:
+        super().__init__(
+            requested_path,
+            root_dir,
+        )
+
+        self.root_dir = root_dir
+
+    @property
+    def content(self) -> str:
+        return f'Requested path {self.requested_path!r} was outside of permitted root directory {self.root_dir!r}.'
+
+
+class RequestedPathWrongTypeError(RequestedPathError):
+    def __init__(
+            self,
+            requested_path: str,
+            *,
+            expected_type: str,
+            actual_type: str | None = None,
+    ) -> None:
+        super().__init__(
+            requested_path,
+            expected_type,
+            actual_type,
+        )
+
+        self.expected_type = expected_type
+        self.actual_type = actual_type
+
+    @property
+    def content(self) -> str:
+        return ''.join([
+            f'Requested path {self.requested_path!r} must be of type {self.expected_type!r}',
+            *([f', but it is actually of type {self.actual_type!r}'] if self.actual_type is not None else []),
+            '.',
+        ])
+
+
+class RequestedPathDoesNotExistError(RequestedPathError):
+    def __init__(
+            self,
+            requested_path: str,
+            *,
+            suggested_paths: ta.Sequence[str] | None = None,
+    ) -> None:
+        super().__init__(
+            requested_path,
+            suggested_paths,
+        )
+
+        self.suggested_paths = suggested_paths
+
+    @property
+    def content(self) -> str:
+        return ''.join([
+            f'Requested path {self.requested_path!r} does not exist.',
+            *([f' Did you mean one of these valid paths: {self.suggested_paths!r}?'] if self.suggested_paths else []),
+        ])
+
+
+class RequestedPathWriteNotPermittedError(RequestedPathError):
+    @property
+    def content(self) -> str:
+        return f'Writes are not permitted to requested path {self.requested_path!r}.'

ommlds/minichain/lib/fs/suggestions.py

@@ -0,0 +1,36 @@
+import difflib
+import os.path
+import typing as ta
+
+
+##
+
+
+def get_path_suggestions(
+        bad_path: str,
+        n: int = 3,
+        *,
+        filter: ta.Callable[[os.DirEntry], bool] | None = None,  # noqa
+        cutoff: float = .6,
+) -> ta.Sequence[str] | None:
+    dn = os.path.dirname(bad_path)
+    try:
+        sdi = os.scandir(dn)
+    except FileNotFoundError:
+        return None
+
+    fl = [
+        e.name
+        for e in sdi
+        if (filter is None or filter(e))
+    ]
+
+    return [
+        os.path.join(dn, sn)
+        for sn in difflib.get_close_matches(
+            os.path.basename(bad_path),
+            fl,
+            n,
+            cutoff=cutoff,
+        )
+    ]

ommlds/minichain/lib/fs/tools/edit.py

@@ -0,0 +1,104 @@
+"""
+TODO:
+ - must read file before editing
+ - must re-read file if file has been modified
+ - loosened replacer helpers
+ - accept diff format impl
+ - injectable confirmation, diff format
+"""
+from omlish import lang
+
+from ....tools.execution.catalog import ToolCatalogEntry
+from ....tools.execution.reflect import reflect_tool_catalog_entry
+from ..context import tool_fs_context
+from ..errors import RequestedPathError
+
+
+##
+
+
+class EditToolError(RequestedPathError, lang.Abstract):
+    pass
+
+
+class EmptyNewStringError(EditToolError):
+    @property
+    def content(self) -> str:
+        return f'The requested edit to {self.requested_path!r} was given an empty "old_string" parameter.'
+
+
+class OldStringNotPresentError(EditToolError):
+    @property
+    def content(self) -> str:
+        return f'The requested edit to {self.requested_path!r} did not contain the given "old_string" parameter.'
+
+
+class OldStringPresentMultipleTimesError(EditToolError):
+    @property
+    def content(self) -> str:
+        return f'The requested edit to {self.requested_path!r} contained the given "old_string" parameter multiple times.'  # noqa
+
+
+##
+
+
+def execute_edit_tool(
+        *,
+        file_path: str,
+        old_string: str,
+        new_string: str,
+        replace_all: bool = False,
+) -> str:
+    """
+    Edits the given file by replacing the string given by the 'old_string' parameter with the string given by the
+    'new_string' parameter.
+
+    The file must exist, must be a valid text file, and must be given as an absolute path.
+
+    If the 'replace_all' parameter is false (the default) then 'new_string' must be present exactly once in the file,
+    otherwise the operation will fail. If 'replace_all' is true then all instances of 'old_string' will be replaced by
+    'new_string', but the operation will fail if there are no instances of 'old_string'
+    present in the file.
+
+    For the operation to succeed, both 'old_string' and 'new_string' must be EXACT, including all exact indentation and
+    other whitespace. This *includes* trailing newlines - this operates on the file as a single string, not a list of
+    lines.
+
+    Args:
+        file_path: The path of the file to edit. Must be an absolute path.
+        old_string: The old string to be replaced. May not be empty, and must be exact, including all exact whitespace.
+        new_string: The new string to replace the old string with.
+        replace_all: If false (the default) then exactly one instance of 'old_string' must be present in the file to be
+            replaced. If true then all instances of 'old_string' will be replaced by 'new_string', but at least one
+            instance of 'old_string' must be present in the file.
+    """
+
+    if not old_string:
+        raise EmptyNewStringError(file_path)
+
+    ctx = tool_fs_context()
+    ctx.check_stat_file(file_path, text=True, write=True)
+
+    with open(file_path) as f:
+        old_file = f.read()
+
+    n = old_file.count(old_string)
+    if not n:
+        raise OldStringNotPresentError(file_path)
+
+    if not replace_all and n != 1:
+        raise OldStringPresentMultipleTimesError(file_path)
+
+    new_file = old_file.replace(old_string, new_string)
+
+    # FIXME: confirm lol
+
+    with open(file_path, 'w') as f:
+        f.write(new_file)
+
+    return 'The file has been edited successfully.'
+
+
+@lang.cached_function
+def edit_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_edit_tool)

ommlds/minichain/lib/fs/tools/ls.py

@@ -0,0 +1,38 @@
+import io
+import os
+
+from omlish import lang
+
+from ....tools.execution.catalog import ToolCatalogEntry
+from ....tools.execution.reflect import reflect_tool_catalog_entry
+from ..context import tool_fs_context
+
+
+##
+
+
+def execute_ls_tool(
+        dir_path: str,
+) -> str:
+    """
+    Lists the contents of the specified dir.
+
+    Args:
+        dir_path: The dir to list the contents of. Must be an absolute path.
+    """
+
+    ctx = tool_fs_context()
+    ctx.check_stat_dir(dir_path)
+
+    out = io.StringIO()
+    out.write('<dir>\n')
+    for e in sorted(os.scandir(dir_path), key=lambda e: e.name):  # noqa
+        out.write(f'{e.name}{"/" if e.is_dir() else ""}\n')
+    out.write('</dir>\n')
+
+    return out.getvalue()
+
+
+@lang.cached_function
+def ls_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_ls_tool)

ommlds/minichain/lib/fs/tools/read.py

@@ -0,0 +1,115 @@
+"""
+TODO:
+ - better bad unicode handling
+ - read whole file if < some size, report filesize / num lines / mtime inline
+ - fs cache
+ - track changes
+"""
+import io
+import itertools
+
+from omlish import lang
+
+from ....tools.execution.catalog import ToolCatalogEntry
+from ....tools.execution.reflect import reflect_tool_catalog_entry
+from ....tools.reflect import tool_spec_override
+from ....tools.types import ToolParam
+from ..context import tool_fs_context
+
+
+##
+
+
+DEFAULT_MAX_NUM_LINES = 2_000
+
+MAX_LINE_LENGTH = 2_000
+
+
+@tool_spec_override(
+    desc=f"""
+         Reads a file from the local filesystem. You can access any file directly by using this tool.
+
+        Assume this tool is able to read all files on the machine. If the User provides a path to a file assume that
+        path is valid. It is okay to read a file that does not exist; an error will be returned.
+
+        Usage:
+        - The file_path parameter must be an absolute path, not a relative path.
+        - By default, it reads up to {DEFAULT_MAX_NUM_LINES} lines starting from the beginning of the file.
+        - You can optionally specify a line offset and limit (especially handy for long files), but it's recommended to
+          read the whole file by not providing these parameters.
+        - Any lines longer than {MAX_LINE_LENGTH} characters will be truncated with "...".
+        - Invalid unicode characters will be replaced with the unicode replacement character "\\ufffd".
+        - Results are returned using cat -n format, with line numbers starting at 1 and suffixed with a pipe character
+          "|".
+        - This tool cannot read binary files, including images.
+    """,
+    params=[
+        ToolParam(
+            'file_path',
+            desc='The absolute path to the file to read.',
+        ),
+        ToolParam(
+            'line_offset',
+            desc='The line number to start reading from (0-based).',
+        ),
+        ToolParam(
+            'num_lines',
+            desc=f'The number of lines to read (defaults to {DEFAULT_MAX_NUM_LINES}).',
+        ),
+    ],
+)
+def execute_read_tool(
+        file_path: str,
+        *,
+        line_offset: int = 0,
+        num_lines: int = DEFAULT_MAX_NUM_LINES,
+) -> str:
+    ctx = tool_fs_context()
+    ctx.check_stat_file(file_path, text=True)
+
+    out = io.StringIO()
+    out.write('<file>\n')
+
+    zp = len(str(line_offset + num_lines))
+    n = line_offset
+    has_trunc = False  # noqa
+    with open(file_path, errors='replace') as f:
+        fi = iter(f)
+
+        for line in itertools.islice(fi, line_offset, line_offset + num_lines):
+            out.write(f'{str(n + 1).zfill(zp):}|')
+            line = line.removesuffix('\n')
+            if len(line) > MAX_LINE_LENGTH:
+                has_trunc = True  # noqa
+                out.write(line[:MAX_LINE_LENGTH])
+                out.write('...')
+            else:
+                out.write(line)
+            out.write('\n')
+            n += 1
+
+        # tl = n
+        # if (ml := lang.ilen(fi)):
+        #     check.state(n == num_lines)
+        #     tl += ml
+
+        try:
+            next(fi)
+        except StopIteration:
+            has_more = False
+        else:
+            has_more = True
+
+    out.write(f'</file>\n')
+
+    if has_more:
+        out.write(
+            f'\n(File has more lines. Use "line_offset" parameter to read beyond line {line_offset + num_lines}.)\n',
+        )
+
+    return out.getvalue()
+
+
+@lang.cached_function
+def read_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_read_tool)

ommlds/minichain/lib/fs/tools/recursivels/execution.py

@@ -0,0 +1,40 @@
+from omlish import lang
+
+from .....tools.execution.catalog import ToolCatalogEntry
+from .....tools.execution.reflect import reflect_tool_catalog_entry
+from ...context import tool_fs_context
+from .rendering import LsLinesRenderer
+from .running import LsRunner
+
+
+##
+
+
+def execute_recursive_ls_tool(
+        base_path: str,
+) -> str:
+    """
+    Recursively lists the directory contents of the given base path.
+
+    Args:
+        base_path: The path of the directory to list the contents of. Must be absolute.
+
+    Returns:
+        A formatted string of the recursive directory contents.
+    """
+
+    ft_ctx = tool_fs_context()
+    ft_ctx.check_requested_path(base_path)
+
+    root = LsRunner().run(base_path)
+    lines = LsLinesRenderer().render(root)
+    return '\n'.join([
+        '<dir>',
+        *lines.lines,
+        '</dir>',
+    ])
+
+
+@lang.cached_function
+def recursive_ls_tool() -> ToolCatalogEntry:
+    return reflect_tool_catalog_entry(execute_recursive_ls_tool)