lionagi 0.11.0__tar.gz → 0.12.1__tar.gz

lionagi-0.12.1/.coveragerc

@@ -0,0 +1,13 @@
+[run]
+source = lionagi
+omit =
+    tests/*
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    pass
+    raise ImportError

lionagi-0.12.1/.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: ci
+on: [pull_request]
+permissions:
+  contents: read
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: docker:dind
+        options: --privileged
+    strategy:
+      matrix:
+        py: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "${{ matrix.py }}"
+      - run: pip install uv
+      - name: Create and activate virtual environment
+        run: |
+          uv venv .venv
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
+          echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+      - name: Install dependencies
+        run: |
+          uv sync --extra all
+      - name: Run tests
+        run: uv run pytest -q --cov=lionagi --cov-report=xml
+      # - name: Upload coverage to Codecov
+      #   uses: codecov/codecov-action@v3
+      #   with:
+      #     token: ${{ secrets.CODECOV_TOKEN }}
+      #     fail_ci_if_error: true

{lionagi-0.11.0 → lionagi-0.12.1}/.github/workflows/docs.yml

@@ -4,6 +4,9 @@ on:
   push:
     branches: ["main"]
 
+permissions:
+  contents: write
+
 jobs:
   build-deploy:
     runs-on: ubuntu-latest

{lionagi-0.11.0 → lionagi-0.12.1}/.github/workflows/release.yml

@@ -27,13 +27,9 @@ jobs:
 
       - name: Install dependencies
         run: |
-          uv venv
-          uv build
-
+          uv sync --extra all
       - name: Run tests
-        run: |
-          source .venv/bin/activate
-          uv run pytest --asyncio-mode=auto --maxfail=1 --disable-warnings tests/
+        run: uv run pytest -q --cov=lionagi --cov-report=xml
 
   deploy:
     needs: test

{lionagi-0.11.0 → lionagi-0.12.1}/.gitignore

@@ -175,4 +175,8 @@ notebooks/.lock
 notebooks/meta.json
 logs/
 temp_logs/
-dev/
+dev/
+*.DS_Store
+.roomodes
+.khive/
+.roo/

{lionagi-0.11.0 → lionagi-0.12.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.11.0
+Version: 0.12.1
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>
 License:            Apache License
@@ -219,22 +219,19 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
-Requires-Dist: aiocache>=0.12.0
-Requires-Dist: aiohttp>=3.11.14
 Requires-Dist: jinja2>=3.0.0
+Requires-Dist: khive>=0.2.6
 Requires-Dist: pandas>=2.0.0
-Requires-Dist: pillow>=10.0.0
-Requires-Dist: pydantic>=2.0.0
-Requires-Dist: python-dotenv>=1.1.0
 Requires-Dist: tiktoken>=0.8.0
-Requires-Dist: toml>=0.9.0
+Provides-Extra: all
+Requires-Dist: khive[ollama,reader]; extra == 'all'
+Requires-Dist: litellm; extra == 'all'
 Provides-Extra: llms
-Requires-Dist: litellm>=1.59.5; extra == 'llms'
+Requires-Dist: litellm; extra == 'llms'
 Provides-Extra: ollama
-Requires-Dist: ollama>=0.4.7; extra == 'ollama'
-Requires-Dist: openai>=1.60.0; extra == 'ollama'
+Requires-Dist: khive[ollama]; extra == 'ollama'
 Provides-Extra: tools
-Requires-Dist: docling>=2.15.1; extra == 'tools'
+Requires-Dist: khive[reader]; extra == 'tools'
 Description-Content-Type: text/markdown
 
 ![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935)

lionagi-0.12.1/lionagi/adapters/__init__.py

@@ -0,0 +1 @@
+"""deprecated, will be removed in v0.13.0"""

{lionagi-0.11.0 → lionagi-0.12.1}/lionagi/fields/file.py

@@ -71,7 +71,7 @@ class File(HashableModel):
         header: str | None = None,
         footer: str | None = None,
     ) -> Path:
-        from lionagi.utils import create_path
+        from lionagi.libs.file.create_path import create_path
 
         fp = create_path(
             directory=directory,

{lionagi-0.11.0 → lionagi-0.12.1}/lionagi/fields/reason.py

@@ -4,8 +4,8 @@
 
 from pydantic import Field, field_validator
 
+from lionagi.libs.parse.to_num import to_num
 from lionagi.models import FieldModel, HashableModel
-from lionagi.utils import to_num
 
 __all__ = ("Reason",)
 

{lionagi-0.11.0 → lionagi-0.12.1}/lionagi/libs/file/concat.py

@@ -1,7 +1,12 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from pathlib import Path
 from typing import Any
 
-from lionagi.utils import create_path, lcall
+from lionagi.libs.file.create_path import create_path
+from lionagi.utils import lcall
 
 from .process import dir_to_files
 

{lionagi-0.11.0 → lionagi-0.12.1}/lionagi/libs/file/concat_files.py

@@ -1,6 +1,10 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
 from pathlib import Path
 
-from lionagi.utils import create_path
+from lionagi.libs.file.create_path import create_path
 
 from .process import dir_to_files
 

lionagi-0.12.1/lionagi/libs/file/create_path.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import uuid
+from datetime import datetime
+from pathlib import Path
+
+
+def create_path(
+    directory: Path | str,
+    filename: str,
+    extension: str = None,
+    timestamp: bool = False,
+    dir_exist_ok: bool = True,
+    file_exist_ok: bool = False,
+    time_prefix: bool = False,
+    timestamp_format: str | None = None,
+    random_hash_digits: int = 0,
+) -> Path:
+    """
+    Generate a new file path with optional timestamp and a random suffix.
+
+    Args:
+        directory: The directory where the file will be created.
+        filename: The base name of the file to create.
+        extension: The file extension, if not part of filename.
+        timestamp: If True, add a timestamp to the filename.
+        dir_exist_ok: If True, don't error if directory exists.
+        file_exist_ok: If True, allow overwriting existing files.
+        time_prefix: If True, timestamp is prefixed instead of suffixed.
+        timestamp_format: Custom format for timestamp (default: "%Y%m%d%H%M%S").
+        random_hash_digits: Number of hex digits for a random suffix.
+
+    Returns:
+        The full Path to the new or existing file.
+
+    Raises:
+        ValueError: If filename is invalid.
+        FileExistsError: If file exists and file_exist_ok=False.
+    """
+    if "/" in filename:
+        sub_dir, filename = filename.split("/")[:-1], filename.split("/")[-1]
+        directory = Path(directory) / "/".join(sub_dir)
+
+    if "\\" in filename:
+        raise ValueError("Filename cannot contain directory separators.")
+
+    directory = Path(directory)
+
+    # Extract name and extension from filename if present
+    if "." in filename:
+        name, ext = filename.rsplit(".", 1)
+    else:
+        name, ext = filename, extension
+
+    # Ensure extension has a single leading dot
+    ext = f".{ext.lstrip('.')}" if ext else ""
+
+    # Add timestamp if requested
+    if timestamp:
+        ts_str = datetime.now().strftime(timestamp_format or "%Y%m%d%H%M%S")
+        name = f"{ts_str}_{name}" if time_prefix else f"{name}_{ts_str}"
+
+    # Add random suffix if requested
+    if random_hash_digits > 0:
+        # Use UUID4 and truncate its hex for random suffix
+        random_suffix = uuid.uuid4().hex[:random_hash_digits]
+        name = f"{name}-{random_suffix}"
+
+    full_path = directory / f"{name}{ext}"
+
+    # Check if file or directory existence
+    full_path.parent.mkdir(parents=True, exist_ok=dir_exist_ok)
+    if full_path.exists() and not file_exist_ok:
+        raise FileExistsError(
+            f"File {full_path} already exists and file_exist_ok is False."
+        )
+
+    return full_path

lionagi-0.12.1/lionagi/libs/file/file_util.py

@@ -0,0 +1,358 @@
+# Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, Literal
+
+
+class FileUtil:
+
+    @staticmethod
+    def chunk_by_chars(
+        text: str,
+        chunk_size: int = 2048,
+        overlap: float = 0,
+        threshold: int = 256,
+    ) -> list[str]:
+        from .chunk import chunk_by_chars
+
+        return chunk_by_chars(
+            text,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+        )
+
+    @staticmethod
+    def chunk_by_tokens(
+        text: str,
+        tokenizer: Callable[[str], list[str]] = str.split,
+        chunk_size: int = 1024,
+        overlap: float = 0,
+        threshold: int = 256,
+    ) -> list[str]:
+        from .chunk import chunk_by_tokens
+
+        return chunk_by_tokens(
+            text,
+            tokenizer=tokenizer,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+        )
+
+    @staticmethod
+    def chunk_content(
+        content: str,
+        chunk_by: Literal["chars", "tokens"] = "chars",
+        tokenizer: Callable[[str], list[str]] = str.split,
+        chunk_size: int = 1024,
+        overlap: float = 0,
+        threshold: int = 256,
+    ) -> list[str]:
+        from .chunk import chunk_content
+
+        return chunk_content(
+            content,
+            chunk_by=chunk_by,
+            tokenizer=tokenizer,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+        )
+
+    @staticmethod
+    def concat_files(
+        data_path: str | Path | list,
+        file_types: list[str],
+        output_dir: str | Path = None,
+        output_filename: str = None,
+        file_exist_ok: bool = True,
+        recursive: bool = True,
+        verbose: bool = True,
+        threshold: int = 0,
+        return_fps: bool = False,
+        return_files: bool = False,
+        **kwargs,
+    ) -> (
+        list[str] | str | tuple[list[str], list[Path]] | tuple[str, list[Path]]
+    ):
+        from .concat_files import concat_files
+
+        return concat_files(
+            data_path,
+            file_types=file_types,
+            output_dir=output_dir,
+            output_filename=output_filename,
+            file_exist_ok=file_exist_ok,
+            recursive=recursive,
+            verbose=verbose,
+            threshold=threshold,
+            return_fps=return_fps,
+            return_files=return_files,
+            **kwargs,
+        )
+
+    @staticmethod
+    def concat(
+        data_path: str | Path | list,
+        file_types: list[str],
+        output_dir: str | Path = None,
+        output_filename: str = None,
+        file_exist_ok: bool = True,
+        recursive: bool = True,
+        verbose: bool = True,
+        threshold: int = 0,
+        return_fps: bool = False,
+        return_files: bool = False,
+        exclude_patterns: list[str] = None,
+        **kwargs,
+    ) -> dict[str, Any]:
+        from .concat import concat
+
+        return concat(
+            data_path,
+            file_types=file_types,
+            output_dir=output_dir,
+            output_filename=output_filename,
+            file_exist_ok=file_exist_ok,
+            recursive=recursive,
+            verbose=verbose,
+            threshold=threshold,
+            return_fps=return_fps,
+            return_files=return_files,
+            exclude_patterns=exclude_patterns,
+            **kwargs,
+        )
+
+    @staticmethod
+    def copy_file(src: Path | str, dest: Path | str) -> None:
+        from .file_ops import copy_file
+
+        copy_file(src, dest)
+
+    @staticmethod
+    def get_file_size(path: Path | str) -> int:
+        from .file_ops import get_file_size
+
+        return get_file_size(path)
+
+    @staticmethod
+    def list_files(
+        dir_path: Path | str, extension: str | None = None
+    ) -> list[Path]:
+        from .file_ops import list_files
+
+        return list_files(dir_path, extension=extension)
+
+    @staticmethod
+    def read_file(file_path: Path | str, encoding: str = "utf-8") -> str:
+        from .file_ops import read_file
+
+        return read_file(file_path, encoding=encoding)
+
+    @staticmethod
+    def read_image_to_base64(image_path: str | Path) -> str:
+        import base64
+
+        import cv2  # type: ignore[import]
+
+        image_path = str(image_path)
+        image = cv2.imread(image_path, cv2.COLOR_BGR2RGB)
+
+        if image is None:
+            raise ValueError(f"Could not read image from path: {image_path}")
+
+        file_extension = "." + image_path.split(".")[-1]
+
+        success, buffer = cv2.imencode(file_extension, image)
+        if not success:
+            raise ValueError(
+                f"Could not encode image to {file_extension} format."
+            )
+        encoded_image = base64.b64encode(buffer).decode("utf-8")
+        return encoded_image
+
+    @staticmethod
+    def pdf_to_images(
+        pdf_path: str, output_folder: str, dpi: int = 300, fmt: str = "jpeg"
+    ) -> list:
+        """
+        Convert a PDF file into images, one image per page.
+
+        Args:
+            pdf_path (str): Path to the input PDF file.
+            output_folder (str): Directory to save the output images.
+            dpi (int): Dots per inch (resolution) for conversion (default: 300).
+            fmt (str): Image format (default: 'jpeg'). Use 'png' if preferred.
+
+        Returns:
+            list: A list of file paths for the saved images.
+        """
+        import os
+
+        from lionagi.utils import check_import
+
+        convert_from_path = check_import(
+            "pdf2image", import_name="convert_from_path"
+        )
+
+        # Ensure the output folder exists
+        os.makedirs(output_folder, exist_ok=True)
+
+        # Convert PDF to a list of PIL Image objects
+        images = convert_from_path(pdf_path, dpi=dpi)
+
+        saved_paths = []
+        for i, image in enumerate(images):
+            # Construct the output file name
+            image_file = os.path.join(output_folder, f"page_{i + 1}.{fmt}")
+            image.save(image_file, fmt.upper())
+            saved_paths.append(image_file)
+
+        return saved_paths
+
+    @staticmethod
+    def dir_to_files(
+        directory: str | Path,
+        file_types: list[str] | None = None,
+        max_workers: int | None = None,
+        ignore_errors: bool = False,
+        verbose: bool = False,
+        recursive: bool = False,
+    ) -> list[Path]:
+        from .process import dir_to_files
+
+        return dir_to_files(
+            directory,
+            file_types=file_types,
+            max_workers=max_workers,
+            ignore_errors=ignore_errors,
+            verbose=verbose,
+            recursive=recursive,
+        )
+
+    @staticmethod
+    def file_to_chunks(
+        file_path: str | Path,
+        chunk_by: Literal["chars", "tokens"] = "chars",
+        chunk_size: int = 1500,
+        overlap: float = 0.1,
+        threshold: int = 200,
+        encoding: str = "utf-8",
+        custom_metadata: dict[str, Any] | None = None,
+        output_dir: str | Path | None = None,
+        verbose: bool = False,
+        timestamp: bool = True,
+        random_hash_digits: int = 4,
+        as_node: bool = False,
+    ) -> list[dict[str, Any]]:
+        from .file_ops import file_to_chunks
+
+        return file_to_chunks(
+            file_path,
+            chunk_by=chunk_by,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            encoding=encoding,
+            custom_metadata=custom_metadata,
+            output_dir=output_dir,
+            verbose=verbose,
+            timestamp=timestamp,
+            random_hash_digits=random_hash_digits,
+            as_node=as_node,
+        )
+
+    @staticmethod
+    def chunk(
+        *,
+        text: str | None = None,
+        url_or_path: str | Path = None,
+        file_types: list[str] | None = None,  # only local files
+        recursive: bool = False,  # only local files
+        tokenizer: Callable[[str], list[str]] = None,
+        chunk_by: Literal["chars", "tokens"] = "chars",
+        chunk_size: int = 1500,
+        overlap: float = 0.1,
+        threshold: int = 200,
+        output_file: str | Path | None = None,
+        metadata: dict[str, Any] | None = None,
+        reader_tool: Callable = None,
+        as_node: bool = False,
+    ) -> list:
+        from .process import chunk
+
+        return chunk(
+            text=text,
+            url_or_path=url_or_path,
+            file_types=file_types,
+            recursive=recursive,
+            tokenizer=tokenizer,
+            chunk_by=chunk_by,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            output_file=output_file,
+            metadata=metadata,
+            reader_tool=reader_tool,
+            as_node=as_node,
+        )
+
+    @staticmethod
+    def save_to_file(
+        text: str,
+        directory: str | Path,
+        filename: str,
+        extension: str = "txt",
+        timestamp: bool = True,
+        dir_exist_ok: bool = True,
+        file_exist_ok: bool = True,
+        time_prefix: bool = False,
+        timestamp_format: str = "%Y%m%d_%H%M%S",
+        random_hash_digits: int = 4,
+        verbose: bool = False,
+    ) -> Path:
+        from .save import save_to_file
+
+        return save_to_file(
+            text,
+            directory=directory,
+            filename=filename,
+            extension=extension,
+            timestamp=timestamp,
+            dir_exist_ok=dir_exist_ok,
+            file_exist_ok=file_exist_ok,
+            time_prefix=time_prefix,
+            timestamp_format=timestamp_format,
+            random_hash_digits=random_hash_digits,
+            verbose=verbose,
+        )
+
+    @staticmethod
+    def create_path(
+        directory: Path | str,
+        filename: str,
+        extension: str = None,
+        timestamp: bool = False,
+        dir_exist_ok: bool = True,
+        file_exist_ok: bool = False,
+        time_prefix: bool = False,
+        timestamp_format: str | None = None,
+        random_hash_digits: int = 0,
+    ) -> Path:
+        from .create_path import create_path
+
+        return create_path(
+            directory=directory,
+            filename=filename,
+            extension=extension,
+            timestamp=timestamp,
+            dir_exist_ok=dir_exist_ok,
+            file_exist_ok=file_exist_ok,
+            time_prefix=time_prefix,
+            timestamp_format=timestamp_format,
+            random_hash_digits=random_hash_digits,
+        )

{lionagi-0.11.0 → lionagi-0.12.1}/lionagi/libs/file/save.py

@@ -7,7 +7,7 @@ import logging
 from pathlib import Path
 from typing import Any
 
-from lionagi.utils import create_path
+from .create_path import create_path
 
 
 def save_to_file(