MemoryOS 2.0.3__py3-none-any.whl

memos/chunkers/simple_chunker.py

@@ -0,0 +1,50 @@
+class SimpleTextSplitter:
+    """Simple text splitter wrapper."""
+
+    def __init__(self, chunk_size: int, chunk_overlap: int):
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+
+    def chunk(self, text: str, **kwargs) -> list[str]:
+        return self._simple_split_text(text, self.chunk_size, self.chunk_overlap)
+
+    def _simple_split_text(self, text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
+        """
+        Simple text splitter as fallback when langchain is not available.
+
+        Args:
+            text: Text to split
+            chunk_size: Maximum size of chunks
+            chunk_overlap: Overlap between chunks
+
+        Returns:
+            List of text chunks
+        """
+        if not text or len(text) <= chunk_size:
+            return [text] if text.strip() else []
+
+        chunks = []
+        start = 0
+        text_len = len(text)
+
+        while start < text_len:
+            # Calculate end position
+            end = min(start + chunk_size, text_len)
+
+            # If not the last chunk, try to break at a good position
+            if end < text_len:
+                # Try to break at newline, sentence end, or space
+                for separator in ["\n\n", "\n", "。", "！", "？", ". ", "! ", "? ", " "]:
+                    last_sep = text.rfind(separator, start, end)
+                    if last_sep != -1:
+                        end = last_sep + len(separator)
+                        break
+
+            chunk = text[start:end].strip()
+            if chunk:
+                chunks.append(chunk)
+
+            # Move start position with overlap
+            start = max(start + 1, end - chunk_overlap)
+
+        return chunks

memos/cli.py

@@ -0,0 +1,113 @@
+"""
+MemOS CLI Tool
+This script provides command-line interface for MemOS operations.
+"""
+
+import argparse
+import json
+import os
+import zipfile
+
+from io import BytesIO
+
+
+def export_openapi(output: str) -> bool:
+    """Export OpenAPI schema to JSON file."""
+    from memos.api.server_api import app
+
+    # Create directory if it doesn't exist
+    if os.path.dirname(output):
+        os.makedirs(os.path.dirname(output), exist_ok=True)
+
+    with open(output, "w") as f:
+        json.dump(app.openapi(), f, indent=2)
+        f.write("\n")
+
+    print(f"✅ OpenAPI schema exported to: {output}")
+    return True
+
+
+def download_examples(dest: str) -> bool:
+    import requests
+
+    """Download examples from the MemOS repository."""
+    zip_url = "https://github.com/MemTensor/MemOS/archive/refs/heads/main.zip"
+    print(f"📥 Downloading examples from {zip_url}...")
+
+    try:
+        response = requests.get(zip_url)
+        response.raise_for_status()
+
+        with zipfile.ZipFile(BytesIO(response.content)) as z:
+            extracted_files = []
+            for file in z.namelist():
+                if "MemOS-main/examples/" in file and not file.endswith("/"):
+                    # Remove the prefix and extract to dest
+                    relative_path = file.replace("MemOS-main/examples/", "")
+                    extract_path = os.path.join(dest, relative_path)
+
+                    # Create directory if it doesn't exist
+                    os.makedirs(os.path.dirname(extract_path), exist_ok=True)
+
+                    # Extract the file
+                    with z.open(file) as source, open(extract_path, "wb") as target:
+                        target.write(source.read())
+                    extracted_files.append(extract_path)
+
+        print(f"✅ Examples downloaded to: {dest}")
+        print(f"📁 {len(extracted_files)} files extracted")
+
+    except requests.RequestException as e:
+        print(f"❌ Error downloading examples: {e}")
+        return False
+    except Exception as e:
+        print(f"❌ Error extracting examples: {e}")
+        return False
+
+    return True
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        prog="memos",
+        description="MemOS Command Line Interface",
+    )
+
+    # Create subparsers for different commands
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # Download examples command
+    examples_parser = subparsers.add_parser("download_examples", help="Download example files")
+    examples_parser.add_argument(
+        "--dest",
+        type=str,
+        default="./examples",
+        help="Destination directory for examples (default: ./examples)",
+    )
+
+    # Export API command
+    api_parser = subparsers.add_parser("export_openapi", help="Export OpenAPI schema to JSON file")
+    api_parser.add_argument(
+        "--output",
+        type=str,
+        default="openapi.json",
+        help="Output path for OpenAPI schema (default: openapi.json)",
+    )
+
+    # Parse arguments
+    args = parser.parse_args()
+
+    # Handle commands
+    if args.command == "download_examples":
+        success = download_examples(args.dest)
+        exit(0 if success else 1)
+    elif args.command == "export_openapi":
+        success = export_openapi(args.output)
+        exit(0 if success else 1)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

memos/configs/base.py

@@ -0,0 +1,82 @@
+import os
+
+from typing import Any
+
+import yaml
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from memos.log import get_logger
+
+
+logger = get_logger(__name__)
+
+
+class BaseConfig(BaseModel):
+    """Base configuration.
+
+    All configurations should inherit from this class.
+    This class uses Pydantic's ConfigDict to enforce strict validation
+    and forbids extra fields."""
+
+    model_schema: str = Field(
+        "NOT_SET",
+        description="Schema for configuration. This value will be automatically set.",
+        exclude=True,
+    )
+
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+    @model_validator(mode="after")
+    def set_default_schema(self) -> "BaseConfig":
+        dot_path_schema = self.__module__ + "." + self.__class__.__name__
+        if self.model_schema == dot_path_schema:
+            return self
+        if self.model_schema != "NOT_SET":
+            logger.warning(
+                f"Schema is set to {self.model_schema}, but it should be {dot_path_schema}. "
+                "Changing schema to the default value."
+            )
+        self.model_schema = dot_path_schema
+        return self
+
+    @classmethod
+    def from_json_file(cls, json_path: str) -> Any:
+        """Load configuration from a JSON file."""
+        with open(json_path, encoding="utf-8") as f:
+            data = f.read()
+        return cls.model_validate_json(data)
+
+    def to_json_file(self, json_path: str) -> None:
+        """Dump configuration to a JSON file."""
+        dir_path = os.path.dirname(json_path)
+        if dir_path:
+            os.makedirs(dir_path, exist_ok=True)
+        with open(json_path, "w", encoding="utf-8") as f:
+            f.write(self.model_dump_json(indent=2, warnings="none"))
+
+    @classmethod
+    def from_yaml_file(cls, yaml_path: str) -> Any:
+        """Load configuration from a YAML file."""
+        with open(yaml_path, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        return cls.model_validate(data)
+
+    def to_yaml_file(self, yaml_path: str) -> None:
+        """Dump configuration to a YAML file."""
+
+        dir_path = os.path.dirname(yaml_path)
+        if dir_path:
+            os.makedirs(dir_path, exist_ok=True)
+
+        with open(yaml_path, "w", encoding="utf-8") as f:
+            yaml.safe_dump(
+                self.model_dump(mode="json", warnings="none"),
+                f,
+                default_flow_style=False,
+                allow_unicode=True,
+                indent=2,
+            )
+
+    def get(self, key, default=None):
+        return getattr(self, key, default)

memos/configs/chunker.py

@@ -0,0 +1,59 @@
+from typing import Any, ClassVar
+
+from pydantic import Field, field_validator, model_validator
+
+from memos.configs.base import BaseConfig
+
+
+class BaseChunkerConfig(BaseConfig):
+    """Base configuration class for chunkers."""
+
+    tokenizer_or_token_counter: str = Field(
+        default="gpt2", description="Tokenizer model name or a token counting function"
+    )
+    chunk_size: int = Field(default=512, description="Maximum tokens per chunk")
+    chunk_overlap: int = Field(default=128, description="Overlap between chunks")
+    min_sentences_per_chunk: int = Field(default=1, description="Minimum sentences in each chunk")
+
+
+class SentenceChunkerConfig(BaseChunkerConfig):
+    """Configuration for sentence-based text chunker."""
+
+
+class MarkdownChunkerConfig(BaseChunkerConfig):
+    """Configuration for markdown-based text chunker."""
+
+    headers_to_split_on: list[tuple[str, str]] = Field(
+        default=[("#", "Header 1"), ("##", "Header 2"), ("###", "Header 3")],
+        description="Headers to split on",
+    )
+    strip_headers: bool = Field(default=True, description="Strip headers from the text")
+    recursive: bool = Field(
+        default=False, description="Whether to use recursive character text splitter"
+    )
+
+
+class ChunkerConfigFactory(BaseConfig):
+    """Factory class for creating chunker configurations."""
+
+    backend: str = Field(..., description="Backend for chunker")
+    config: dict[str, Any] = Field(..., description="Configuration for the chunker backend")
+
+    backend_to_class: ClassVar[dict[str, Any]] = {
+        "sentence": SentenceChunkerConfig,
+        "markdown": MarkdownChunkerConfig,
+    }
+
+    @field_validator("backend")
+    @classmethod
+    def validate_backend(cls, backend: str) -> str:
+        """Validate the backend field."""
+        if backend not in cls.backend_to_class:
+            raise ValueError(f"Invalid backend: {backend}")
+        return backend
+
+    @model_validator(mode="after")
+    def create_config(self) -> "ChunkerConfigFactory":
+        config_class = self.backend_to_class[self.backend]
+        self.config = config_class(**self.config)
+        return self

memos/configs/embedder.py

@@ -0,0 +1,88 @@
+from typing import Any, ClassVar
+
+from pydantic import Field, field_validator, model_validator
+
+from memos.configs.base import BaseConfig
+
+
+class BaseEmbedderConfig(BaseConfig):
+    """Base configuration class for embedding models."""
+
+    model_name_or_path: str = Field(..., description="Model name or path")
+    embedding_dims: int | None = Field(
+        default=None, description="Number of dimensions for the embedding"
+    )
+    max_tokens: int | None = Field(
+        default=8192,
+        description="Maximum number of tokens per text. Texts exceeding this limit will be automatically truncated. Set to None to disable truncation.",
+    )
+    headers_extra: dict[str, Any] | None = Field(
+        default=None,
+        description="Extra headers for the embedding model, only for universal_api backend",
+    )
+
+
+class OllamaEmbedderConfig(BaseEmbedderConfig):
+    api_base: str = Field(default="http://localhost:11434", description="Base URL for Ollama API")
+
+
+class ArkEmbedderConfig(BaseEmbedderConfig):
+    api_key: str = Field(..., description="Ark API key")
+    api_base: str = Field(
+        default="https://ark.cn-beijing.volces.com/api/v3/", description="Base URL for Ark API"
+    )
+    chunk_size: int = Field(default=1, description="Chunk size for Ark API")
+    multi_modal: bool = Field(
+        default=False,
+        description="Whether to use multi-modal embedding (text + image) with Ark",
+    )
+
+
+class SenTranEmbedderConfig(BaseEmbedderConfig):
+    """Configuration class for Sentence Transformer embeddings."""
+
+    trust_remote_code: bool = Field(
+        default=True,
+        description="Whether to trust remote code when loading the model",
+    )
+
+
+class UniversalAPIEmbedderConfig(BaseEmbedderConfig):
+    """
+    Configuration class for universal API embedding providers, e.g.,
+    OpenAI, etc.
+    """
+
+    provider: str = Field(..., description="Provider name, e.g., 'openai'")
+    api_key: str = Field(..., description="API key for the embedding provider")
+    base_url: str | None = Field(
+        default=None, description="Optional base URL for custom or proxied endpoint"
+    )
+
+
+class EmbedderConfigFactory(BaseConfig):
+    """Factory class for creating embedder configurations."""
+
+    backend: str = Field(..., description="Backend for embedding model")
+    config: dict[str, Any] = Field(..., description="Configuration for the embedding model backend")
+
+    backend_to_class: ClassVar[dict[str, Any]] = {
+        "ollama": OllamaEmbedderConfig,
+        "sentence_transformer": SenTranEmbedderConfig,
+        "ark": ArkEmbedderConfig,
+        "universal_api": UniversalAPIEmbedderConfig,
+    }
+
+    @field_validator("backend")
+    @classmethod
+    def validate_backend(cls, backend: str) -> str:
+        """Validate the backend field."""
+        if backend not in cls.backend_to_class:
+            raise ValueError(f"Invalid backend: {backend}")
+        return backend
+
+    @model_validator(mode="after")
+    def create_config(self) -> "EmbedderConfigFactory":
+        config_class = self.backend_to_class[self.backend]
+        self.config = config_class(**self.config)
+        return self

memos/configs/graph_db.py

@@ -0,0 +1,236 @@
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from memos.configs.base import BaseConfig
+from memos.configs.vec_db import VectorDBConfigFactory
+
+
+class BaseGraphDBConfig(BaseConfig):
+    """Base class for all graph database configurations."""
+
+    uri: str | list
+    user: str
+    password: str
+
+
+class Neo4jGraphDBConfig(BaseGraphDBConfig):
+    """
+    Neo4j-specific configuration.
+
+    This config supports:
+    1) Physical isolation (multi-db) — each user gets a dedicated Neo4j database.
+    2) Logical isolation (single-db) — all users share one or more databases, but each node is tagged with `user_name`.
+
+    How to use:
+    - If `use_multi_db=True`, then `db_name` should usually be the same as `user_name`.
+      Each user gets a separate database for physical isolation.
+      Example: db_name = "alice", user_name = None or "alice".
+
+    - If `use_multi_db=False`, then `db_name` is your shared database (e.g., "neo4j" or "shared_db").
+      You must provide `user_name` to logically isolate each user's data.
+      All nodes and queries must respect this tag.
+
+    Example configs:
+    ---
+    # Physical isolation:
+    db_name = "alice"
+    use_multi_db = True
+    user_name = None
+
+    # Logical isolation:
+    db_name = "shared_db_student_group"
+    use_multi_db = False
+    user_name = "alice"
+    """
+
+    db_name: str = Field(..., description="The name of the target Neo4j database")
+    auto_create: bool = Field(
+        default=False,
+        description="If True, automatically create the target db_name in multi-db mode if it does not exist.",
+    )
+
+    use_multi_db: bool = Field(
+        default=True,
+        description=(
+            "If True: use Neo4j's multi-database feature for physical isolation; "
+            "each user typically gets a separate database. "
+            "If False: use a single shared database with logical isolation by user_name."
+        ),
+    )
+
+    user_name: str | None = Field(
+        default=None,
+        description=(
+            "Logical user or tenant ID for data isolation. "
+            "Required if use_multi_db is False. "
+            "All nodes must be tagged with this and all queries must filter by this."
+        ),
+    )
+
+    embedding_dimension: int = Field(default=768, description="Dimension of vector embedding")
+
+    @model_validator(mode="after")
+    def validate_config(self):
+        """Validate logical constraints to avoid misconfiguration."""
+        if not self.use_multi_db and not self.user_name:
+            raise ValueError(
+                "In single-database mode (use_multi_db=False), `user_name` must be provided for logical isolation."
+            )
+        return self
+
+
+class Neo4jCommunityGraphDBConfig(Neo4jGraphDBConfig):
+    """
+    Community edition config for Neo4j.
+
+    Notes:
+    - Must set `use_multi_db = False`
+    - Must provide `user_name` for logical isolation
+    - Embedding vector DB config is required
+    """
+
+    vec_config: VectorDBConfigFactory = Field(
+        ..., description="Vector DB config for embedding search"
+    )
+
+    @model_validator(mode="after")
+    def validate_community(self):
+        if self.use_multi_db:
+            raise ValueError("Neo4j Community Edition does not support use_multi_db=True.")
+        if not self.user_name:
+            raise ValueError("Neo4j Community config requires user_name for logical isolation.")
+        return self
+
+
+class NebulaGraphDBConfig(BaseGraphDBConfig):
+    """
+    NebulaGraph-specific configuration.
+
+    Key concepts:
+    - `space`: Equivalent to a database or namespace. All tag/edge/schema live within a space.
+    - `user_name`: Used for logical tenant isolation if needed.
+    - `auto_create`: Whether to automatically create the target space if it does not exist.
+
+    Example:
+    ---
+    hosts = ["127.0.0.1:9669"]
+    user = "root"
+    password = "nebula"
+    space = "shared_graph"
+    user_name = "alice"
+    """
+
+    space: str = Field(
+        ..., description="The name of the target NebulaGraph space (like a database)"
+    )
+    user_name: str | None = Field(
+        default=None,
+        description="Logical user or tenant ID for data isolation (optional, used in metadata tagging)",
+    )
+    auto_create: bool = Field(
+        default=False,
+        description="Whether to auto-create the space if it does not exist",
+    )
+    use_multi_db: bool = Field(
+        default=True,
+        description=(
+            "If True: use Neo4j's multi-database feature for physical isolation; "
+            "each user typically gets a separate database. "
+            "If False: use a single shared database with logical isolation by user_name."
+        ),
+    )
+    max_client: int = Field(
+        default=1000,
+        description=("max_client"),
+    )
+    embedding_dimension: int = Field(default=3072, description="Dimension of vector embedding")
+
+    @model_validator(mode="after")
+    def validate_config(self):
+        """Validate config."""
+        if not self.space:
+            raise ValueError("`space` must be provided")
+        return self
+
+
+class PolarDBGraphDBConfig(BaseConfig):
+    """
+    PolarDB-specific configuration.
+
+    Key concepts:
+    - `db_name`: The name of the target PolarDB database
+    - `user_name`: Used for logical tenant isolation if needed
+    - `auto_create`: Whether to automatically create the target database if it does not exist
+    - `use_multi_db`: Whether to use multi-database mode for physical isolation
+
+    Example:
+    ---
+    host = "localhost"
+    port = 5432
+    user = "postgres"
+    password = "password"
+    db_name = "memos_db"
+    user_name = "alice"
+    use_multi_db = True
+    auto_create = True
+    """
+
+    host: str = Field(..., description="Database host")
+    port: int = Field(default=5432, description="Database port")
+    user: str = Field(..., description="Database user")
+    password: str = Field(..., description="Database password")
+    db_name: str = Field(..., description="The name of the target PolarDB database")
+    user_name: str | None = Field(
+        default=None,
+        description="Logical user or tenant ID for data isolation (optional, used in metadata tagging)",
+    )
+    auto_create: bool = Field(
+        default=False,
+        description="Whether to auto-create the database if it does not exist",
+    )
+    use_multi_db: bool = Field(
+        default=True,
+        description=(
+            "If True: use multi-database mode for physical isolation; "
+            "each tenant typically gets a separate database. "
+            "If False: use a single shared database with logical isolation by user_name."
+        ),
+    )
+    embedding_dimension: int = Field(default=1024, description="Dimension of vector embedding")
+    maxconn: int = Field(
+        default=100,
+        description="Maximum number of connections in the connection pool",
+    )
+
+    @model_validator(mode="after")
+    def validate_config(self):
+        """Validate config."""
+        if not self.db_name:
+            raise ValueError("`db_name` must be provided")
+        return self
+
+
+class GraphDBConfigFactory(BaseModel):
+    backend: str = Field(..., description="Backend for graph database")
+    config: dict[str, Any] = Field(..., description="Configuration for the graph database backend")
+
+    backend_to_class: ClassVar[dict[str, Any]] = {
+        "neo4j": Neo4jGraphDBConfig,
+        "neo4j-community": Neo4jCommunityGraphDBConfig,
+        "nebular": NebulaGraphDBConfig,
+        "polardb": PolarDBGraphDBConfig,
+    }
+
+    @field_validator("backend")
+    @classmethod
+    def validate_backend(cls, backend: str) -> str:
+        if backend not in cls.backend_to_class:
+            raise ValueError(f"Unsupported graph db backend: {backend}")
+        return backend
+
+    @model_validator(mode="after")
+    def instantiate_config(self):
+        config_class = self.backend_to_class[self.backend]
+        self.config = config_class(**self.config)
+        return self