hammad-python 0.0.10__py3-none-any.whl → 0.0.11__py3-none-any.whl

hammad/data/types/files/configuration.py

@@ -0,0 +1,475 @@
+"""hammad.data.types.files.configuration"""
+
+import os
+import configparser
+from pathlib import Path
+from typing import Any, Self
+from dotenv import load_dotenv, dotenv_values
+import httpx
+import msgspec
+
+from .file import File, FileSource
+from ....based.fields import basedfield
+
+__all__ = ("Configuration",)
+
+
+class Configuration(File):
+    """Model / structure representation for configuration objects
+    for both module or application level usage. This class is
+    nothing more than a glorified key-value store with a
+    few extra features.
+
+    Inherits from File to provide file operations and extends
+    with configuration-specific functionality."""
+
+    # Configuration-specific fields
+    config_data: dict[str, Any] = basedfield(default_factory=dict)
+    """The actual configuration key-value pairs."""
+
+    format_type: str | None = basedfield(default=None)
+    """The format type of the configuration (json, toml, yaml, ini, env)."""
+
+    def __post_init__(self):
+        """Initialize configuration data from file data if available."""
+        super().__post_init__()
+
+        # If we have data but no config_data, try to parse it
+        if self.data is not None and not self.config_data:
+            self._parse_data()
+
+    def _parse_data(self) -> None:
+        """Parse the file data into configuration format."""
+        if not self.data:
+            return
+
+        content = self.data if isinstance(self.data, str) else self.data.decode("utf-8")
+
+        # Determine format from extension or type
+        format_type = self._detect_format()
+
+        try:
+            if format_type == "json":
+                self.config_data = msgspec.json.decode(content.encode("utf-8"))
+            elif format_type == "toml":
+                self.config_data = msgspec.toml.decode(content.encode("utf-8"))
+            elif format_type == "yaml":
+                self.config_data = msgspec.yaml.decode(content.encode("utf-8"))
+            elif format_type == "ini":
+                parser = configparser.ConfigParser()
+                parser.read_string(content)
+                self.config_data = {
+                    section: dict(parser[section]) for section in parser.sections()
+                }
+            elif format_type == "env":
+                # Parse as dotenv format
+                lines = content.strip().split("\n")
+                config_data = {}
+                for line in lines:
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        key, value = line.split("=", 1)
+                        config_data[key.strip()] = value.strip().strip("\"'")
+                self.config_data = config_data
+
+            self.format_type = format_type
+        except Exception as e:
+            raise ValueError(
+                f"Failed to parse configuration data as {format_type}: {e}"
+            )
+
+    def _detect_format(self) -> str:
+        """Detect the configuration format from extension or content."""
+        if self.format_type:
+            return self.format_type
+
+        # Try to detect from file extension
+        # Get extension directly from source path to avoid caching issues
+        if self.source.path:
+            ext = self.source.path.suffix.lower()
+            if ext in [".json"]:
+                return "json"
+            elif ext in [".toml"]:
+                return "toml"
+            elif ext in [".yaml", ".yml"]:
+                return "yaml"
+            elif ext in [".ini", ".cfg", ".conf"]:
+                return "ini"
+            elif ext in [".env"]:
+                return "env"
+        elif self.extension:
+            ext = self.extension.lower()
+            if ext in [".json"]:
+                return "json"
+            elif ext in [".toml"]:
+                return "toml"
+            elif ext in [".yaml", ".yml"]:
+                return "yaml"
+            elif ext in [".ini", ".cfg", ".conf"]:
+                return "ini"
+            elif ext in [".env"]:
+                return "env"
+
+        # Try to detect from MIME type
+        if self.type:
+            if "json" in self.type:
+                return "json"
+            elif "yaml" in self.type:
+                return "yaml"
+
+        # Default fallback - try to parse as JSON first
+        return "json"
+
+    def _serialize_data(self, format_type: str | None = None) -> str:
+        """Serialize configuration data to string format."""
+        format_type = format_type or self.format_type or "json"
+
+        if format_type == "json":
+            return msgspec.json.encode(self.config_data).decode("utf-8")
+        elif format_type == "toml":
+            return msgspec.toml.encode(self.config_data).decode("utf-8")
+        elif format_type == "yaml":
+            return msgspec.yaml.encode(self.config_data).decode("utf-8")
+        elif format_type == "ini":
+            parser = configparser.ConfigParser()
+            for section_name, section_data in self.config_data.items():
+                parser[section_name] = section_data
+            import io
+
+            output = io.StringIO()
+            parser.write(output)
+            return output.getvalue()
+        elif format_type == "env":
+            lines = []
+            for key, value in self.config_data.items():
+                # Simple escaping for shell variables
+                if isinstance(value, str) and (
+                    " " in value or '"' in value or "'" in value
+                ):
+                    value = f'"{value}"'
+                lines.append(f"{key}={value}")
+            return "\n".join(lines)
+        else:
+            raise ValueError(f"Unsupported format: {format_type}")
+
+    @classmethod
+    def from_dotenv(cls, path: str | Path | None = None) -> Self:
+        """Loads a .env file and creates a configuration object
+        from it.
+
+        NOTE: This does not set any environment variables.
+
+        Args:
+            path: The path to the .env file to load. If not provided,
+                the .env file in the current working directory will be used.
+        """
+        if path is None:
+            path = Path.cwd() / ".env"
+        else:
+            path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(f"Environment file not found: {path}")
+
+        # Use dotenv_values to parse without setting environment variables
+        config_data = dotenv_values(path)
+
+        return cls(
+            config_data=dict(config_data),
+            format_type="env",
+            source=FileSource(
+                is_file=True,
+                path=path,
+                size=path.stat().st_size if path.exists() else None,
+            ),
+            type="text/plain",
+        )
+
+    @classmethod
+    def from_os_prefix(cls, prefix: str) -> Self:
+        """Creates a new configuration object using all variables
+        that begin with the given prefix.
+
+        Args:
+            prefix: The prefix to use to filter the variables.
+        """
+        config_data = {}
+        for key, value in os.environ.items():
+            if key.startswith(prefix):
+                # Remove prefix and convert to lowercase
+                config_key = key[len(prefix) :].lstrip("_").lower()
+                config_data[config_key] = value
+
+        return cls(
+            config_data=config_data,
+            format_type="env",
+            source=FileSource(),
+            type="text/plain",
+        )
+
+    @classmethod
+    def from_os_vars(cls, vars: list[str]) -> Self:
+        """Pulls a certain set of environment variables and
+        creates a configuration object from them.
+
+        Args:
+            vars: A list of environment variable names to pull.
+        """
+        config_data = {}
+        for var in vars:
+            if var in os.environ:
+                config_data[var] = os.environ[var]
+
+        return cls(
+            config_data=config_data,
+            format_type="env",
+            source=FileSource(),
+            type="text/plain",
+        )
+
+    @classmethod
+    def from_file(
+        cls,
+        path: str | Path,
+    ) -> Self:
+        """Parses a file to return a configuration object. This
+        utilizes the following file types:
+
+        - json
+        - toml
+        - yaml
+        - ini
+        - env
+        """
+        # Use the parent File class to load the file
+        file_obj = File.from_path(path, lazy=False)
+
+        # Create a Configuration object from the File object
+        config = cls(
+            data=file_obj.data,
+            type=file_obj.type,
+            source=file_obj.source,
+        )
+
+        # Parse the data
+        config._parse_data()
+
+        return config
+
+    @classmethod
+    def from_url(
+        cls,
+        url: str,
+        *,
+        timeout: float = 30.0,
+        headers: dict[str, str] | None = None,
+    ) -> Self:
+        """Load configuration from a URL supporting various formats.
+
+        Args:
+            url: The URL to load configuration from
+            timeout: Request timeout in seconds
+            headers: Optional HTTP headers to include in the request
+
+        Returns:
+            A new Configuration instance
+        """
+        with httpx.Client(timeout=timeout) as client:
+            response = client.get(url, headers=headers or {})
+            response.raise_for_status()
+
+            # Get content as text
+            content = response.text
+
+            # Determine format from URL extension or content-type
+            format_type = None
+            if url.endswith(".json"):
+                format_type = "json"
+            elif url.endswith((".yaml", ".yml")):
+                format_type = "yaml"
+            elif url.endswith(".toml"):
+                format_type = "toml"
+            elif url.endswith((".ini", ".cfg", ".conf")):
+                format_type = "ini"
+            elif url.endswith(".env"):
+                format_type = "env"
+            else:
+                # Try to detect from content-type header
+                content_type = response.headers.get("content-type", "").lower()
+                if "json" in content_type:
+                    format_type = "json"
+                elif "yaml" in content_type:
+                    format_type = "yaml"
+
+        config = cls(
+            data=content,
+            type=response.headers.get("content-type"),
+            format_type=format_type,
+            source=FileSource(
+                is_url=True,
+                url=url,
+                size=len(content.encode("utf-8")),
+                encoding=response.encoding,
+            ),
+        )
+
+        config._parse_data()
+        return config
+
+    def to_file(
+        self,
+        path: str | Path,
+        *,
+        overwrite: bool = False,
+        format_type: str | None = None,
+    ) -> None:
+        """Saves the configuration object to a file. This
+        utilizes the following file types:
+
+        - json
+        - toml
+        - yaml
+        - ini
+        - env
+
+        Args:
+            path: The path to the file to save the configuration to.
+            overwrite: Whether to overwrite the file if it already exists.
+            format_type: Override the format type for saving.
+        """
+        save_path = Path(path)
+
+        if save_path.exists() and not overwrite:
+            raise FileExistsError(f"File already exists: {save_path}")
+
+        # Determine format from path extension if not specified
+        if format_type is None:
+            ext = save_path.suffix.lower()
+            if ext in [".json"]:
+                format_type = "json"
+            elif ext in [".toml"]:
+                format_type = "toml"
+            elif ext in [".yaml", ".yml"]:
+                format_type = "yaml"
+            elif ext in [".ini", ".cfg", ".conf"]:
+                format_type = "ini"
+            elif ext in [".env"]:
+                format_type = "env"
+            else:
+                format_type = self.format_type or "json"
+
+        # Serialize and save
+        content = self._serialize_data(format_type)
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        save_path.write_text(content, encoding="utf-8")
+
+    def update_file(
+        self,
+        path: str | Path,
+        exclude: list[str] | None = None,
+        exclude_none: bool = True,
+    ) -> None:
+        """Updates a valid configuration file with only the
+        differing values.
+
+        Args:
+            path: The path to the file to update.
+            exclude: A list of keys to exclude from the update.
+            exclude_none: Whether to exclude keys with None values.
+        """
+        path = Path(path)
+
+        if not path.exists():
+            raise FileNotFoundError(f"Configuration file not found: {path}")
+
+        # Load existing configuration
+        existing_config = Configuration.from_file(path)
+
+        # Prepare data to update
+        update_data = self.config_data.copy()
+
+        if exclude:
+            for key in exclude:
+                update_data.pop(key, None)
+
+        if exclude_none:
+            update_data = {k: v for k, v in update_data.items() if v is not None}
+
+        # Merge with existing data
+        existing_config.config_data.update(update_data)
+
+        # Save back to file
+        existing_config.to_file(path, overwrite=True)
+
+    def to_os(
+        self,
+        prefix: str | None = None,
+        exclude: list[str] | None = None,
+    ) -> None:
+        """Pushes the configuration object's values as active
+        environment variables. This will overwrite any existing
+        values for the session.
+
+        Args:
+            prefix: The prefix to use to filter the variables.
+            exclude: A list of keys to exclude from the update.
+        """
+        exclude = exclude or []
+
+        for key, value in self.config_data.items():
+            if key in exclude:
+                continue
+
+            # Convert value to string
+            env_value = str(value) if value is not None else ""
+
+            # Apply prefix if specified
+            env_key = f"{prefix}_{key}".upper() if prefix else key.upper()
+
+            # Set environment variable
+            os.environ[env_key] = env_value
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Get a configuration value by key.
+
+        Args:
+            key: The configuration key
+            default: Default value if key is not found
+
+        Returns:
+            The configuration value or default
+        """
+        return self.config_data.get(key, default)
+
+    def set(self, key: str, value: Any) -> None:
+        """Set a configuration value.
+
+        Args:
+            key: The configuration key
+            value: The value to set
+        """
+        self.config_data[key] = value
+
+    def __getitem__(self, key: str) -> Any:
+        """Get configuration value using dict-like access."""
+        return self.config_data[key]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        """Set configuration value using dict-like access."""
+        self.config_data[key] = value
+
+    def __contains__(self, key: str) -> bool:
+        """Check if configuration contains a key."""
+        return key in self.config_data
+
+    def keys(self):
+        """Return configuration keys."""
+        return self.config_data.keys()
+
+    def values(self):
+        """Return configuration values."""
+        return self.config_data.values()
+
+    def items(self):
+        """Return configuration key-value pairs."""
+        return self.config_data.items()

hammad/data/types/files/document.py

@@ -0,0 +1,195 @@
+"""hammad.data.types.files.document"""
+
+import httpx
+from typing import Any, Self, Iterator
+from markdown_it import MarkdownIt
+
+from .file import File, FileSource
+from ....based.fields import basedfield
+
+__all__ = ("Document",)
+
+
+class Document(File):
+    """A representation of a document, that is loadable from both a URL, file path
+    or bytes. This document can additionally be used to represent web pages, as well
+    as implement markdown formatting for both documents and web pages."""
+
+    # Cached properties for text processing
+    _lines: list[str] | None = basedfield(default=None)
+    _content: str | None = basedfield(default=None)
+    _md_parser: MarkdownIt | None = basedfield(default=None)
+    metadata: dict[str, Any] = basedfield(default_factory=dict)
+
+    @property
+    def content(self) -> str:
+        """Get the document content as string."""
+        if self._content is None:
+            data = self.read()
+            self._content = (
+                data
+                if isinstance(data, str)
+                else data.decode(self.source.encoding or "utf-8")
+            )
+        return self._content
+
+    @property
+    def lines(self) -> list[str]:
+        """Get lines of the document (cached for efficiency)."""
+        if self._lines is None:
+            self._lines = self.content.splitlines(keepends=False)
+        return self._lines
+
+    @property
+    def line_count(self) -> int:
+        """Get the number of lines in the document."""
+        return len(self.lines)
+
+    @property
+    def word_count(self) -> int:
+        """Get the word count of the document."""
+        return len(self.content.split())
+
+    @property
+    def char_count(self) -> int:
+        """Get the character count of the document."""
+        return len(self.content)
+
+    @property
+    def is_markdown(self) -> bool:
+        """Check if the document is a markdown file."""
+        return self.extension in {".md", ".markdown", ".mdown", ".mkd", ".mdx"}
+
+    @property
+    def md_parser(self) -> MarkdownIt:
+        """Get the markdown parser (lazy initialization)."""
+        if self._md_parser is None:
+            self._md_parser = MarkdownIt()
+        return self._md_parser
+
+    def iter_lines(self, *, strip: bool = False) -> Iterator[str]:
+        """Iterate over lines in the document.
+
+        Args:
+            strip: If True, strip whitespace from each line.
+
+        Yields:
+            Lines from the document.
+        """
+        for line in self.lines:
+            yield line.strip() if strip else line
+
+    def iter_paragraphs(self) -> Iterator[str]:
+        """Iterate over paragraphs (text blocks separated by empty lines)."""
+        paragraph = []
+        for line in self.lines:
+            if line.strip():
+                paragraph.append(line)
+            elif paragraph:
+                yield "\n".join(paragraph)
+                paragraph = []
+        if paragraph:
+            yield "\n".join(paragraph)
+
+    def search(
+        self, pattern: str, *, case_sensitive: bool = False
+    ) -> list[tuple[int, str]]:
+        """Search for a pattern in the document.
+
+        Args:
+            pattern: The pattern to search for.
+            case_sensitive: If True, search is case-sensitive.
+
+        Returns:
+            List of tuples (line_number, line_content) for matching lines.
+        """
+        results = []
+        search_pattern = pattern if case_sensitive else pattern.lower()
+
+        for i, line in enumerate(self.lines):
+            search_line = line if case_sensitive else line.lower()
+            if search_pattern in search_line:
+                results.append((i + 1, line))  # 1-indexed line numbers
+
+        return results
+
+    def render_markdown(self) -> str:
+        """Render markdown content to HTML."""
+        if not self.is_markdown:
+            return self.content
+        return self.md_parser.render(self.content)
+
+    def extract_headers(self) -> list[tuple[int, str]]:
+        """Extract headers from markdown documents.
+
+        Returns:
+            List of tuples (level, text) for each header.
+        """
+        headers = []
+        if self.is_markdown:
+            tokens = self.md_parser.parse(self.content)
+            i = 0
+            while i < len(tokens):
+                if tokens[i].type == "heading_open":
+                    level = int(tokens[i].tag[1])  # h1 -> 1, h2 -> 2, etc.
+                    # Next token should be inline with the content
+                    if i + 1 < len(tokens) and tokens[i + 1].type == "inline":
+                        headers.append((level, tokens[i + 1].content))
+                i += 1
+        else:
+            # For non-markdown files, look for common header patterns
+            for line in self.lines:
+                stripped = line.strip()
+                if stripped.startswith("#"):
+                    level = len(line) - len(line.lstrip("#"))
+                    text = line.lstrip("#").strip()
+                    headers.append((level, text))
+        return headers
+
+    @classmethod
+    def from_url(
+        cls,
+        url: str,
+        *,
+        lazy: bool = True,
+        timeout: float = 30.0,
+    ) -> Self:
+        """Download and create a document from a URL.
+
+        Args:
+            url: The URL to download from.
+            lazy: If True, defer loading content until needed.
+            timeout: Request timeout in seconds.
+
+        Returns:
+            A new Document instance.
+        """
+        data = None
+        size = None
+        encoding = None
+        type = None
+
+        if not lazy:
+            with httpx.Client(timeout=timeout) as client:
+                response = client.get(url)
+                response.raise_for_status()
+
+                # Always get text for documents
+                data = response.text
+                size = len(data.encode("utf-8"))
+                encoding = response.encoding
+
+                # Get content type
+                content_type = response.headers.get("content-type", "")
+                type = content_type.split(";")[0] if content_type else "text/plain"
+
+        return cls(
+            data=data,
+            type=type,
+            source=FileSource(
+                is_url=True,
+                url=url,
+                size=size,
+                encoding=encoding,
+            ),
+        )