ikichunk 0.2.0__py3-none-any.whl

ikichunk/__init__.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .facade import Partition, __version__, load_plugins
+
+partition = Partition()
+load_plugins(partition)
+
+__all__ = ["partition", "Partition", "__version__"]

ikichunk/automation/__init__.py

@@ -0,0 +1,3 @@
+from .tasks import run_task
+
+__all__ = ["run_task"]

ikichunk/automation/tasks.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+
+def run_task(name: str, **kwargs):
+    return {"name": name, **kwargs}

ikichunk/cli.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from . import partition
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="ikichunk")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    inspect_parser = subparsers.add_parser("inspect")
+    inspect_parser.add_argument("path")
+    inspect_parser.set_defaults(func=_cmd_inspect)
+
+    split_parser = subparsers.add_parser("split")
+    split_parser.add_argument("path")
+    split_parser.add_argument("--rows", type=int, default=100)
+    split_parser.set_defaults(func=_cmd_split)
+
+    return parser
+
+
+def _cmd_inspect(args: argparse.Namespace) -> int:
+    print(json.dumps(partition.inspect(args.path), indent=2))
+    return 0
+
+
+def _cmd_split(args: argparse.Namespace) -> int:
+    parts = partition.split_file(args.path, by="rows", rows=args.rows, out_dir=str(
+        Path(args.path).parent / "parts"))
+    print(json.dumps(parts, indent=2))
+    return 0
+
+
+def main() -> int:
+    parser = build_parser()
+    args = parser.parse_args()
+    return args.func(args)

ikichunk/concurrency/__init__.py

@@ -0,0 +1,3 @@
+from .parallel import pmap
+
+__all__ = ["pmap"]

ikichunk/concurrency/parallel.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import pickle
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from typing import Optional
+
+from ..exceptions import PartitionParallelError
+
+
+def _call_with_retries(func, item, retries):
+    for attempt in range(retries + 1):
+        try:
+            return func(item)
+        except Exception:
+            if attempt >= retries:
+                raise
+
+
+def pmap(func, items, *, workers: Optional[int] = None, backend: str = "thread", retries: int = 0, progress: bool = False, ordered: bool = True):
+    items = list(items)
+    if workers is None:
+        workers = max(
+            1, min(32, len(items) if hasattr(items, '__len__') else 1))
+    if backend not in {"thread", "process"}:
+        raise ValueError("backend must be 'thread' or 'process'")
+    if backend == "process":
+        try:
+            pickle.dumps(func)
+            for item in items:
+                pickle.dumps(item)
+        except Exception as exc:
+            raise PartitionParallelError(
+                "func and items must be picklable for backend='process'") from exc
+
+    if backend == "process":
+        executor_cls = ProcessPoolExecutor
+    else:
+        executor_cls = ThreadPoolExecutor
+
+    with executor_cls(max_workers=workers) as executor:
+        futures = [executor.submit(
+            _call_with_retries, func, item, retries) for item in items]
+        return [future.result() for future in futures]

ikichunk/configuration/__init__.py

@@ -0,0 +1,3 @@
+from .loader import env, load_config
+
+__all__ = ["load_config", "env"]

ikichunk/configuration/loader.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+from ..io import cat as _io_cat
+from ..io import read as _io_read
+
+
+def load_config(*sources: str, secrets: Optional[str] = None, env_prefix: str = "", **kwargs) -> dict:
+    merged: dict = {}
+    for source in sources:
+        if os.path.exists(source):
+            data = _io_read(source)
+            if isinstance(data, dict):
+                merged.update(data)
+    if secrets and os.path.exists(secrets):
+        merged.update(_parse_dotenv(secrets))
+    if env_prefix:
+        env_items = {k[len(env_prefix):]: v for k,
+                     v in os.environ.items() if k.startswith(env_prefix)}
+        merged.update(env_items)
+    return merged
+
+
+def _parse_dotenv(path: str) -> dict:
+    out: dict = {}
+    for line in _io_cat(path).splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        out[key.strip()] = value.strip()
+    return out
+
+
+def env(key: str, default: Any = None, cast=str) -> Any:
+    if key not in os.environ:
+        return default
+    value = os.environ[key]
+    if cast is str:
+        return value
+    return cast(value)

ikichunk/exceptions.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+
+class IkichunkError(Exception):
+    """Base exception for intentionally raised ikichunk errors."""
+
+
+class UnknownFormatError(IkichunkError):
+    """Raised when a file format cannot be determined safely."""
+
+
+class MissingDependencyError(IkichunkError, ImportError):
+    """Raised when an optional dependency is missing."""
+
+
+class UnsafeCommandError(IkichunkError):
+    """Raised when a shell command string is considered unsafe."""
+
+
+class PartitionParallelError(IkichunkError):
+    """Raised when process-based parallel execution cannot be pickled."""
+
+
+class UnsafeArchiveError(IkichunkError):
+    """Raised when an archive would extract outside the target directory."""

ikichunk/facade.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Iterator, Optional
+
+from . import automation as _automation
+from . import concurrency as _concurrency
+from . import configuration as _configuration
+from . import inspection as _inspection
+from . import integrity as _integrity
+from . import io as _io
+from . import net as _net
+from . import observability as _observability
+from . import partitioning as _partitioning
+from . import plugins as _plugins
+from . import resilience as _resilience
+from . import storage as _storage
+from . import system as _system
+from . import templates as _templates
+from . import validation as _validation
+
+__version__ = "0.2.0"
+
+
+class Partition:
+    def __init__(self, log_level: str = "INFO", env_prefix: str = "") -> None:
+        self.log_level = log_level
+        self.env_prefix = env_prefix
+
+    # I/O
+    def read(self, path: str, fmt: Optional[str] = None, **kwargs) -> Any:
+        return _io.read(path, fmt=fmt, **kwargs)
+
+    def write(self, path: str, data: Any, *, atomic: bool = True, backup: bool = False, fmt: Optional[str] = None, **kwargs) -> str:
+        return _io.write(path, data, atomic=atomic, backup=backup, fmt=fmt, **kwargs)
+
+    def exists(self, path: str) -> bool:
+        return _io.exists(path)
+
+    def ensure_dir(self, path: str) -> str:
+        return _io.ensure_dir(path)
+
+    def list_files(self, path: str = ".", pattern: str = "*", recursive: bool = False) -> list[str]:
+        return _io.list_files(path, pattern=pattern, recursive=recursive)
+
+    ls = list_files
+
+    def cat(self, path: str) -> str:
+        return _io.cat(path)
+
+    def stream(self, path: str, fmt: Optional[str] = None, chunk_size: Optional[int] = None) -> Iterator[Any]:
+        return _io.stream(path, fmt=fmt, chunk_size=chunk_size)
+
+    # Inspect
+    def inspect(self, obj_or_path: Any, sample: int = 3, **kwargs) -> dict:
+        return _inspection.inspect(obj_or_path, sample=sample, **kwargs)
+
+    def head(self, obj_or_path: Any, n: int = 5, **kwargs) -> Any:
+        return _inspection.head(obj_or_path, n=n, **kwargs)
+
+    def redact(self, data: dict) -> dict:
+        return _inspection.redact(data)
+
+    # Config
+    def config(self, *sources: str, secrets: Optional[str] = None, env_prefix: Optional[str] = None, **kwargs) -> dict:
+        return _configuration.load_config(*sources, secrets=secrets, env_prefix=env_prefix or self.env_prefix, **kwargs)
+
+    def env(self, key: str, default: Any = None, cast=str) -> Any:
+        return _configuration.env(key, default=default, cast=cast)
+
+    # Logging / time
+    def log(self, name: Optional[str] = None, level: Optional[str] = None, **kwargs):
+        return _observability.get_logger(name or "partition", level or self.log_level, **kwargs)
+
+    def now(self, fmt: str = "%Y-%m-%d %H:%M:%S") -> str:
+        return _observability.now(fmt)
+
+    def timer(self, name: str = "block"):
+        return _observability.timer(name)
+
+    def duration(self, seconds: float) -> str:
+        return _observability.duration(seconds)
+
+    # Retry
+    def retry(self, tries: int = 3, delay: float = 1.0, backoff: float = 2.0, exceptions=(Exception,)) -> Callable:
+        return _resilience.retry(tries=tries, delay=delay, backoff=backoff, exceptions=exceptions)
+
+    # Parallel
+    def pmap(self, func: Callable, items, *, workers: Optional[int] = None, backend: str = "thread", retries: int = 0, progress: bool = False, ordered: bool = True):
+        return _concurrency.pmap(func, items, workers=workers, backend=backend, retries=retries, progress=progress, ordered=ordered)
+
+    # Partitioning
+    def split_file(self, path: str, *, by: str = "size", size: Optional[str] = None, rows: Optional[int] = None, count: Optional[int] = None, parts: Optional[int] = None, out_dir: Optional[str] = None, fmt: Optional[str] = None, prefix: Optional[str] = None) -> list[str]:
+        return _partitioning.split_file(path, by=by, size=size, rows=rows, count=count, parts=parts, out_dir=out_dir, fmt=fmt, prefix=prefix)
+
+    def smart_split(self, path: str, *, goal: str = "parallel", workers: Optional[int] = None, chunk_size: Optional[int] = None, out_dir: Optional[str] = None, fmt: Optional[str] = None, explain: bool = False):
+        return _partitioning.smart_split(path, goal=goal, workers=workers, chunk_size=chunk_size, out_dir=out_dir, fmt=fmt, explain=explain)
+
+    def chunks(self, iterable, size: int):
+        return _partitioning.chunks(iterable, size)
+
+    def manifest(self, paths, *, hash_algo: str = "sha256") -> dict:
+        return _partitioning.manifest(paths, hash_algo=hash_algo)
+
+    # Integrity
+    def hash(self, path_or_bytes, algo: str = "sha256") -> str:
+        return _integrity.hash(path_or_bytes, algo=algo)
+
+    def hash_file(self, path: str, algo: str = "sha256") -> str:
+        return self.hash(path, algo=algo)
+
+    def verify(self, path: str, expected_hash: str, algo: str = "sha256") -> bool:
+        return _integrity.verify(path, expected_hash, algo=algo)
+
+    # Storage
+    def compress(self, path: str, *, algo: str = "gzip", out: Optional[str] = None, keep_original: bool = True) -> str:
+        return _storage.compress(path, algo=algo, out=out, keep_original=keep_original)
+
+    def decompress(self, path: str, *, out: Optional[str] = None) -> str:
+        return _storage.decompress(path, out=out)
+
+    def archive(self, source: str, out_path: str, *, fmt: str = "tar.gz") -> str:
+        return _storage.archive(source, out_path, fmt=fmt)
+
+    def extract(self, archive_path: str, *, out_dir: Optional[str] = None) -> str:
+        return _storage.extract(archive_path, out_dir=out_dir)
+
+    def register_codec(self, name: str, codec) -> None:
+        _storage.register_codec(name, codec)
+
+    # System/platform/process/net
+    def platform_info(self) -> dict:
+        return _system.platform_info()
+
+    def which(self, cmd: str) -> Optional[str]:
+        return _system.which(cmd)
+
+    def normalize_path(self, path: str) -> str:
+        return _system.normalize_path(path)
+
+    def is_running(self, pid: int) -> bool:
+        return _system.is_running(pid)
+
+    def is_port_open(self, host: str, port: int, timeout: float = 0.2) -> bool:
+        return _system.is_port_open(host, port, timeout=timeout)
+
+    def run(self, command, *, check: bool = False, shell: bool = False, timeout: Optional[float] = None):
+        return _system.run(command, check=check, shell=shell, timeout=timeout)
+
+    def fetch(self, url: str, *, timeout: float = 5.0) -> str:
+        return _net.fetch(url, timeout=timeout)
+
+    def render(self, template: str, variables: dict, *, out: Optional[str] = None, strict: bool = True) -> str:
+        return _templates.render(template, variables, out=out, strict=strict)
+
+    # Validation and plugins
+    def register_format(self, name: str, handler, extensions: Optional[list[str]] = None) -> None:
+        _io.register_format(name, handler, extensions=extensions)
+
+    def register_split_goal(self, name: str, fn: Callable[[int, dict, int], dict]) -> None:
+        _partitioning.register_goal(name, fn)
+
+    def validate(self, value: Any, schema: Optional[dict] = None) -> bool:
+        return _validation.validate(value, schema=schema)
+
+
+def load_plugins(instance: Partition) -> None:
+    _plugins.load_plugins(instance)

ikichunk/inspection/__init__.py

@@ -0,0 +1,3 @@
+from .inspectors import head, inspect, redact, register_inspector
+
+__all__ = ["inspect", "head", "redact", "register_inspector"]

ikichunk/inspection/inspectors.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import functools
+from pathlib import Path
+from typing import Any
+
+from ..io import read as _io_read
+
+_SECRET_KEY_PATTERNS = ("_key", "_token", "_secret",
+                        "_password", "password", "secret", "token")
+
+
+def redact(d: dict) -> dict:
+    out = {}
+    for key, value in d.items():
+        if any(pattern in str(key).lower() for pattern in _SECRET_KEY_PATTERNS):
+            out[key] = "<redacted>"
+        else:
+            out[key] = value
+    return out
+
+
+@functools.singledispatch
+def _inspect_obj(obj: Any, sample: int = 3) -> dict:
+    return {"type": type(obj).__name__, "repr": repr(obj)[:200]}
+
+
+@_inspect_obj.register
+def _inspect_list(obj: list, sample: int = 3) -> dict:
+    info = {"type": "list", "length": len(obj)}
+    if obj:
+        info["sample"] = obj[:sample]
+    return info
+
+
+@_inspect_obj.register
+def _inspect_dict(obj: dict, sample: int = 3) -> dict:
+    return {"type": "dict", "keys": list(obj.keys()), "sample": redact(dict(list(obj.items())[:sample]))}
+
+
+@_inspect_obj.register
+def _inspect_str(obj: str, sample: int = 3) -> dict:
+    lines = obj.splitlines()
+    return {"type": "text", "length": len(obj), "lines": len(lines), "sample": lines[:sample]}
+
+
+def register_inspector(python_type, func) -> None:
+    _inspect_obj.register(python_type, func)
+
+
+def inspect(obj_or_path: Any, sample: int = 3, **kwargs) -> dict:
+    if isinstance(obj_or_path, str) and Path(obj_or_path).exists():
+        return _inspect_obj(_io_read(obj_or_path), sample=sample)
+    return _inspect_obj(obj_or_path, sample=sample)
+
+
+def head(obj_or_path: Any, n: int = 5, **kwargs) -> Any:
+    info = inspect(obj_or_path, sample=n, **kwargs)
+    return info.get("sample", info)

ikichunk/integrity/__init__.py

@@ -0,0 +1,3 @@
+from .hashing import hash, verify
+
+__all__ = ["hash", "verify"]

ikichunk/integrity/hashing.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+from typing import Any
+
+
+def hash(path_or_bytes, algo: str = "sha256") -> str:
+    hasher = hashlib.new(algo)
+    if isinstance(path_or_bytes, (bytes, bytearray)):
+        hasher.update(bytes(path_or_bytes))
+        return hasher.hexdigest()
+    path = Path(path_or_bytes)
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def verify(path: str, expected_hash: str, algo: str = "sha256") -> bool:
+    return hash(path, algo=algo) == expected_hash

ikichunk/io/__init__.py

@@ -0,0 +1,6 @@
+from .core import cat, ensure_dir, exists, list_files, read, write
+from .formats import FormatHandler, register_format
+from .streaming import stream
+
+__all__ = ["read", "write", "exists", "ensure_dir", "list_files",
+           "cat", "stream", "FormatHandler", "register_format"]

ikichunk/io/atomic.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+from typing import Callable
+
+
+def atomic_write(target: Path, serialize: Callable[[Path], None]) -> None:
+    target.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_path = tempfile.mkstemp(
+        prefix=target.name + ".", dir=str(target.parent))
+    os.close(fd)
+    tmp = Path(tmp_path)
+    try:
+        serialize(tmp)
+        os.replace(tmp, target)
+    finally:
+        if tmp.exists():
+            tmp.unlink(missing_ok=True)

ikichunk/io/core.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Any, List, Optional
+
+from .atomic import atomic_write
+from .formats import detect_fmt, get_handler
+
+
+def read(path: str, fmt: Optional[str] = None, **kwargs) -> Any:
+    p = Path(path)
+    handler = get_handler(detect_fmt(p, fmt))
+    return handler.read(p, kwargs)
+
+
+def write(path: str, data: Any, *, atomic: bool = True, backup: bool = False, fmt: Optional[str] = None, **kwargs) -> str:
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    resolved_fmt = fmt or detect_fmt(p, None)
+    if atomic:
+        atomic_write(p, lambda tmp: _serialize(
+            tmp, data, fmt=resolved_fmt, **kwargs))
+    else:
+        _serialize(p, data, fmt=resolved_fmt, **kwargs)
+    if backup and p.exists():
+        backup_path = p.with_suffix(p.suffix + ".bak")
+        shutil.copy2(p, backup_path)
+    return str(p)
+
+
+def _serialize(path: Path, data: Any, *, fmt: Optional[str] = None, **kwargs) -> None:
+    handler = get_handler(detect_fmt(path, fmt))
+    handler.write(path, data, kwargs)
+
+
+def exists(path: str) -> bool:
+    return Path(path).exists()
+
+
+def ensure_dir(path: str) -> str:
+    Path(path).mkdir(parents=True, exist_ok=True)
+    return str(path)
+
+
+def list_files(path: str = ".", pattern: str = "*", recursive: bool = False) -> List[str]:
+    base = Path(path)
+    if recursive:
+        matches = [p for p in base.rglob(pattern) if p.is_file()]
+    else:
+        matches = [p for p in base.glob(pattern) if p.is_file()]
+    return sorted(str(p) for p in matches)
+
+
+def cat(path: str) -> str:
+    return read(path, fmt="text")