npcpy 1.3.17__py3-none-any.whl → 1.3.19__py3-none-any.whl

npcpy/data/web.py

@@ -66,7 +66,6 @@ def search_perplexity(
         "top_p": top_p,
         "return_images": False,
         "return_related_questions": False,
-        "search_recency_filter": "month",
         "top_k": 0,
         "stream": False,
         "presence_penalty": 0,

npcpy/memory/command_history.py

@@ -611,7 +611,9 @@ class CommandHistory:
             Column('reasoning_content', Text),  # For thinking tokens / chain of thought
             Column('tool_calls', Text),  # JSON array of tool calls made by assistant
             Column('tool_results', Text),  # JSON array of tool call results
-            Column('parent_message_id', String(50))  # Links assistant response to parent user message for broadcast grouping
+            Column('parent_message_id', String(50)),  # Links assistant response to parent user message for broadcast grouping
+            Column('device_id', String(255)),  # UUID of the device that created this message
+            Column('device_name', String(255))  # Human-readable device name
         )
         
         Table('message_attachments', metadata,
@@ -867,6 +869,8 @@ class CommandHistory:
         tool_calls=None,
         tool_results=None,
         parent_message_id=None,
+        device_id=None,
+        device_name=None,
     ):
         if isinstance(content, (dict, list)):
             content = json.dumps(content, cls=CustomJSONEncoder)
@@ -882,14 +886,15 @@ class CommandHistory:
 
         stmt = """
             INSERT INTO conversation_history
-            (message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team, reasoning_content, tool_calls, tool_results, parent_message_id)
-            VALUES (:message_id, :timestamp, :role, :content, :conversation_id, :directory_path, :model, :provider, :npc, :team, :reasoning_content, :tool_calls, :tool_results, :parent_message_id)
+            (message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team, reasoning_content, tool_calls, tool_results, parent_message_id, device_id, device_name)
+            VALUES (:message_id, :timestamp, :role, :content, :conversation_id, :directory_path, :model, :provider, :npc, :team, :reasoning_content, :tool_calls, :tool_results, :parent_message_id, :device_id, :device_name)
         """
         params = {
             "message_id": message_id, "timestamp": timestamp, "role": role, "content": content,
             "conversation_id": conversation_id, "directory_path": normalized_directory_path, "model": model,
             "provider": provider, "npc": npc, "team": team, "reasoning_content": reasoning_content,
-            "tool_calls": tool_calls, "tool_results": tool_results, "parent_message_id": parent_message_id
+            "tool_calls": tool_calls, "tool_results": tool_results, "parent_message_id": parent_message_id,
+            "device_id": device_id, "device_name": device_name
         }
         with self.engine.begin() as conn:
             conn.execute(text(stmt), params)
@@ -1461,6 +1466,8 @@ def save_conversation_message(
     tool_results: List[Dict] = None,
     parent_message_id: str = None,
     skip_if_exists: bool = True,
+    device_id: str = None,
+    device_name: str = None,
     ):
     """
     Saves a conversation message linked to a conversation ID with optional attachments.
@@ -1495,7 +1502,9 @@ def save_conversation_message(
         reasoning_content=reasoning_content,
         tool_calls=tool_calls,
         tool_results=tool_results,
-        parent_message_id=parent_message_id)
+        parent_message_id=parent_message_id,
+        device_id=device_id,
+        device_name=device_name)
 def retrieve_last_conversation(
     command_history: CommandHistory, conversation_id: str
     ) -> str:

npcpy/ml_funcs.py

@@ -16,7 +16,6 @@ Same interface pattern as llm_funcs:
 from __future__ import annotations
 import copy
 import itertools
-import pickle
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
@@ -708,21 +707,67 @@ def cross_validate(
 
 # ==================== Utility Functions ====================
 
-def serialize_model(model: Any, path: str = None) -> bytes:
-    """Serialize model to bytes or file"""
-    data = pickle.dumps(model)
-    if path:
-        with open(path, 'wb') as f:
-            f.write(data)
-    return data
-
-
-def deserialize_model(data: Union[bytes, str]) -> Any:
-    """Deserialize model from bytes or file path"""
-    if isinstance(data, str):
-        with open(data, 'rb') as f:
-            data = f.read()
-    return pickle.loads(data)
+def serialize_model(model: Any, path: str, format: str = "joblib") -> None:
+    """
+    Serialize model to file using safe formats (no pickle).
+
+    Args:
+        model: The model to serialize
+        path: File path to write to (required)
+        format: Serialization format - "joblib" (default) or "safetensors"
+
+    Raises:
+        ImportError: If required library is not installed
+        ValueError: If format is not supported for the model type
+    """
+    if format == "safetensors":
+        from safetensors.torch import save_file
+        if hasattr(model, 'state_dict'):
+            save_file(model.state_dict(), path)
+        else:
+            raise ValueError("safetensors format requires model with state_dict (PyTorch)")
+    elif format == "joblib":
+        import joblib
+        joblib.dump(model, path)
+    else:
+        raise ValueError(f"Unsupported format: {format}. Use 'joblib' or 'safetensors'.")
+
+
+def deserialize_model(path: str, format: str = "auto") -> Any:
+    """
+    Deserialize model from file using safe formats (no pickle).
+
+    Args:
+        path: File path to load from
+        format: "auto" (detect from extension), "joblib", or "safetensors"
+
+    Returns:
+        The deserialized model
+
+    Raises:
+        ImportError: If required library is not installed
+        ValueError: If format cannot be determined
+    """
+    # Auto-detect format from extension
+    if format == "auto":
+        if path.endswith('.safetensors'):
+            format = "safetensors"
+        elif path.endswith('.joblib'):
+            format = "joblib"
+        else:
+            raise ValueError(
+                f"Cannot auto-detect format for {path}. "
+                "Use .joblib or .safetensors extension, or specify format explicitly."
+            )
+
+    if format == "safetensors":
+        from safetensors.torch import load_file
+        return load_file(path)
+    elif format == "joblib":
+        import joblib
+        return joblib.load(path)
+    else:
+        raise ValueError(f"Unsupported format: {format}. Use 'joblib' or 'safetensors'.")
 
 
 def get_model_params(model: Any) -> Dict[str, Any]:

npcpy/npc_array.py

@@ -20,7 +20,6 @@ Example:
 from __future__ import annotations
 import copy
 import itertools
-import pickle
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field
 from typing import (
@@ -47,6 +46,7 @@ class OpType(Enum):
     REDUCE = "reduce"
     CHAIN = "chain"
     EVOLVE = "evolve"
+    JINX = "jinx"  # Execute a Jinx workflow across models
 
 
 @dataclass
@@ -328,6 +328,61 @@ class NPCArray:
 
         return cls(specs)
 
+    @classmethod
+    def from_matrix(
+        cls,
+        matrix: List[Dict[str, Any]]
+    ) -> 'NPCArray':
+        """
+        Create NPCArray from a matrix of model configurations.
+
+        This is particularly useful for defining model arrays in Jinx templates
+        where you want explicit control over each model configuration.
+
+        Args:
+            matrix: List of model configuration dicts. Each dict should have:
+                - 'model': model name/reference (required)
+                - 'provider': provider name (optional)
+                - 'type': model type - 'llm', 'npc', 'sklearn', 'torch' (default: 'llm')
+                - Any additional config parameters
+
+        Example:
+            >>> # In a Jinx template, define a matrix of models:
+            >>> matrix = [
+            ...     {'model': 'gpt-4', 'provider': 'openai', 'temperature': 0.7},
+            ...     {'model': 'claude-3-opus', 'provider': 'anthropic', 'temperature': 0.5},
+            ...     {'model': 'llama3.2', 'provider': 'ollama', 'temperature': 0.8},
+            ... ]
+            >>> arr = NPCArray.from_matrix(matrix)
+
+            >>> # Mixed model types:
+            >>> matrix = [
+            ...     {'model': 'gpt-4', 'type': 'llm', 'provider': 'openai'},
+            ...     {'model': my_npc, 'type': 'npc'},
+            ...     {'model': sklearn_model, 'type': 'sklearn'},
+            ... ]
+        """
+        specs = []
+        for config in matrix:
+            model_type = config.get('type', 'llm')
+            model_ref = config.get('model')
+            provider = config.get('provider')
+
+            # Extract config params (everything except type, model, provider)
+            extra_config = {
+                k: v for k, v in config.items()
+                if k not in ('type', 'model', 'provider')
+            }
+
+            specs.append(ModelSpec(
+                model_type=model_type,
+                model_ref=model_ref,
+                provider=provider,
+                config=extra_config
+            ))
+
+        return cls(specs)
+
     # ==================== Properties ====================
 
     @property
@@ -490,6 +545,43 @@ class NPCArray:
 
         return NPCArray(self._specs, new_node)
 
+    def jinx(
+        self,
+        jinx_name: str,
+        inputs: Optional[Dict[str, Any]] = None,
+        **kwargs
+    ) -> 'LazyResult':
+        """
+        Execute a Jinx workflow across all models in the array.
+
+        Each model in the array will be used as the 'npc' context for the jinx,
+        allowing you to run the same workflow template with different models.
+
+        Args:
+            jinx_name: Name of the jinx workflow to execute (e.g., 'analyze', 'summarize')
+            inputs: Input values for the jinx template variables
+            **kwargs: Additional execution parameters
+
+        Returns:
+            LazyResult with workflow outputs from each model
+
+        Example:
+            >>> models = NPCArray.from_llms(['gpt-4', 'claude-3'])
+            >>> results = models.jinx('analyze', inputs={'topic': 'AI safety'}).collect()
+        """
+        new_node = GraphNode(
+            op_type=OpType.JINX,
+            params={
+                "jinx_name": jinx_name,
+                "inputs": inputs or {},
+                **kwargs
+            },
+            parents=[self._graph],
+            shape=(len(self._specs),)
+        )
+
+        return LazyResult(self._specs, new_node)
+
 
 class LazyResult:
     """
@@ -792,6 +884,7 @@ class GraphExecutor:
             OpType.REDUCE: self._exec_reduce,
             OpType.CHAIN: self._exec_chain,
             OpType.EVOLVE: self._exec_evolve,
+            OpType.JINX: self._exec_jinx,
         }
 
         handler = handlers.get(node.op_type)
@@ -1136,6 +1229,61 @@ class GraphExecutor:
             metadata={"operation": "evolve", "generation": 1}
         )
 
+    def _exec_jinx(self, node, specs, prompts, parents) -> ResponseTensor:
+        """Execute a Jinx workflow across models"""
+        from npcpy.npc_compiler import NPC, Jinx
+
+        jinx_name = node.params.get("jinx_name")
+        inputs = node.params.get("inputs", {})
+        extra_kwargs = {k: v for k, v in node.params.items()
+                       if k not in ("jinx_name", "inputs")}
+
+        results = []
+
+        def run_jinx_single(spec: ModelSpec) -> str:
+            """Run jinx for a single model spec"""
+            try:
+                if spec.model_type == "npc":
+                    # Use the NPC directly
+                    npc = spec.model_ref
+                else:
+                    # Create a temporary NPC with the model
+                    npc = NPC(
+                        name=f"array_npc_{spec.model_ref}",
+                        model=spec.model_ref,
+                        provider=spec.provider
+                    )
+
+                # Execute the jinx
+                result = npc.execute_jinx(
+                    jinx_name=jinx_name,
+                    input_values=inputs,
+                    **extra_kwargs
+                )
+                return result.get("output", str(result))
+            except Exception as e:
+                return f"Error: {e}"
+
+        if self.parallel and len(specs) > 1:
+            with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+                futures = {executor.submit(run_jinx_single, spec): i
+                          for i, spec in enumerate(specs)}
+                results = [None] * len(specs)
+                for future in as_completed(futures):
+                    idx = futures[future]
+                    try:
+                        results[idx] = future.result()
+                    except Exception as e:
+                        results[idx] = f"Error: {e}"
+        else:
+            results = [run_jinx_single(spec) for spec in specs]
+
+        return ResponseTensor(
+            data=np.array(results, dtype=object),
+            model_specs=specs,
+            metadata={"operation": "jinx", "jinx_name": jinx_name, **inputs}
+        )
+
 
 def _compute_response_variance(responses: List[str]) -> float:
     """Compute semantic variance across responses"""

npcpy/npc_compiler.py

@@ -52,6 +52,7 @@ import fnmatch
 import subprocess
 from typing import Any, Dict, List, Optional, Union, Callable, Tuple
 from jinja2 import Environment, FileSystemLoader, Template, Undefined, DictLoader
+from jinja2.sandbox import SandboxedEnvironment
 from sqlalchemy import create_engine, text
 import npcpy as npy 
 from npcpy.tools import auto_tools
@@ -231,7 +232,8 @@ def load_yaml_file(file_path):
 
         # First pass: render Jinja2 templates to produce valid YAML
         # This allows {% if %} and other control structures to work
-        jinja_env = Environment(undefined=SilentUndefined)
+        # Use SandboxedEnvironment to prevent template injection attacks
+        jinja_env = SandboxedEnvironment(undefined=SilentUndefined)
         # Configure tojson filter to handle SilentUndefined
         jinja_env.policies['json.dumps_function'] = _json_dumps_with_undefined
         template = jinja_env.from_string(content)
@@ -694,7 +696,8 @@ class Jinx:
                 jinja_env: Optional[Environment] = None):
         
         if jinja_env is None:
-            jinja_env = Environment(
+            # Use SandboxedEnvironment to prevent template injection attacks
+            jinja_env = SandboxedEnvironment(
                 loader=DictLoader({}),
                 undefined=SilentUndefined,
             )
@@ -771,21 +774,24 @@ class Jinx:
         
         self._log_debug(f"DEBUG: Executing step '{step_name}' with rendered code: {rendered_code}")
 
+        # Import NPCArray for array operations in jinx
+        from npcpy.npc_array import NPCArray, infer_matrix, ensemble_vote
+
         exec_globals = {
             "__builtins__": __builtins__,
             "npc": active_npc,
             "context": context, # Pass context by reference
-            "math": math, 
-            "random": random, 
+            "math": math,
+            "random": random,
             "datetime": datetime,
             "Image": Image,
             "pd": pd,
             "plt": plt,
-            "sys": sys, 
+            "sys": sys,
             "subprocess": subprocess,
             "np": np,
             "os": os,
-            're': re, 
+            're': re,
             "json": json,
             "Path": pathlib.Path,
             "fnmatch": fnmatch,
@@ -793,6 +799,10 @@ class Jinx:
             "subprocess": subprocess,
             "get_llm_response": npy.llm_funcs.get_llm_response,
             "CommandHistory": CommandHistory,
+            # NPCArray support for compute graph operations in jinx
+            "NPCArray": NPCArray,
+            "infer_matrix": infer_matrix,
+            "ensemble_vote": ensemble_vote,
         }
         
         if extra_globals:
@@ -1261,7 +1271,8 @@ class NPC:
             dirs.append(self.jinxs_directory)
             
         # This jinja_env is for the *second pass* (runtime variable resolution in Jinx.execute)
-        self.jinja_env = Environment(
+        # Use SandboxedEnvironment to prevent template injection attacks
+        self.jinja_env = SandboxedEnvironment(
             loader=FileSystemLoader([
                 os.path.expanduser(d) for d in dirs
             ]),
@@ -1389,13 +1400,13 @@ class NPC:
 
             combined_raw_jinxs_dict = {j.jinx_name: j for j in all_available_raw_jinxs}
 
-            npc_first_pass_jinja_env = Environment(undefined=SilentUndefined)
-            
+            npc_first_pass_jinja_env = SandboxedEnvironment(undefined=SilentUndefined)
+
             jinx_macro_globals = {}
             for raw_jinx in combined_raw_jinxs_dict.values():
                 def create_jinx_callable(jinx_obj_in_closure):
                     def callable_jinx(**kwargs):
-                        temp_jinja_env = Environment(undefined=SilentUndefined)
+                        temp_jinja_env = SandboxedEnvironment(undefined=SilentUndefined)
                         rendered_target_steps = []
                         for target_step in jinx_obj_in_closure._raw_steps:
                             temp_rendered_step = {}
@@ -2506,7 +2517,7 @@ class Team:
         self._raw_jinxs_list: List['Jinx'] = [] # Temporary storage for raw Team-level Jinx objects
         self.jinx_tool_catalog: Dict[str, Dict[str, Any]] = {}  # Jinx-derived tool defs ready for MCP/LLM
         
-        self.jinja_env_for_first_pass = Environment(undefined=SilentUndefined) # Env for macro expansion
+        self.jinja_env_for_first_pass = SandboxedEnvironment(undefined=SilentUndefined) # Env for macro expansion
 
         self.db_conn = db_conn
         self.team_path = os.path.expanduser(team_path) if team_path else None
@@ -2700,7 +2711,7 @@ class Team:
                 def callable_jinx(**kwargs):
                     # This callable will be invoked by the Jinja renderer during the first pass.
                     # It needs to render the target Jinx's *raw* steps with the provided kwargs.
-                    temp_jinja_env = Environment(undefined=SilentUndefined)
+                    temp_jinja_env = SandboxedEnvironment(undefined=SilentUndefined)
                     
                     rendered_target_steps = []
                     for target_step in jinx_obj_in_closure._raw_steps:

npcpy/npc_sysenv.py

@@ -15,6 +15,129 @@ import json
 
 import requests
 ON_WINDOWS = platform.system() == "Windows"
+ON_MACOS = platform.system() == "Darwin"
+
+
+# ==================== XDG/Platform-Specific Paths ====================
+
+def get_data_dir() -> str:
+    """
+    Get the platform-specific data directory for npcsh.
+
+    Returns:
+        - Linux: $XDG_DATA_HOME/npcsh or ~/.local/share/npcsh
+        - macOS: ~/Library/Application Support/npcsh
+        - Windows: %LOCALAPPDATA%/npcsh or ~/AppData/Local/npcsh
+
+    Falls back to ~/.npcsh for backwards compatibility if the new location
+    doesn't exist but the old one does.
+    """
+    if ON_WINDOWS:
+        base = os.environ.get('LOCALAPPDATA', os.path.expanduser('~/AppData/Local'))
+        new_path = os.path.join(base, 'npcsh')
+    elif ON_MACOS:
+        new_path = os.path.expanduser('~/Library/Application Support/npcsh')
+    else:
+        # Linux/Unix - use XDG Base Directory Specification
+        xdg_data = os.environ.get('XDG_DATA_HOME', os.path.expanduser('~/.local/share'))
+        new_path = os.path.join(xdg_data, 'npcsh')
+
+    # Backwards compatibility: if old path exists but new doesn't, use old
+    old_path = os.path.expanduser('~/.npcsh')
+    if os.path.exists(old_path) and not os.path.exists(new_path):
+        return old_path
+
+    return new_path
+
+
+def get_config_dir() -> str:
+    """
+    Get the platform-specific config directory for npcsh.
+
+    Returns:
+        - Linux: $XDG_CONFIG_HOME/npcsh or ~/.config/npcsh
+        - macOS: ~/Library/Application Support/npcsh (same as data on macOS)
+        - Windows: %APPDATA%/npcsh or ~/AppData/Roaming/npcsh
+
+    Falls back to ~/.npcsh for backwards compatibility if the new location
+    doesn't exist but the old one does.
+    """
+    if ON_WINDOWS:
+        base = os.environ.get('APPDATA', os.path.expanduser('~/AppData/Roaming'))
+        new_path = os.path.join(base, 'npcsh')
+    elif ON_MACOS:
+        new_path = os.path.expanduser('~/Library/Application Support/npcsh')
+    else:
+        # Linux/Unix - use XDG Base Directory Specification
+        xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config'))
+        new_path = os.path.join(xdg_config, 'npcsh')
+
+    # Backwards compatibility: if old path exists but new doesn't, use old
+    old_path = os.path.expanduser('~/.npcsh')
+    if os.path.exists(old_path) and not os.path.exists(new_path):
+        return old_path
+
+    return new_path
+
+
+def get_cache_dir() -> str:
+    """
+    Get the platform-specific cache directory for npcsh.
+
+    Returns:
+        - Linux: $XDG_CACHE_HOME/npcsh or ~/.cache/npcsh
+        - macOS: ~/Library/Caches/npcsh
+        - Windows: %LOCALAPPDATA%/npcsh/cache
+    """
+    if ON_WINDOWS:
+        base = os.environ.get('LOCALAPPDATA', os.path.expanduser('~/AppData/Local'))
+        return os.path.join(base, 'npcsh', 'cache')
+    elif ON_MACOS:
+        return os.path.expanduser('~/Library/Caches/npcsh')
+    else:
+        xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+        return os.path.join(xdg_cache, 'npcsh')
+
+
+def get_npcshrc_path() -> str:
+    """
+    Get the path to the npcshrc config file.
+
+    Returns the platform-appropriate config file path.
+    Falls back to ~/.npcshrc for backwards compatibility.
+    """
+    old_path = os.path.expanduser('~/.npcshrc')
+    if os.path.exists(old_path):
+        return old_path
+
+    config_dir = get_config_dir()
+    return os.path.join(config_dir, 'npcshrc')
+
+
+def get_history_db_path() -> str:
+    """
+    Get the path to the history database.
+
+    Returns the platform-appropriate database path.
+    Falls back to ~/npcsh_history.db for backwards compatibility.
+    """
+    old_path = os.path.expanduser('~/npcsh_history.db')
+    if os.path.exists(old_path):
+        return old_path
+
+    data_dir = get_data_dir()
+    return os.path.join(data_dir, 'history.db')
+
+
+def get_models_dir() -> str:
+    """Get the directory for storing models."""
+    return os.path.join(get_data_dir(), 'models')
+
+
+def ensure_npcsh_dirs() -> None:
+    """Ensure all npcsh directories exist."""
+    for dir_path in [get_data_dir(), get_config_dir(), get_cache_dir(), get_models_dir()]:
+        os.makedirs(dir_path, exist_ok=True)
 
 try:
     if not ON_WINDOWS:
@@ -309,11 +432,13 @@ def get_locally_available_models(project_directory, airplane_mode=False):
         logging.info(f"Error loading Ollama models or timed out: {e}")
 
     # Scan for local GGUF/GGML models
+    models_dir = get_models_dir()
     gguf_dirs = [
-        os.path.expanduser('~/.npcsh/models/gguf'),
-        os.path.expanduser('~/.npcsh/models'),
+        os.path.join(models_dir, 'gguf'),
+        models_dir,
         os.path.expanduser('~/models'),
-        os.path.expanduser('~/.cache/huggingface/hub'),
+        os.path.join(get_cache_dir(), 'huggingface/hub'),
+        os.path.expanduser('~/.cache/huggingface/hub'),  # Fallback for existing installs
     ]
     env_gguf_dir = os.environ.get('NPCSH_GGUF_DIR')
     if env_gguf_dir:
@@ -358,6 +483,31 @@ def get_locally_available_models(project_directory, airplane_mode=False):
     except Exception as e:
         logging.debug(f"llama.cpp server not available: {e}")
 
+    # Check for MLX server (OpenAI-compatible API on port 8000)
+    try:
+        import requests
+        response = requests.get('http://127.0.0.1:8000/v1/models', timeout=1)
+        if response.ok:
+            data = response.json()
+            for model in data.get('data', []):
+                model_id = model.get('id', model.get('name', 'unknown'))
+                available_models[model_id] = "mlx"
+    except Exception as e:
+        logging.debug(f"MLX server not available: {e}")
+
+    # Also check common alternative MLX port 5000
+    try:
+        import requests
+        response = requests.get('http://127.0.0.1:5000/v1/models', timeout=1)
+        if response.ok:
+            data = response.json()
+            for model in data.get('data', []):
+                model_id = model.get('id', model.get('name', 'unknown'))
+                if model_id not in available_models:  # Avoid duplicates
+                    available_models[model_id] = "mlx"
+    except Exception as e:
+        logging.debug(f"MLX server (port 5000) not available: {e}")
+
     # Scan for LoRA adapters (fine-tuned models with adapter_config.json)
     lora_dirs = [
         os.path.expanduser('~/.npcsh/models'),
@@ -1055,13 +1205,13 @@ def lookup_provider(model: str) -> str:
 
 def load_custom_providers():
     """
-    Load custom provider configurations from .npcshrc
-    
+    Load custom provider configurations from npcshrc config file.
+
     Returns:
         dict: Custom provider configurations keyed by provider name
     """
     custom_providers = {}
-    npcshrc_path = os.path.expanduser("~/.npcshrc")
+    npcshrc_path = get_npcshrc_path()
     
     if os.path.exists(npcshrc_path):
         with open(npcshrc_path, "r") as f:

npcpy/serve.py

@@ -58,12 +58,12 @@ import base64
 import shutil
 import uuid
 
-from npcpy.llm_funcs import gen_image, breathe                                                                                                                                                                
+from npcpy.llm_funcs import gen_image, gen_video, breathe                                                                                                                                                                
 
 from sqlalchemy import create_engine, text
 from sqlalchemy.orm import sessionmaker
 
-from npcpy.npc_sysenv import get_locally_available_models
+from npcpy.npc_sysenv import get_locally_available_models, get_data_dir, get_models_dir, get_cache_dir
 from npcpy.memory.command_history import (
     CommandHistory,
     save_conversation_message,
@@ -2675,7 +2675,7 @@ def inject_individuals():
 
 @app.route("/api/ml/train", methods=["POST"])
 def train_ml_model():
-    import pickle
+    import joblib
     import numpy as np
     from sklearn.linear_model import LinearRegression, LogisticRegression
     from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
@@ -2683,7 +2683,7 @@ def train_ml_model():
     from sklearn.cluster import KMeans
     from sklearn.model_selection import train_test_split
     from sklearn.metrics import mean_squared_error, r2_score, accuracy_score
-    
+
     data = request.json
     model_name = data.get("name")
     model_type = data.get("type")
@@ -2691,13 +2691,13 @@ def train_ml_model():
     features = data.get("features")
     training_data = data.get("data")
     hyperparams = data.get("hyperparameters", {})
-    
+
     df = pd.DataFrame(training_data)
     X = df[features].values
-    
+
     metrics = {}
     model = None
-    
+
     if model_type == "linear_regression":
         y = df[target].values
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
@@ -2708,7 +2708,7 @@ def train_ml_model():
             "r2_score": r2_score(y_test, y_pred),
             "rmse": np.sqrt(mean_squared_error(y_test, y_pred))
         }
-    
+
     elif model_type == "logistic_regression":
         y = df[target].values
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
@@ -2716,7 +2716,7 @@ def train_ml_model():
         model.fit(X_train, y_train)
         y_pred = model.predict(X_test)
         metrics = {"accuracy": accuracy_score(y_test, y_pred)}
-    
+
     elif model_type == "random_forest":
         y = df[target].values
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
@@ -2727,13 +2727,13 @@ def train_ml_model():
             "r2_score": r2_score(y_test, y_pred),
             "rmse": np.sqrt(mean_squared_error(y_test, y_pred))
         }
-    
+
     elif model_type == "clustering":
         n_clusters = hyperparams.get("n_clusters", 3)
         model = KMeans(n_clusters=n_clusters)
         labels = model.fit_predict(X)
         metrics = {"inertia": model.inertia_, "n_clusters": n_clusters}
-    
+
     elif model_type == "gradient_boost":
         y = df[target].values
         X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
@@ -2744,19 +2744,18 @@ def train_ml_model():
             "r2_score": r2_score(y_test, y_pred),
             "rmse": np.sqrt(mean_squared_error(y_test, y_pred))
         }
-    
+
     model_id = f"{model_name}_{int(time.time())}"
-    model_path = os.path.expanduser(f"~/.npcsh/models/{model_id}.pkl")
+    model_path = os.path.join(get_models_dir(), f"{model_id}.joblib")
     os.makedirs(os.path.dirname(model_path), exist_ok=True)
-    
-    with open(model_path, 'wb') as f:
-        pickle.dump({
-            "model": model,
-            "features": features,
-            "target": target,
-            "type": model_type
-        }, f)
-    
+
+    joblib.dump({
+        "model": model,
+        "features": features,
+        "target": target,
+        "type": model_type
+    }, model_path)
+
     return jsonify({
         "model_id": model_id,
         "metrics": metrics,
@@ -2766,26 +2765,25 @@ def train_ml_model():
 
 @app.route("/api/ml/predict", methods=["POST"])
 def ml_predict():
-    import pickle
-    
+    import joblib
+
     data = request.json
     model_name = data.get("model_name")
     input_data = data.get("input_data")
-    
-    model_dir = os.path.expanduser("~/.npcsh/models/")
+
+    model_dir = get_models_dir()
     model_files = [f for f in os.listdir(model_dir) if f.startswith(model_name)]
-    
+
     if not model_files:
         return jsonify({"error": f"Model {model_name} not found"})
-    
+
     model_path = os.path.join(model_dir, model_files[0])
-    
-    with open(model_path, 'rb') as f:
-        model_data = pickle.load(f)
-    
+
+    model_data = joblib.load(model_path)
+
     model = model_data["model"]
     prediction = model.predict([input_data])
-    
+
     return jsonify({
         "prediction": prediction.tolist(),
         "error": None
@@ -4335,6 +4333,89 @@ def get_image_models_api():
         return jsonify({"models": [], "error": str(e)}), 500
 
 
+@app.route("/api/generate_video", methods=["POST"])
+def generate_video_api():
+    """
+    API endpoint for video generation.
+    """
+    try:
+        data = request.get_json()
+        prompt = data.get("prompt", "")
+        model = data.get("model", "veo-3.1-generate-preview")
+        provider = data.get("provider", "gemini")
+        duration = data.get("duration", 5)
+        output_dir = data.get("output_dir")  # Optional user-specified path
+        negative_prompt = data.get("negative_prompt", "")
+        reference_image = data.get("reference_image")  # Optional base64 image
+
+        if not prompt:
+            return jsonify({"error": "Prompt is required"}), 400
+
+        # Create output directory - use user-specified path or default to ~/.npcsh/videos
+        if output_dir:
+            save_dir = os.path.expanduser(output_dir)
+        else:
+            save_dir = os.path.expanduser("~/.npcsh/videos")
+        os.makedirs(save_dir, exist_ok=True)
+
+        # Generate unique filename
+        timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+        output_filename = f"video_{timestamp}.mp4"
+        output_path = os.path.join(save_dir, output_filename)
+
+        # Calculate num_frames based on duration (assuming ~25fps for diffusers)
+        num_frames = int(duration * 25) if provider == "diffusers" else 25
+
+        print(f"Generating video with model={model}, provider={provider}, duration={duration}s")
+
+        result = gen_video(
+            prompt=prompt,
+            model=model,
+            provider=provider,
+            output_path=output_path,
+            num_frames=num_frames,
+            negative_prompt=negative_prompt,
+        )
+
+        if result and "output" in result:
+            # Read the generated video file and encode to base64
+            video_path = output_path
+            if os.path.exists(video_path):
+                with open(video_path, "rb") as f:
+                    video_data = f.read()
+                video_base64 = base64.b64encode(video_data).decode("utf-8")
+
+                return jsonify({
+                    "success": True,
+                    "video_path": video_path,
+                    "video_base64": f"data:video/mp4;base64,{video_base64}",
+                    "message": result.get("output", "Video generated successfully")
+                })
+            else:
+                return jsonify({"error": "Video file was not created"}), 500
+        else:
+            return jsonify({"error": result.get("output", "Video generation failed")}), 500
+
+    except Exception as e:
+        print(f"Error generating video: {e}")
+        traceback.print_exc()
+        return jsonify({"error": str(e)}), 500
+
+
+@app.route("/api/video_models", methods=["GET"])
+def get_video_models_api():
+    """
+    API endpoint to retrieve available video generation models.
+    """
+    video_models = [
+        # Google Veo via Gemini API (requires GEMINI_API_KEY)
+        {"value": "veo-3.1-generate-preview", "display_name": "Veo 3.1 | gemini", "provider": "gemini", "max_duration": 8},
+        {"value": "veo-3.1-fast-generate-preview", "display_name": "Veo 3.1 Fast | gemini", "provider": "gemini", "max_duration": 8},
+        {"value": "veo-2.0-generate-001", "display_name": "Veo 2 | gemini", "provider": "gemini", "max_duration": 8},
+        # Diffusers - damo-vilab/text-to-video-ms-1.7b (local)
+        {"value": "damo-vilab/text-to-video-ms-1.7b", "display_name": "ModelScope 1.7B (Local) | diffusers", "provider": "diffusers", "max_duration": 4},
+    ]
+    return jsonify({"models": video_models, "error": None})
 
 
 
@@ -6253,12 +6334,14 @@ def scan_gguf_models():
     """Scan for GGUF/GGML model files in specified or default directories."""
     directory = request.args.get('directory')
 
-    # Default directories to scan
+    # Default directories to scan (using platform-specific paths)
+    models_dir = get_models_dir()
     default_dirs = [
-        os.path.expanduser('~/.npcsh/models/gguf'),
-        os.path.expanduser('~/.npcsh/models'),
+        os.path.join(models_dir, 'gguf'),
+        models_dir,
         os.path.expanduser('~/models'),
-        os.path.expanduser('~/.cache/huggingface/hub'),
+        os.path.join(get_cache_dir(), 'huggingface/hub'),
+        os.path.expanduser('~/.cache/huggingface/hub'),  # Fallback
     ]
 
     # Add env var directory if set
@@ -6300,7 +6383,8 @@ def download_hf_model():
     """Download a GGUF model from HuggingFace."""
     data = request.json
     url = data.get('url', '')
-    target_dir = data.get('target_dir', '~/.npcsh/models/gguf')
+    default_target = os.path.join(get_models_dir(), 'gguf')
+    target_dir = data.get('target_dir', default_target)
 
     target_dir = os.path.expanduser(target_dir)
     os.makedirs(target_dir, exist_ok=True)
@@ -6466,7 +6550,8 @@ def download_hf_file():
     data = request.json
     repo_id = data.get('repo_id', '')
     filename = data.get('filename', '')
-    target_dir = data.get('target_dir', '~/.npcsh/models/gguf')
+    default_target = os.path.join(get_models_dir(), 'gguf')
+    target_dir = data.get('target_dir', default_target)
 
     if not repo_id or not filename:
         return jsonify({'error': 'repo_id and filename are required'}), 400

{npcpy-1.3.17.dist-info → npcpy-1.3.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: npcpy
-Version: 1.3.17
+Version: 1.3.19
 Summary: npcpy is the premier open-source library for integrating LLMs and Agents into python systems.
 Home-page: https://github.com/NPC-Worldwide/npcpy
 Author: Christopher Agostino

{npcpy-1.3.17.dist-info → npcpy-1.3.19.dist-info}/RECORD

@@ -2,12 +2,12 @@ npcpy/__init__.py,sha256=uJcJGjR1mWvE69GySNAufkgiRwJA28zdObDBWaxp0tY,505
 npcpy/build_funcs.py,sha256=vOz6pjV0zS-kYKo0ux-pn9AcppVaR8KIDi2ldOxb3RQ,7479
 npcpy/llm_funcs.py,sha256=M7GSSjqpcO2kxh7G2sGRBU34lmdW7Imd5KxYqc1PiO0,75114
 npcpy/main.py,sha256=RWoRIj6VQLxKdOKvdVyaq2kwG35oRpeXPvp1CAAoG-w,81
-npcpy/ml_funcs.py,sha256=UI7k7JR4XOH_VXR-xxLaO4r9Kyx_jBaEnp3TUIY7ZLQ,22657
-npcpy/npc_array.py,sha256=fVTxcMiXV-lvltmuwaRnTU9D3ikPq3-7k5wzp7MA5OY,40224
-npcpy/npc_compiler.py,sha256=W1umvhsbyCYoRYajPUKa642FcsX5Fcadh78n-Vzu2hM,120983
-npcpy/npc_sysenv.py,sha256=JjsvQaeMNeL3bDEbNrrldrRM8X9cMBI9j-W_1phb5TA,39995
+npcpy/ml_funcs.py,sha256=smgeOLnjxGWjmDngE-bcA2ozXX_IzY_7_pS9h2iocEg,24249
+npcpy/npc_array.py,sha256=5qjaA9KjmJ_Zk_VxLrCyVrj73aDXpm3iJf0ngq1yIJk,45721
+npcpy/npc_compiler.py,sha256=6-SYOddpi2jTJ1KtbMxtNGC7ksaiuuLd59bmc0eiOUA,121619
+npcpy/npc_sysenv.py,sha256=1E2zwMj7aPrtRJuJSowGkvNApi07Vue3FhXsipi1XDs,45251
 npcpy/npcs.py,sha256=eExuVsbTfrRobTRRptRpDm46jCLWUgbvy4_U7IUQo-c,744
-npcpy/serve.py,sha256=ACGhxr96H5VqGw1KWkp7WcW25Ok_aAwDTYhYg4jkHRQ,275626
+npcpy/serve.py,sha256=G8slQRslfOnGnSUD8GP1IOiqQhUWu6vF_azClvI7jLs,279284
 npcpy/tools.py,sha256=A5_oVmZkzGnI3BI-NmneuxeXQq-r29PbpAZP4nV4jrc,5303
 npcpy/data/__init__.py,sha256=1tcoChR-Hjn905JDLqaW9ElRmcISCTJdE7BGXPlym2Q,642
 npcpy/data/audio.py,sha256=o4auV8DQrAmZ4y84U3SofiwEuq5-ZBjGEZipQ9zPpGQ,22816
@@ -16,7 +16,7 @@ npcpy/data/image.py,sha256=UQcioNPDd5HYMLL_KStf45SuiIPXDcUY-dEFHwSWUeE,6564
 npcpy/data/load.py,sha256=rVe1xSHerIpo6MDaY5eIeqRSm0gssX5sHukNsUNVwJw,9228
 npcpy/data/text.py,sha256=jP0a1qZZaSJdK-LdZTn2Jjdxqmkd3efxDLEoxflJQeY,5010
 npcpy/data/video.py,sha256=H-V3mTu_ktD9u-QhYeo4aW3u9z0AtoAdRZmvRPEpE98,2887
-npcpy/data/web.py,sha256=pcjCLVAoqfw9enV5a7Dg1A_V7USG0302e6C7wUz2UgE,5235
+npcpy/data/web.py,sha256=cc4ikPZWMYsSz6itWI19ZkM_g5nQE3lPI-AJZEcoA04,5193
 npcpy/ft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 npcpy/ft/diff.py,sha256=0ScRR4AxXtVX2bgZ-Jr_dSwv3LAlU1JXDUq4F4n1Ea4,12839
 npcpy/ft/ge.py,sha256=0VzIiXq2wCzGcK1x0Wd-myJ3xRf-FNaPg0GkHEZegUM,3552
@@ -34,7 +34,7 @@ npcpy/gen/response.py,sha256=EYsIOvNOmn6dBs-4j3SyZNMvDf5N9lW-QxMbpjnF7Kw,57081
 npcpy/gen/video_gen.py,sha256=RFi3Zcq_Hn3HIcfoF3mijQ6G7RYFZaM_9pjPTh-8E64,3239
 npcpy/gen/world_gen.py,sha256=_8ytE7E3QVQ5qiX8DmOby-xd0d9zV20rRI6Wkpf-qcY,18922
 npcpy/memory/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-npcpy/memory/command_history.py,sha256=pjqcSBHXzdQTViSjHsBP2ohRSYnJ33h2bYARGcLvBfs,62253
+npcpy/memory/command_history.py,sha256=AXHrtP4GKObSVtiWNGZJ16rUyJD5FLor4EuaDEvsku0,62707
 npcpy/memory/kg_vis.py,sha256=TrQQCRh_E7Pyr-GPAHLSsayubAfGyf4HOEFrPB6W86Q,31280
 npcpy/memory/knowledge_graph.py,sha256=X3qqlDcuzGUjRgQWleQzafGKgNw8QRz2ar2gYuCvUq8,48600
 npcpy/memory/memory_processor.py,sha256=6PfVnSBA9ag5EhHJinXoODfEPTlDDoaT0PtCCuZO6HI,2598
@@ -53,8 +53,8 @@ npcpy/work/browser.py,sha256=p2PeaoZdAXipFuAgKCCB3aXXLE_p3yIRqC87KlZKZWc,679
 npcpy/work/desktop.py,sha256=F3I8mUtJp6LAkXodsh8hGZIncoads6c_2Utty-0EdDA,2986
 npcpy/work/plan.py,sha256=QyUwg8vElWiHuoS-xK4jXTxxHvkMD3VkaCEsCmrEPQk,8300
 npcpy/work/trigger.py,sha256=P1Y8u1wQRsS2WACims_2IdkBEar-iBQix-2TDWoW0OM,9948
-npcpy-1.3.17.dist-info/licenses/LICENSE,sha256=j0YPvce7Ng9e32zYOu0EmXjXeJ0Nwawd0RA3uSGGH4E,1070
-npcpy-1.3.17.dist-info/METADATA,sha256=oK8MXksfeMCM5acIJioPKaBjep1Yimzhn6rjl1Nbm44,37870
-npcpy-1.3.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-npcpy-1.3.17.dist-info/top_level.txt,sha256=g1pbSvrOOncB74Bg5-J0Olg4V0A5VzDw-Xz5YObq8BU,6
-npcpy-1.3.17.dist-info/RECORD,,
+npcpy-1.3.19.dist-info/licenses/LICENSE,sha256=j0YPvce7Ng9e32zYOu0EmXjXeJ0Nwawd0RA3uSGGH4E,1070
+npcpy-1.3.19.dist-info/METADATA,sha256=18bGYxznkKrwP59mZ_06rQfMFQBe_hdAjB_VbHyZBYY,37870
+npcpy-1.3.19.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92
+npcpy-1.3.19.dist-info/top_level.txt,sha256=g1pbSvrOOncB74Bg5-J0Olg4V0A5VzDw-Xz5YObq8BU,6
+npcpy-1.3.19.dist-info/RECORD,,

{npcpy-1.3.17.dist-info → npcpy-1.3.19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.1)
 Root-Is-Purelib: true
 Tag: py3-none-any