cortex-llm 1.0.0__py3-none-any.whl → 1.0.8__py3-none-any.whl

cortex/__init__.py

@@ -5,7 +5,7 @@ A high-performance terminal interface for running Hugging Face LLMs locally
 with exclusive GPU acceleration via Metal Performance Shaders (MPS) and MLX.
 """
 
-__version__ = "1.0.0"
+__version__ = "1.0.8"
 __author__ = "Cortex Development Team"
 __license__ = "MIT"
 
@@ -70,4 +70,4 @@ __all__ = [
     "ConversationManager",
     "initialize_cortex",
     "verify_system_requirements"
-]
+]

cortex/__main__.py

@@ -9,6 +9,13 @@ import warnings
 # This prevents the semaphore leak warning from transformers library
 os.environ['PYTHONWARNINGS'] = 'ignore::UserWarning:multiprocessing.resource_tracker'
 
+# Apply MLX compatibility shims before any MLX/MLX-LM imports.
+try:
+    from cortex.metal.mlx_compat import patch_mlx_lm_device_info
+    patch_mlx_lm_device_info()
+except Exception:
+    pass
+
 # Alternative: Monkey-patch the resource tracker before it's used
 try:
     from multiprocessing import resource_tracker
@@ -80,4 +87,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

cortex/config.py

@@ -74,7 +74,7 @@ class InferenceConfig(BaseModel):
     top_p: float = Field(default=0.95, ge=0.0, le=1.0)
     top_k: int = Field(default=40, ge=0)
     repetition_penalty: float = Field(default=1.1, ge=0.0, le=2.0)
-    max_tokens: int = Field(default=2048, ge=1)
+    max_tokens: int = Field(default=4096, ge=1)
     stream_output: bool = True
     seed: int = Field(default=-1)
 

cortex/inference_engine.py

@@ -25,6 +25,8 @@ try:
 except ImportError:
     mlx_generate = None
     mlx_stream_generate = None
+from cortex.metal.mlx_compat import patch_mlx_lm_device_info
+patch_mlx_lm_device_info()
 
 from cortex.config import Config
 from cortex.model_manager import ModelManager, ModelFormat
@@ -136,7 +138,7 @@ class InferenceEngine:
                 use_fp16=True,
                 use_channels_last=True,
                 optimize_memory=True,
-                max_batch_size=self.config.performance.batch_size
+                max_batch_size=self.config.performance.max_batch_size
             )
             self.mps_optimizer = MPSOptimizer(mps_config)
         
@@ -151,7 +153,7 @@ class InferenceEngine:
                 fuse_operations=True,
                 lazy_evaluation=True,
                 rotating_kv_cache=True,
-                kv_cache_size=self.config.model.context_length if hasattr(self.config.model, 'context_length') else 4096,
+                kv_cache_size=self.config.performance.context_length,
                 quantization_bits=4
             )
             self.mlx_accelerator = MLXAccelerator(mlx_config)
@@ -202,6 +204,9 @@ class InferenceEngine:
                     yield from self._generate_pytorch(model, tokenizer, request)
                 elif model_info.format == ModelFormat.SAFETENSORS:
                     yield from self._generate_safetensors(model, tokenizer, request)
+                elif model_info.format == ModelFormat.QUANTIZED:
+                    # Quantized models are loaded as PyTorch-compatible modules
+                    yield from self._generate_pytorch(model, tokenizer, request)
                 elif model_info.format == ModelFormat.GGUF:
                     yield from self._generate_gguf(model, tokenizer, request)
                 else:
@@ -399,7 +404,18 @@ class InferenceEngine:
         last_metrics_update = time.time()
         
         try:
-            device = torch.device("mps")
+            # Use the model's device when available (quantized models may be CPU-only on macOS)
+            device = None
+            try:
+                first_param = next(model.parameters())
+                device = first_param.device
+            except Exception:
+                device = None
+
+            if device is None or str(device) == "meta":
+                device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
+            elif device.type == "mps" and not torch.backends.mps.is_available():
+                device = torch.device("cpu")
             
             inputs = tokenizer(request.prompt, return_tensors="pt").to(device)
             
@@ -724,4 +740,4 @@ class InferenceEngine:
                 pass
             
         except Exception as e:
-            print(f"Warning: GPU warmup failed: {e}")
+            print(f"Warning: GPU warmup failed: {e}")

cortex/metal/mlx_accelerator.py

@@ -21,6 +21,8 @@ except ImportError:
     # Fallback if mlx_lm is not available
     generate = None
     stream_generate = None
+from cortex.metal.mlx_compat import patch_mlx_lm_device_info
+patch_mlx_lm_device_info()
 
 @dataclass
 class MLXConfig:
@@ -675,4 +677,4 @@ class MLXAccelerator:
         }
         
         logger.debug(f"Benchmark results: {result}")
-        return result
+        return result

cortex/metal/mlx_compat.py

@@ -0,0 +1,105 @@
+"""Compatibility helpers for MLX / mlx_lm API changes."""
+
+from __future__ import annotations
+
+import contextlib
+from typing import Optional, List, Any
+
+
+def _get_device_info(mx) -> dict:
+    try:
+        return mx.device_info()
+    except Exception:
+        return {}
+
+def patch_mlx_device_info() -> None:
+    """Redirect deprecated mx.metal.device_info to mx.device_info when possible."""
+    try:
+        import mlx.core as mx
+    except Exception:
+        return
+
+    if hasattr(mx, "device_info") and hasattr(mx, "metal") and hasattr(mx.metal, "device_info"):
+        try:
+            mx.metal.device_info = mx.device_info  # type: ignore[attr-defined]
+        except Exception:
+            pass
+
+
+def patch_mlx_lm_device_info() -> None:
+    """Patch mlx_lm call sites to use mx.device_info() instead of mx.metal.device_info()."""
+    try:
+        import mlx.core as mx
+        from mlx.utils import tree_reduce
+    except Exception:
+        return
+
+    if not hasattr(mx, "device_info"):
+        return
+
+    patch_mlx_device_info()
+
+    try:
+        import mlx_lm.generate as mlx_generate
+    except Exception:
+        mlx_generate = None
+
+    try:
+        import mlx_lm.server as mlx_server
+    except Exception:
+        mlx_server = None
+
+    if mlx_generate is not None and getattr(mlx_generate, "__cortex_patched__", False) is False:
+        @contextlib.contextmanager
+        def wired_limit(model: Any, streams: Optional[List[Any]] = None):
+            if not mx.metal.is_available():
+                try:
+                    yield
+                finally:
+                    pass
+                return
+
+            model_bytes = tree_reduce(
+                lambda acc, x: acc + x.nbytes if isinstance(x, mx.array) else acc, model, 0
+            )
+            info = _get_device_info(mx)
+            max_rec_size = info.get("max_recommended_working_set_size")
+
+            if max_rec_size and model_bytes > 0.9 * max_rec_size:
+                model_mb = model_bytes // 2**20
+                max_rec_mb = max_rec_size // 2**20
+                print(
+                    f"[WARNING] Generating with a model that requires {model_mb} MB "
+                    f"which is close to the maximum recommended size of {max_rec_mb} "
+                    "MB. This can be slow. See the documentation for possible work-arounds: "
+                    "https://github.com/ml-explore/mlx-lm/tree/main#large-models"
+                )
+
+            old_limit = None
+            if max_rec_size:
+                old_limit = mx.set_wired_limit(max_rec_size)
+
+            try:
+                yield
+            finally:
+                if streams is not None:
+                    for s in streams:
+                        mx.synchronize(s)
+                else:
+                    mx.synchronize()
+                if old_limit is not None:
+                    mx.set_wired_limit(old_limit)
+
+        mlx_generate.wired_limit = wired_limit
+        mlx_generate.__cortex_patched__ = True
+
+    if mlx_server is not None and getattr(mlx_server, "__cortex_patched__", False) is False:
+        def get_system_fingerprint():
+            gpu_arch = ""
+            if mx.metal.is_available():
+                info = _get_device_info(mx)
+                gpu_arch = info.get("architecture", "") if isinstance(info, dict) else ""
+            return f"{mlx_server.__version__}-{mx.__version__}-{mlx_server.platform.platform()}-{gpu_arch}"
+
+        mlx_server.get_system_fingerprint = get_system_fingerprint
+        mlx_server.__cortex_patched__ = True

cortex/metal/mlx_converter.py

@@ -66,9 +66,22 @@ class MLXConverter:
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         self.conversion_cache = self.cache_dir / "conversion_cache.json"
         self._load_conversion_cache()
+        self._warned_mlx_lm_compat = False
         
         logger.info(f"MLX Converter initialized with cache dir: {self.cache_dir}")
         logger.info(f"MLX LM available: {mlx_utils is not None and load is not None}")
+
+    def _warn_mlx_lm_compat(self, missing: str) -> None:
+        """Warn once when mlx-lm is missing newer helper APIs."""
+        if self._warned_mlx_lm_compat:
+            return
+        self._warned_mlx_lm_compat = True
+        message = (
+            f"[WARN] mlx-lm is missing '{missing}'. Using compatibility fallback. "
+            "For best support, upgrade mlx-lm to a newer version."
+        )
+        logger.warning(message)
+        print(message)
     
     def _load_conversion_cache(self) -> None:
         """Load conversion cache metadata."""
@@ -206,6 +219,83 @@ class MLXConverter:
         
         return download_dir
 
+    def _mlx_get_model_path(self, source_path: Path) -> Tuple[Path, Optional[str]]:
+        """Resolve model path with MLX LM compatibility fallbacks."""
+        if mlx_utils is not None and hasattr(mlx_utils, "get_model_path"):
+            return mlx_utils.get_model_path(str(source_path))
+        self._warn_mlx_lm_compat("get_model_path")
+
+        # Fallback: local path or direct HF download.
+        model_path = Path(source_path)
+        if model_path.exists():
+            hf_repo = None
+            try:
+                from huggingface_hub import ModelCard
+
+                card_path = model_path / "README.md"
+                if card_path.is_file():
+                    card = ModelCard.load(card_path)
+                    hf_repo = getattr(card.data, "base_model", None)
+            except Exception:
+                hf_repo = None
+            return model_path, hf_repo
+
+        try:
+            model_path = Path(
+                snapshot_download(
+                    str(source_path),
+                    allow_patterns=[
+                        "*.json",
+                        "model*.safetensors",
+                        "*.py",
+                        "tokenizer.model",
+                        "*.tiktoken",
+                        "tiktoken.model",
+                        "*.txt",
+                        "*.jsonl",
+                        "*.jinja",
+                    ],
+                )
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to download model from Hugging Face: {e}") from e
+
+        return model_path, str(source_path)
+
+    def _mlx_fetch_from_hub(
+        self,
+        model_path: Path,
+        trust_remote_code: bool = False
+    ) -> Tuple[Any, Dict[str, Any], Any]:
+        """Fetch model/config/tokenizer with MLX LM compatibility fallbacks."""
+        if mlx_utils is not None and hasattr(mlx_utils, "fetch_from_hub"):
+            return mlx_utils.fetch_from_hub(
+                model_path,
+                lazy=True,
+                trust_remote_code=trust_remote_code
+            )
+        self._warn_mlx_lm_compat("fetch_from_hub")
+
+        if mlx_utils is not None and hasattr(mlx_utils, "load_model") and hasattr(mlx_utils, "load_tokenizer"):
+            model, model_config = mlx_utils.load_model(model_path, lazy=True)
+            try:
+                tokenizer = mlx_utils.load_tokenizer(
+                    model_path,
+                    eos_token_ids=model_config.get("eos_token_id", None),
+                    tokenizer_config_extra={"trust_remote_code": trust_remote_code},
+                )
+            except TypeError:
+                tokenizer = mlx_utils.load_tokenizer(
+                    model_path,
+                    eos_token_ids=model_config.get("eos_token_id", None),
+                )
+            return model, model_config, tokenizer
+
+        raise RuntimeError(
+            "mlx_lm.utils is missing required helpers (fetch_from_hub/load_model). "
+            "Upgrade mlx-lm to a newer version."
+        )
+
     def _requires_sentencepiece(self, model_path: Path) -> bool:
         """Return True if the model likely needs SentencePiece."""
         # If a fast tokenizer is present, SentencePiece should not be required.
@@ -379,10 +469,17 @@ class MLXConverter:
             # Build quantization configuration
             quantize_config = self._build_quantization_config(config)
 
-            model_path, hf_repo = mlx_utils.get_model_path(str(source_path))
-            model, model_config, tokenizer = mlx_utils.fetch_from_hub(
-                model_path, lazy=True, trust_remote_code=False
-            )
+            try:
+                model_path, hf_repo = self._mlx_get_model_path(Path(source_path))
+            except Exception as e:
+                return False, f"Model path resolution failed: {e}", None
+
+            try:
+                model, model_config, tokenizer = self._mlx_fetch_from_hub(
+                    model_path, trust_remote_code=False
+                )
+            except Exception as e:
+                return False, f"Model fetch failed: {e}", None
 
             dtype = model_config.get("torch_dtype", None)
             if dtype in ["float16", "bfloat16", "float32"]:
@@ -398,6 +495,8 @@ class MLXConverter:
                 model.update(tree_map_with_path(set_dtype, model.parameters()))
 
             if config.quantization != QuantizationRecipe.NONE:
+                if mlx_utils is None or not hasattr(mlx_utils, "quantize_model"):
+                    return False, "MLX LM quantize_model not available; upgrade mlx-lm.", None
                 quant_predicate = None
                 if quantize_config and "quant_predicate" in quantize_config:
                     quant_predicate = quantize_config["quant_predicate"]
@@ -411,6 +510,8 @@ class MLXConverter:
                 )
 
             normalized_hf_repo = self._normalize_hf_repo(hf_repo)
+            if mlx_utils is None or not hasattr(mlx_utils, "save"):
+                return False, "MLX LM save() not available; upgrade mlx-lm.", None
             mlx_utils.save(output_path, model_path, model, tokenizer, model_config, hf_repo=normalized_hf_repo)
             logger.info("MLX conversion completed")
             

cortex/model_manager.py

@@ -133,7 +133,8 @@ class ModelManager:
         self.quantizer = DynamicQuantizer(QuantizationConfig(
             mode=QuantizationMode.DYNAMIC,
             per_channel=True,
-            cache_quantized=True
+            cache_quantized=True,
+            cache_dir=self.config.model.quantization_cache
         ))
         
         # Initialize MLX converter for native conversion
@@ -201,6 +202,39 @@ class ModelManager:
         level = getattr(self.config.gpu, "gpu_optimization_level", "maximum")
         level = str(level).lower().strip()
         return level in {"maximum", "max", "speed", "fast", "performance"}
+
+    def _get_default_quant_recipe(self) -> Optional[QuantizationRecipe]:
+        """Map configured default_quantization to an MLX quantization recipe."""
+        raw = getattr(self.config.model, "default_quantization", "") or ""
+        value = str(raw).strip().lower()
+        if not value or value == "auto":
+            return None
+
+        mapping = {
+            "q4_k_m": QuantizationRecipe.SPEED_4BIT,
+            "q5_k_m": QuantizationRecipe.BALANCED_5BIT,
+            "q6_k": QuantizationRecipe.QUALITY_8BIT,  # closest available MLX recipe
+            "q8_0": QuantizationRecipe.QUALITY_8BIT,
+            "4bit": QuantizationRecipe.SPEED_4BIT,
+            "5bit": QuantizationRecipe.BALANCED_5BIT,
+            "8bit": QuantizationRecipe.QUALITY_8BIT,
+            "mixed": QuantizationRecipe.MIXED_PRECISION,
+            "none": QuantizationRecipe.NONE,
+        }
+
+        recipe = mapping.get(value)
+        if recipe is None:
+            logger.warning("Unknown default_quantization value: %s", raw)
+            return None
+
+        supported = getattr(self.config.model, "supported_quantizations", None)
+        if supported:
+            supported_norm = {str(s).strip().lower() for s in supported}
+            if value.startswith("q") and value not in supported_norm:
+                logger.warning("default_quantization '%s' not in supported_quantizations", raw)
+                return None
+
+        return recipe
     
     def load_model(
         self,
@@ -374,6 +408,10 @@ class ModelManager:
                         except Exception as e:
                             logger.warning(f"Could not estimate model parameters: {e}, defaulting to 4-bit")
                             quant_recipe = QuantizationRecipe.SPEED_4BIT  # Fallback
+
+                    default_recipe = self._get_default_quant_recipe()
+                    if default_recipe is not None:
+                        quant_recipe = default_recipe
                     
                     if quantization:
                         quant_map = {
@@ -452,6 +490,10 @@ class ModelManager:
                 else:
                     quant_recipe = QuantizationRecipe.SPEED_4BIT  # Default for larger models
                 
+                default_recipe = self._get_default_quant_recipe()
+                if default_recipe is not None:
+                    quant_recipe = default_recipe
+                
                 if quantization:
                     quant_map = {
                         "4bit": QuantizationRecipe.SPEED_4BIT,
@@ -563,6 +605,8 @@ class ModelManager:
         )
         
         if not can_load and can_apply_quantization:
+            if not getattr(self.config.model, "auto_quantize", True):
+                return False, f"GPU incompatible: {message} (auto_quantize disabled)"
             # Check if quantization would help
             gpu_status = self.gpu_validator.get_gpu_memory_status()
             available_gb = gpu_status['available_gb']

cortex/quantization/dynamic_quantizer.py

@@ -3,7 +3,7 @@
 import torch
 import torch.nn as nn
 from typing import Dict, Any, Optional, Tuple, Union
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 import gc
 from pathlib import Path
@@ -40,6 +40,7 @@ class QuantizationConfig:
     cache_quantized: bool = True  # Cache quantized models to disk
     compress_cache: bool = False  # Compress cached models (slower but smaller)
     validate_quantization: bool = True  # Validate quantized models work correctly
+    cache_dir: Path = field(default_factory=lambda: Path.home() / ".cortex" / "quantized_models")
     
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary for serialization."""
@@ -118,6 +119,8 @@ class DynamicQuantizer:
     def __init__(self, config: Optional[QuantizationConfig] = None):
         """Initialize quantizer with configuration."""
         self.config = config or QuantizationConfig()
+        self.config.cache_dir = Path(self.config.cache_dir).expanduser()
+        self.config.cache_dir.mkdir(parents=True, exist_ok=True)
         self.device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
         self._quantization_cache: Dict[str, Dict[str, Any]] = {}
     
@@ -681,10 +684,10 @@ class DynamicQuantizer:
         
         # Generate cache key including model metadata
         cache_key = hashlib.md5(
-            f"{model_path}_{model_mtime}_{model_size}_{json.dumps(quantization_info)}".encode()
+            f"{model_path}_{model_mtime}_{model_size}_{json.dumps(self.config.to_dict())}".encode()
         ).hexdigest()
         
-        cache_dir = Path.home() / ".cortex" / "quantized_cache"
+        cache_dir = self.config.cache_dir
         cache_dir.mkdir(parents=True, exist_ok=True)
         
         cache_path = cache_dir / f"{cache_key}.pt"
@@ -723,7 +726,7 @@ class DynamicQuantizer:
             f"{model_path}_{model_mtime}_{model_size}_{json.dumps(config.to_dict())}".encode()
         ).hexdigest()
         
-        cache_path = Path.home() / ".cortex" / "quantized_cache" / f"{cache_key}.pt"
+        cache_path = Path(self.config.cache_dir) / f"{cache_key}.pt"
         
         if cache_path.exists():
             try:
@@ -733,4 +736,4 @@ class DynamicQuantizer:
                 # Cache corrupted, will re-quantize
                 cache_path.unlink()
         
-        return None
+        return None

cortex/ui/cli.py

@@ -30,7 +30,7 @@ from cortex.conversation_manager import ConversationManager, MessageRole
 from cortex.model_downloader import ModelDownloader
 from cortex.template_registry import TemplateRegistry
 from cortex.fine_tuning import FineTuneWizard
-from cortex.ui.markdown_render import ThinkMarkdown, PrefixedRenderable
+from cortex.ui.markdown_render import ThinkMarkdown, PrefixedRenderable, render_plain_with_think
 
 
 class CortexCLI:
@@ -1135,15 +1135,15 @@ class CortexCLI:
                 logger.debug(f"Could not get stop sequences: {e}")
         
         # Create generation request with formatted prompt
-        # Use lower temperature for more focused responses
         request = GenerationRequest(
             prompt=formatted_prompt,
             max_tokens=self.config.inference.max_tokens,
-            temperature=0.3,  # Lower temperature for less randomness
-            top_p=0.9,  # Slightly lower top_p
+            temperature=self.config.inference.temperature,
+            top_p=self.config.inference.top_p,
             top_k=self.config.inference.top_k,
             repetition_penalty=self.config.inference.repetition_penalty,
-            stream=True,
+            stream=self.config.inference.stream_output,
+            seed=self.config.inference.seed if self.config.inference.seed >= 0 else None,
             stop_sequences=stop_sequences
         )
         
@@ -1167,50 +1167,65 @@ class CortexCLI:
             prefix_style = Style(color="cyan")
 
             def build_renderable(text: str):
-                markdown = ThinkMarkdown(text, code_theme="monokai", use_line_numbers=False)
-                return PrefixedRenderable(markdown, prefix="⏺ ", prefix_style=prefix_style, indent="  ")
+                if getattr(self.config.ui, "markdown_rendering", True):
+                    markdown = ThinkMarkdown(
+                        text,
+                        code_theme="monokai",
+                        use_line_numbers=False,
+                        syntax_highlighting=getattr(self.config.ui, "syntax_highlighting", True),
+                    )
+                    renderable = markdown
+                else:
+                    renderable = render_plain_with_think(text)
 
-            with Live(
-                build_renderable(""),
-                console=self.console,
-                refresh_per_second=20,
-                transient=False,
-            ) as live:
-                for token in self.inference_engine.generate(request):
-                    if first_token_time is None:
-                        first_token_time = time.time()
+                return PrefixedRenderable(renderable, prefix="⏺ ", prefix_style=prefix_style, indent="  ")
 
-                    generated_text += token
-                    token_count += 1
+            original_console_width = self.console._width
+            target_width = max(40, int(self.get_terminal_width() * 0.75))
+            self.console.width = target_width
+            try:
+                with Live(
+                    build_renderable(""),
+                    console=self.console,
+                    auto_refresh=False,
+                    refresh_per_second=20,
+                    transient=False,
+                    vertical_overflow="visible",
+                ) as live:
+                    for token in self.inference_engine.generate(request):
+                        if first_token_time is None:
+                            first_token_time = time.time()
 
-                    display_token = token
-                    if uses_reasoning_template and template_profile and template_profile.supports_streaming():
-                        display_token, should_display = template_profile.process_streaming_response(
-                            token, accumulated_response
-                        )
-                        accumulated_response += token
-                        if not should_display:
-                            display_token = ""
+                        generated_text += token
+                        token_count += 1
+
+                        display_token = token
+                        if uses_reasoning_template and template_profile and template_profile.supports_streaming():
+                            display_token, should_display = template_profile.process_streaming_response(
+                                token, accumulated_response
+                            )
+                            accumulated_response += token
+                            if not should_display:
+                                display_token = ""
 
-                    if display_token:
-                        display_text += display_token
+                        if display_token:
+                            display_text += display_token
 
-                    now = time.time()
-                    if display_token and ("\n" in display_token or now - last_render_time >= render_interval):
-                        live.update(build_renderable(display_text))
-                        last_render_time = now
+                        now = time.time()
+                        if display_token and ("\n" in display_token or now - last_render_time >= render_interval):
+                            live.update(build_renderable(display_text), refresh=True)
+                            last_render_time = now
 
-                if uses_reasoning_template and template_profile:
-                    final_text = template_profile.process_response(generated_text)
-                    generated_text = final_text
-                    if not template_profile.config.show_reasoning:
-                        display_text = final_text
+                    if uses_reasoning_template and template_profile:
+                        final_text = template_profile.process_response(generated_text)
+                        generated_text = final_text
+                        if not template_profile.config.show_reasoning:
+                            display_text = final_text
 
-                live.update(build_renderable(display_text))
+                    live.update(build_renderable(display_text), refresh=True)
+            finally:
+                self.console._width = original_console_width
 
-            # Add blank line for spacing between response and metrics
-            print()
-            
             # Display final metrics in a clean, professional way
             elapsed = time.time() - start_time
             if token_count > 0 and elapsed > 0:
@@ -1238,6 +1253,9 @@ class CortexCLI:
                 metrics_line = " · ".join(metrics_parts)
                 print(f"  \033[2m{metrics_line}\033[0m")
             
+            if token_count >= request.max_tokens:
+                print(f"  \033[2m(output truncated at max_tokens={request.max_tokens}; increase in config.yaml)\033[0m")
+            
             # Add assistant message to conversation history
             self.conversation_manager.add_message(MessageRole.ASSISTANT, generated_text)
             
@@ -1405,8 +1423,8 @@ class CortexCLI:
         else:
             print()  # Empty line if no model loaded
         
-        # Move cursor to input position inside the box
-        sys.stdout.write("\033[3A")  # Move up 3 lines to the input line
+        # Move cursor to input position inside the box (center of 3 interior lines)
+        sys.stdout.write("\033[4A")  # Move up 4 lines to the input line
         sys.stdout.write(f"\r{DIM}│{RESET} > ")  # Position at prompt
         sys.stdout.flush()
         

cortex/ui/markdown_render.py

@@ -3,10 +3,12 @@
 from typing import List
 
 from rich.console import Console
+from rich.cells import cell_len
 from rich.markdown import Markdown
 from rich.segment import Segment
 from rich.style import Style
 from rich.syntax import Syntax
+from rich.text import Text
 
 THINK_START_MARKER = "[[[THINK_START]]]"
 THINK_END_MARKER = "[[[THINK_END]]]"
@@ -45,6 +47,14 @@ class CodeBlockWithLineNumbers(Markdown.elements["fence"]):
         yield syntax
 
 
+class CodeBlockPlain(Markdown.elements["fence"]):
+    """Markdown code block rendered as plain text (no syntax highlighting)."""
+
+    def __rich_console__(self, console: Console, options):
+        code = str(self.text).rstrip()
+        yield Text(code)
+
+
 class MarkdownWithLineNumbers(Markdown):
     """Markdown renderer that keeps line numbers for fenced code blocks."""
 
@@ -55,6 +65,26 @@ class MarkdownWithLineNumbers(Markdown):
     })
 
 
+class MarkdownPlainCode(Markdown):
+    """Markdown renderer that disables syntax highlighting for code blocks."""
+
+    elements = Markdown.elements.copy()
+    elements.update({
+        "fence": CodeBlockPlain,
+        "code_block": CodeBlockPlain,
+    })
+
+
+class MarkdownPlainCodeWithLineNumbers(Markdown):
+    """Markdown renderer with plain code blocks and line numbers."""
+
+    elements = MarkdownWithLineNumbers.elements.copy()
+    elements.update({
+        "fence": CodeBlockPlain,
+        "code_block": CodeBlockPlain,
+    })
+
+
 class ThinkMarkdown:
     """Markdown renderer that dims content inside <think> tags."""
 
@@ -63,10 +93,15 @@ class ThinkMarkdown:
         markup: str,
         code_theme: str = "monokai",
         use_line_numbers: bool = False,
+        syntax_highlighting: bool = True,
     ) -> None:
         marked = _mark_think_sections(markup)
-        markdown_cls = MarkdownWithLineNumbers if use_line_numbers else Markdown
-        self._markdown = markdown_cls(marked, code_theme=code_theme)
+        if syntax_highlighting:
+            markdown_cls = MarkdownWithLineNumbers if use_line_numbers else Markdown
+            self._markdown = markdown_cls(marked, code_theme=code_theme)
+        else:
+            markdown_cls = MarkdownPlainCodeWithLineNumbers if use_line_numbers else MarkdownPlainCode
+            self._markdown = markdown_cls(marked)
 
     def __rich_console__(self, console: Console, options):
         segments = console.render(self._markdown, options)
@@ -162,9 +197,15 @@ class PrefixedRenderable:
         self.indent = indent if indent is not None else " " * len(prefix)
 
     def __rich_console__(self, console: Console, options):
+        prefix_width = cell_len(self.prefix)
+        indent_width = cell_len(self.indent) if self.indent is not None else prefix_width
+        offset = max(prefix_width, indent_width)
+        inner_width = max(1, options.max_width - offset)
+        inner_options = options.update_width(inner_width)
+
         yield Segment(self.prefix, self.prefix_style)
 
-        for segment in console.render(self.renderable, options):
+        for segment in console.render(self.renderable, inner_options):
             if segment.control:
                 yield segment
                 continue
@@ -183,3 +224,27 @@ class PrefixedRenderable:
                 if index < len(parts) - 1:
                     yield Segment("\n", style)
                     yield Segment(self.indent, None)
+
+
+def render_plain_with_think(text: str) -> Text:
+    """Render plain text while dimming content inside <think> tags."""
+    output = Text()
+    dim_style = Style(dim=True)
+    idx = 0
+    in_think = False
+
+    while idx < len(text):
+        if text.startswith("<think>", idx):
+            in_think = True
+            idx += len("<think>")
+            continue
+        if text.startswith("</think>", idx):
+            in_think = False
+            idx += len("</think>")
+            continue
+
+        char = text[idx]
+        output.append(char, dim_style if in_think else None)
+        idx += 1
+
+    return output

cortex_llm-1.0.8.dist-info/METADATA

@@ -0,0 +1,169 @@
+Metadata-Version: 2.4
+Name: cortex-llm
+Version: 1.0.8
+Summary: GPU-Accelerated LLM Terminal for Apple Silicon
+Home-page: https://github.com/faisalmumtaz/Cortex
+Author: Cortex Development Team
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/faisalmumtaz/Cortex
+Project-URL: Bug Tracker, https://github.com/faisalmumtaz/Cortex/issues
+Project-URL: Documentation, https://github.com/faisalmumtaz/Cortex/wiki
+Keywords: llm,gpu,metal,mps,apple-silicon,ai,machine-learning,terminal,mlx,pytorch
+Platform: darwin
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: MacOS
+Classifier: Environment :: Console
+Classifier: Environment :: GPU
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.1.0
+Requires-Dist: mlx>=0.30.4
+Requires-Dist: mlx-lm>=0.30.5
+Requires-Dist: transformers>=4.36.0
+Requires-Dist: safetensors>=0.4.0
+Requires-Dist: huggingface-hub>=0.19.0
+Requires-Dist: accelerate>=0.25.0
+Requires-Dist: llama-cpp-python>=0.2.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: pydantic>=2.5.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: psutil>=5.9.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: packaging>=23.0
+Requires-Dist: requests>=2.31.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.8.0; extra == "dev"
+Provides-Extra: optional
+Requires-Dist: sentencepiece>=0.1.99; extra == "optional"
+Requires-Dist: auto-gptq>=0.7.0; extra == "optional"
+Requires-Dist: autoawq>=0.2.0; extra == "optional"
+Requires-Dist: bitsandbytes>=0.41.0; extra == "optional"
+Requires-Dist: optimum>=1.16.0; extra == "optional"
+Requires-Dist: torchvision>=0.16.0; extra == "optional"
+Requires-Dist: torchaudio>=2.1.0; extra == "optional"
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: platform
+Dynamic: requires-python
+
+# Cortex
+
+GPU-accelerated local LLMs on Apple Silicon, built for the terminal.
+
+Cortex is a fast, native CLI for running and fine-tuning LLMs on Apple Silicon using MLX and Metal. It automatically detects chat templates, supports multiple model formats, and keeps your workflow inside the terminal.
+
+## Highlights
+
+- Apple Silicon GPU acceleration via MLX (primary) and PyTorch MPS
+- Multi-format model support: MLX, GGUF, SafeTensors, PyTorch, GPTQ, AWQ
+- Built-in LoRA fine-tuning wizard
+- Chat template auto-detection (ChatML, Llama, Alpaca, Gemma, Reasoning)
+- Conversation history with autosave and export
+
+## Quick Start
+
+```bash
+pipx install cortex-llm
+cortex
+```
+
+Inside Cortex:
+
+- `/download` to fetch a model from HuggingFace
+- `/model` to load or manage models
+- `/status` to confirm GPU acceleration and current settings
+
+## Installation
+
+### Option A: pipx (recommended)
+
+```bash
+pipx install cortex-llm
+```
+
+### Option B: from source
+
+```bash
+git clone https://github.com/faisalmumtaz/Cortex.git
+cd Cortex
+./install.sh
+```
+
+The installer checks Apple Silicon compatibility, creates a venv, installs dependencies from `pyproject.toml`, and sets up the `cortex` command.
+
+## Requirements
+
+- Apple Silicon Mac (M1/M2/M3/M4)
+- macOS 13.3+
+- Python 3.11+
+- 16GB+ unified memory (24GB+ recommended for larger models)
+- Xcode Command Line Tools
+
+## Model Support
+
+Cortex supports:
+
+- **MLX** (recommended)
+- **GGUF** (llama.cpp + Metal)
+- **SafeTensors**
+- **PyTorch** (Transformers + MPS)
+- **GPTQ** / **AWQ** quantized models
+
+## Advanced Features
+
+- **Dynamic quantization fallback** for PyTorch/SafeTensors models that do not fit GPU memory (INT8 preferred, INT4 fallback)
+  - `docs/dynamic-quantization.md`
+- **MLX conversion with quantization recipes** (4/5/8-bit, mixed precision) for speed vs quality control
+  - `docs/mlx-acceleration.md`
+- **LoRA fine-tuning wizard** for local adapters (`/finetune`)
+  - `docs/fine-tuning.md`
+- **Template registry and auto-detection** for chat formatting (ChatML, Llama, Alpaca, Gemma, Reasoning)
+  - `docs/template-registry.md`
+- **Inference engine details** and backend behavior
+  - `docs/inference-engine.md`
+
+## Configuration
+
+Cortex reads `config.yaml` from the current working directory. For tuning GPU memory limits, quantization defaults, and inference parameters, see:
+
+- `docs/configuration.md`
+
+## Documentation
+
+Start here:
+
+- `docs/installation.md`
+- `docs/cli.md`
+- `docs/model-management.md`
+- `docs/troubleshooting.md`
+
+Advanced topics:
+
+- `docs/mlx-acceleration.md`
+- `docs/inference-engine.md`
+- `docs/dynamic-quantization.md`
+- `docs/template-registry.md`
+- `docs/fine-tuning.md`
+- `docs/development.md`
+
+## Contributing
+
+Contributions are welcome. See `docs/development.md` for setup and workflow.
+
+## License
+
+MIT License. See `LICENSE`.
+
+---
+
+Note: Cortex requires Apple Silicon. Intel Macs are not supported.

{cortex_llm-1.0.0.dist-info → cortex_llm-1.0.8.dist-info}/RECORD

@@ -1,11 +1,11 @@
-cortex/__init__.py,sha256=cAZXKEqjOtK2P-kij00tDHwtM3asYNCW1386R4WHyTQ,2201
-cortex/__main__.py,sha256=9k1Cvz8ncltQzbrJlxz8JDv-jHAOLCZoCaKSlJ4YFHE,2664
-cortex/config.py,sha256=txmpJXy3kUEKULZyu1OWb_jkNQRHZClm5ovZfCTX_Zc,13444
+cortex/__init__.py,sha256=lj0fAHYDEHTqAY8WK0eDX-_prqihmKLlsH_ELz6DxMs,2202
+cortex/__main__.py,sha256=I7Njt7BjGoHtPhftDoA44OyOYbwWNNaPwP_qlJSn0J4,2857
+cortex/config.py,sha256=IQnMaXznTflTSvr91aybtPMnNW088r-BYeVMhxny63w,13444
 cortex/conversation_manager.py,sha256=aSTdGjVttsMKIiRPzztP0tOXlqZBkWtgZDNCZGyaR-c,17177
 cortex/gpu_validator.py,sha256=un6vMQ78MWMnKWIz8n-92v9Fb4g_YXqU_E1pUPinncY,16582
-cortex/inference_engine.py,sha256=WUi5YS_sTQFayc-UJbyixHEzZlMN0-ATNdoPNEhHbbo,28543
+cortex/inference_engine.py,sha256=bklCjmiMn3psFp14EZxRzePEuA33NCHJ1bQdsbvMlfg,29343
 cortex/model_downloader.py,sha256=VuPhvxq_66qKjsPjEWcLW-VmUHzOHik6LBMiGDk-cX8,4977
-cortex/model_manager.py,sha256=Blk-JA_kajJcDp-h2A4tplECijHPw8LZ8c_fbq0FGFg,100670
+cortex/model_manager.py,sha256=Ra21TjhtFS-7_hRzDMh9m0BUazIGWoKr7Gye3GiVRJM,102671
 cortex/fine_tuning/__init__.py,sha256=IXKQqNqN1C3mha3na35i7KI-hMnsqqrmUgV4NrPKHy0,269
 cortex/fine_tuning/dataset.py,sha256=hIz_dfFSaJoiFzWZ6vwlwqjpTfdsnFNIEmwhhTD2d9k,15414
 cortex/fine_tuning/mlx_lora_trainer.py,sha256=idNzKtVG8pObwsnSrP0N1rU1EanhrIRvHiNL1asdzr8,22438
@@ -14,13 +14,14 @@ cortex/fine_tuning/wizard.py,sha256=eIRUM3zTqKKATJEbQrBsaOfFfRWfY9BV5FkSAzT82QM,
 cortex/metal/__init__.py,sha256=Ycs81qVOsaYV4UJocCFGW3rPPBySMPy7eOHKzfc4Q7o,8780
 cortex/metal/gpu_validator.py,sha256=1YHKJXqicXvTwKIdSj34n1DgKoluy9yho6S1jWt1UAs,5818
 cortex/metal/memory_pool.py,sha256=g5PFQAiouQe4TyX-SVi-Di1MLysb3YBF77uR4nAEomo,34698
-cortex/metal/mlx_accelerator.py,sha256=HknYFHwFOPzAtdrV5OqxBvENX3qwNoiKC6H8lXGmJPg,25637
-cortex/metal/mlx_converter.py,sha256=lMmIh6PqLwraWInZDLJjktBtumLHD9TYRnhiKCpilrc,26722
+cortex/metal/mlx_accelerator.py,sha256=f3tfHAaRQqc5KteXNkf7n610SxOLo2hoGj5_GgqEL2Y,25726
+cortex/metal/mlx_compat.py,sha256=oZ_RNjJzWs6h6Q3mSNK-K2--BhJwKt7d_tYt3uMN1pM,3531
+cortex/metal/mlx_converter.py,sha256=6tiVDFWOZOmhwTN2uS93_Ny4iM1X0NTrGp6AAik6l0Q,30823
 cortex/metal/mps_optimizer.py,sha256=4r6dj-_KAr3vedCwwu7lR-nIaF4g4D4kkOoF2KiQ0FQ,15307
 cortex/metal/optimizer.py,sha256=9ixKj8ca1iovF-mFHYGa9_DUHcqgGyzLoP_lIRAzfMM,21996
 cortex/metal/performance_profiler.py,sha256=GMxxqwqE2kVJ4WePwVdUp2ADqhrV6wCCNrFnaMfBDpI,12274
 cortex/quantization/__init__.py,sha256=ElLP3ZO_XItddTl-PeoJ5GPb16RYIAk8m5sqwfAVE9s,184
-cortex/quantization/dynamic_quantizer.py,sha256=sAoHoQ6wfs6FvejG-iehB2Qij-0WC9qSTlBfj3D1pTI,31724
+cortex/quantization/dynamic_quantizer.py,sha256=vV0RSPMoWeOPALwFOs0DzqIA2MkGpeEpqB2vTeudhW0,31934
 cortex/template_registry/__init__.py,sha256=O5BWmHRmfMSK-Ukpu8UqFO_kaN0kum-d-Wsz0Ds-sC0,491
 cortex/template_registry/auto_detector.py,sha256=lqI19Ef_w6ClZvD5dzDw1i5gnf2AUN_L4WjCMvW99Yg,5432
 cortex/template_registry/config_manager.py,sha256=vh7cXAUTJ4dLY74u5EHTpTa46jXxj34BlMyWsC_ZIaM,8658
@@ -37,12 +38,12 @@ cortex/template_registry/template_profiles/standard/gemma.py,sha256=D4wZN3_6QzUj
 cortex/template_registry/template_profiles/standard/llama.py,sha256=jz4MyvmISSPtIAcffPE7LrTosHvlC0NoJhzTw1DCvpY,3209
 cortex/template_registry/template_profiles/standard/simple.py,sha256=dGOOcL6HRoJFxkixLrYC4w7c63h-QmOOWC2TsOihYog,2422
 cortex/ui/__init__.py,sha256=t3GrHJMHTVgBEKh2_qt4B9mS594V5jriTDqc3eZKMGc,3409
-cortex/ui/cli.py,sha256=oxHRIhLu_izy4ZfBWqUngnV060UD4P0J8bcvquKz2lU,74632
-cortex/ui/markdown_render.py,sha256=bXt60vkNYT_jbpKeIg_1OlcrxssmdbMO7RB2E1sWw3E,5759
+cortex/ui/cli.py,sha256=QZhiV9z8hP9Fu5mvpzURSWLptDDRaJLmNLm2AqTGlqE,75734
+cortex/ui/markdown_render.py,sha256=D4gSvv0TERFIAXYs3e76eaPsuvvD2cNT98PDKyUPnWI,7776
 cortex/ui/terminal_app.py,sha256=SF3KqcGFyZ4hpTmgX21idPzOTJLdKGkt4QdA-wwUBNE,18317
-cortex_llm-1.0.0.dist-info/licenses/LICENSE,sha256=_frJ3VsZWQGhMznZw2Tgjk7xwfAfDZRcBl43uZh8_4E,1070
-cortex_llm-1.0.0.dist-info/METADATA,sha256=STGY7B8BthPSY29Slojmq1_HiXsQ5NxUtgwQoKPToMg,10087
-cortex_llm-1.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-cortex_llm-1.0.0.dist-info/entry_points.txt,sha256=g83Nuz3iFrNdMLHxGLR2LnscdM7rdQRchuL3WGobQC8,48
-cortex_llm-1.0.0.dist-info/top_level.txt,sha256=79LAeTJJ_pMIBy3mkF7uNaN0mdBRt5tGrnne5N_iAio,7
-cortex_llm-1.0.0.dist-info/RECORD,,
+cortex_llm-1.0.8.dist-info/licenses/LICENSE,sha256=_frJ3VsZWQGhMznZw2Tgjk7xwfAfDZRcBl43uZh8_4E,1070
+cortex_llm-1.0.8.dist-info/METADATA,sha256=p_zPy3yz5xrZO-oIkujQm_agneaE1RNR2smb5nBreQc,5119
+cortex_llm-1.0.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+cortex_llm-1.0.8.dist-info/entry_points.txt,sha256=g83Nuz3iFrNdMLHxGLR2LnscdM7rdQRchuL3WGobQC8,48
+cortex_llm-1.0.8.dist-info/top_level.txt,sha256=79LAeTJJ_pMIBy3mkF7uNaN0mdBRt5tGrnne5N_iAio,7
+cortex_llm-1.0.8.dist-info/RECORD,,

cortex_llm-1.0.0.dist-info/METADATA

@@ -1,275 +0,0 @@
-Metadata-Version: 2.4
-Name: cortex-llm
-Version: 1.0.0
-Summary: GPU-Accelerated LLM Terminal for Apple Silicon
-Home-page: https://github.com/faisalmumtaz/Cortex
-Author: Cortex Development Team
-License: MIT
-Project-URL: Homepage, https://github.com/faisalmumtaz/Cortex
-Project-URL: Bug Tracker, https://github.com/faisalmumtaz/Cortex/issues
-Project-URL: Documentation, https://github.com/faisalmumtaz/Cortex/wiki
-Keywords: llm,gpu,metal,mps,apple-silicon,ai,machine-learning,terminal,mlx,pytorch
-Platform: darwin
-Classifier: Development Status :: 4 - Beta
-Classifier: Intended Audience :: Developers
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Operating System :: MacOS
-Classifier: Environment :: Console
-Classifier: Environment :: GPU
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: torch>=2.1.0
-Requires-Dist: mlx>=0.10.0
-Requires-Dist: mlx-lm>=0.10.0
-Requires-Dist: transformers>=4.36.0
-Requires-Dist: safetensors>=0.4.0
-Requires-Dist: huggingface-hub>=0.19.0
-Requires-Dist: accelerate>=0.25.0
-Requires-Dist: llama-cpp-python>=0.2.0
-Requires-Dist: pyyaml>=6.0
-Requires-Dist: pydantic>=2.5.0
-Requires-Dist: rich>=13.0.0
-Requires-Dist: psutil>=5.9.0
-Requires-Dist: numpy>=1.24.0
-Requires-Dist: packaging>=23.0
-Requires-Dist: requests>=2.31.0
-Provides-Extra: dev
-Requires-Dist: pytest>=7.4.0; extra == "dev"
-Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
-Requires-Dist: black>=23.0.0; extra == "dev"
-Requires-Dist: ruff>=0.1.0; extra == "dev"
-Requires-Dist: mypy>=1.8.0; extra == "dev"
-Provides-Extra: optional
-Requires-Dist: sentencepiece>=0.1.99; extra == "optional"
-Requires-Dist: auto-gptq>=0.7.0; extra == "optional"
-Requires-Dist: autoawq>=0.2.0; extra == "optional"
-Requires-Dist: bitsandbytes>=0.41.0; extra == "optional"
-Requires-Dist: optimum>=1.16.0; extra == "optional"
-Requires-Dist: torchvision>=0.16.0; extra == "optional"
-Requires-Dist: torchaudio>=2.1.0; extra == "optional"
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: platform
-Dynamic: requires-python
-
-# Cortex - LLM Terminal Client for Apple Silicon
-
-Cortex is an LLM terminal interface designed for Apple Silicon, using MLX and PyTorch MPS frameworks for GPU-accelerated inference.
-
-## What It Does
-
-- **GPU-accelerated inference** via MLX (primary) and PyTorch MPS backends
-- **Apple Silicon required** - leverages unified memory architecture
-- **Multiple model formats** - MLX, GGUF, SafeTensors, PyTorch, GPTQ, AWQ
-- **Built-in fine-tuning** - LoRA-based model customization via interactive wizard
-- **Chat template auto-detection** - automatic format detection with confidence scoring
-- **Conversation persistence** - SQLite-backed chat history with branching
-
-## Features
-
-- **GPU-Accelerated Inference** - Delegates to MLX and PyTorch MPS for Metal-based execution
-- **Apple Silicon Only** - Requires Metal GPU; exits if GPU acceleration is unavailable
-- **Model Format Support**:
-  - MLX (Apple's format, loaded via `mlx_lm`)
-  - GGUF (via `llama-cpp-python` with Metal backend)
-  - SafeTensors (via HuggingFace `transformers`)
-  - PyTorch models (via HuggingFace `transformers` with MPS device)
-  - GPTQ quantized (via `auto-gptq`)
-  - AWQ quantized (via `awq`)
-- **Quantization** - 4-bit, 5-bit, 8-bit, and mixed-precision quantization via MLX conversion pipeline
-- **Model Conversion** - Convert HuggingFace models to MLX format with configurable quantization recipes
-- **Template Registry** - Automatic detection of chat templates (ChatML, Llama, Alpaca, Gemma, Reasoning) with confidence scoring and real-time token filtering for reasoning models
-- **Rotating KV Cache** - MLX-based KV cache for long context handling (default 4096 tokens)
-- **Fine-Tuning** - LoRA-based model customization with interactive 6-step wizard
-- **Terminal UI** - ANSI terminal interface with streaming output
-
-## Installation
-
-```bash
-# Clone and install
-git clone https://github.com/faisalmumtaz/Cortex.git
-cd Cortex
-./install.sh
-```
-
-The installer:
-- Checks for Apple Silicon (arm64) compatibility
-- Creates a Python virtual environment
-- Installs dependencies via `pip install -e .` (from `pyproject.toml`)
-- Sets up the `cortex` command in your PATH
-
-### Quick Install (pipx)
-
-If you just want the CLI without cloning the repo, use pipx:
-
-```bash
-pipx install cortex-llm
-```
-
-## Quick Start
-
-```bash
-# After installation, just run:
-cortex
-```
-
-### Downloading Models
-
-```bash
-# Inside Cortex, use the download command:
-cortex
-# Then type: /download
-```
-
-The download feature:
-- **HuggingFace integration** - download any model by repository ID
-- **Automatic loading** - option to load model immediately after download
-
-## Documentation
-
-### User Documentation
-- **[Installation Guide](docs/installation.md)** - Complete setup instructions
-- **[CLI Reference](docs/cli.md)** - Commands and user interface
-- **[Configuration](docs/configuration.md)** - System settings and optimization
-- **[Model Management](docs/model-management.md)** - Loading and managing models
-- **[Template Registry](docs/template-registry.md)** - Automatic chat template detection and management
-- **[Fine-Tuning Guide](docs/fine-tuning.md)** - Customize models with LoRA
-- **[Troubleshooting](docs/troubleshooting.md)** - Common issues and solutions
-
-### Technical Documentation
-- **[MLX Acceleration](docs/mlx-acceleration.md)** - MLX framework integration and optimization
-- **[GPU Validation](docs/gpu-validation.md)** - Hardware requirements and detection
-- **[Inference Engine](docs/inference-engine.md)** - Text generation architecture
-- **[Conversation Management](docs/conversation-management.md)** - Chat history and persistence
-- **[Development Guide](docs/development.md)** - Contributing and architecture
-
-## System Requirements
-
-- Apple Silicon Mac (M1/M2/M3/M4 - all variants supported)
-- macOS 13.3+ (required by MLX framework)
-- Python 3.11+
-- 16GB+ unified memory (24GB+ recommended for larger models)
-- Xcode Command Line Tools
-
-## Performance
-
-Performance depends on your Apple Silicon chip, model size, and quantization level. The inference engine measures tokens/second, first-token latency, and memory usage at runtime.
-
-To check that GPU acceleration is working:
-
-```bash
-source venv/bin/activate
-python tests/test_apple_silicon.py
-```
-
-You should see:
-- All validation checks passing
-- Measured GFLOPS from matrix operations
-- Confirmation of Metal and MLX availability
-
-## GPU Acceleration Architecture
-
-Cortex uses a multi-layer approach, delegating all GPU computation to established frameworks:
-
-1. **MLX Framework (Primary Backend)**
-   - Apple's ML framework with native Metal support
-   - Quantization support (4-bit, 5-bit, 8-bit, mixed-precision)
-   - Rotating KV cache for long contexts
-   - JIT compilation via `mx.compile`
-   - Operation fusion for reduced kernel launches
-
-2. **PyTorch MPS Backend**
-   - Metal Performance Shaders for PyTorch models
-   - FP16 optimization and channels-last tensor format
-
-3. **llama.cpp (GGUF Backend)**
-   - Metal-accelerated inference for GGUF models
-
-4. **Memory Management**
-   - Pre-allocated memory pools with best-fit/first-fit allocation strategies
-   - Automatic pool sizing (60% of available memory, capped at 75% of total)
-   - Defragmentation support
-
-### Understanding "Skipping Kernel" Messages
-
-When loading GGUF models, you may see messages like:
-```
-ggml_metal_init: skipping kernel_xxx_bf16 (not supported)
-```
-
-**These are NORMAL!** They indicate:
-- BF16 kernels being skipped (your GPU uses FP16 instead)
-- GPU acceleration is still fully active
-- The system automatically uses optimal alternatives
-
-## Troubleshooting
-
-If you suspect GPU isn't being used:
-
-1. **Run validation**: `python tests/test_apple_silicon.py`
-2. **Check output**: Should see passing checks and measured GFLOPS
-3. **Monitor tokens/sec**: Displayed during inference
-4. **Verify Metal**: Ensure Xcode Command Line Tools installed
-
-Common issues:
-- **Low performance**: Run `python tests/test_apple_silicon.py` to diagnose
-- **Memory errors**: Reduce `gpu_memory_fraction` in config.yaml
-
-## MLX Model Conversion
-
-Cortex includes an MLX model converter:
-
-```python
-from cortex.metal.mlx_converter import MLXConverter, ConversionConfig, QuantizationRecipe
-
-converter = MLXConverter()
-config = ConversionConfig(
-    quantization=QuantizationRecipe.SPEED_4BIT,  # 4-bit quantization
-    compile_model=True  # JIT compilation
-)
-
-success, message, output_path = converter.convert_model(
-    "microsoft/DialoGPT-medium",
-    config=config
-)
-```
-
-### Quantization Options
-
-- **4-bit**: Maximum speed, 75% size reduction
-- **5-bit**: Balanced speed and quality
-- **8-bit**: Higher quality, 50% size reduction
-- **Mixed Precision**: Custom per-layer quantization
-
-## MLX as Primary Backend
-
-Cortex uses MLX (Apple's machine learning framework) as the primary acceleration backend:
-- **Metal Support**: GPU execution via MLX's built-in Metal operations
-- **Quantization**: Support for 4-bit, 5-bit, 8-bit, and mixed-precision quantization
-- **Model Conversion**: Convert HuggingFace models to MLX format
-
-## Built With
-
-- [MLX](https://github.com/ml-explore/mlx) - Apple's machine learning framework
-- [mlx-lm](https://github.com/ml-explore/mlx-examples) - LLM utilities and LoRA fine-tuning for MLX
-- [PyTorch](https://pytorch.org/) - With Metal Performance Shaders backend
-- [llama.cpp](https://github.com/ggerganov/llama.cpp) - Metal-accelerated GGUF support
-- [Rich](https://github.com/Textualize/rich) - Terminal formatting
-- [HuggingFace](https://huggingface.co/) - Model hub and transformers
-
-## Contributing
-
-We welcome contributions! Please see the [Development Guide](docs/development.md) for contributing guidelines and setup instructions.
-
-## License
-
-MIT License - See [LICENSE](LICENSE) for details.
-
----
-
-**Note**: Cortex requires Apple Silicon. Intel Macs are not supported.