ctxsync 0.8.0__py3-none-any.whl

ctxsync/providers/claude_ai.py

@@ -0,0 +1,192 @@
+import urllib.request
+import urllib.error
+import urllib.parse
+import json
+import gzip
+from datetime import datetime, timezone
+from .base_claude_ai import BaseClaudeAIProvider
+from ..exceptions import ProviderError
+
+
+class ClaudeAIProvider(BaseClaudeAIProvider):
+    def __init__(self, config=None):
+        super().__init__(config)
+
+    def _make_request_internal(  # noqa: C901
+        self, method, endpoint, data, base_url, extra_headers=None
+    ):
+        """Internal method to make HTTP requests with specified base URL."""
+        url = f"{base_url}{endpoint}"
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0",
+            "Content-Type": "application/json",
+            "Accept-Encoding": "gzip",
+        }
+
+        if extra_headers:
+            headers.update(extra_headers)
+
+        session_key, expiry = self.config.get_session_key("claude.ai")
+        cookies = {
+            "sessionKey": session_key,
+        }
+
+        try:
+            self.logger.debug(f"Making {method} request to {url}")
+            self.logger.debug(f"Headers: {headers}")
+            self.logger.debug(f"Cookies: {cookies}")
+            if data:
+                self.logger.debug(f"Request data: {data}")
+
+            # Prepare the request
+            req = urllib.request.Request(url, method=method)
+            for key, value in headers.items():
+                req.add_header(key, value)
+
+            # Add cookies
+            cookie_string = "; ".join([f"{k}={v}" for k, v in cookies.items()])
+            req.add_header("Cookie", cookie_string)
+
+            # Add data if present
+            if data:
+                json_data = json.dumps(data).encode("utf-8")
+                req.data = json_data
+
+            # Make the request
+            with urllib.request.urlopen(req) as response:
+                self.logger.debug(f"Response status code: {response.status}")
+                self.logger.debug(f"Response headers: {response.headers}")
+
+                # Handle gzip encoding
+                if response.headers.get("Content-Encoding") == "gzip":
+                    content = gzip.decompress(response.read())
+                else:
+                    content = response.read()
+
+                content_str = content.decode("utf-8")
+                self.logger.debug(f"Response content: {content_str[:1000]}...")
+
+                if not content:
+                    return None
+
+                return json.loads(content_str)
+
+        except urllib.error.HTTPError as e:
+            self.handle_http_error(e)
+        except urllib.error.URLError as e:
+            self.logger.error(f"URL Error: {str(e)}")
+            raise ProviderError(f"API request failed: {str(e)}")
+        except json.JSONDecodeError as json_err:
+            self.logger.error(f"Failed to parse JSON response: {str(json_err)}")
+            self.logger.error(f"Response content: {content_str}")
+            raise ProviderError(f"Invalid JSON response from API: {str(json_err)}")
+
+    def _make_request(self, method, endpoint, data=None):
+        return self._make_request_internal(method, endpoint, data, self.base_url)
+
+    def handle_http_error(self, e):
+        self.logger.debug(f"Request failed: {str(e)}")
+        self.logger.debug(f"Response status code: {e.code}")
+        self.logger.debug(f"Response headers: {e.headers}")
+
+        try:
+            # Check if the content is gzip-encoded
+            if e.headers.get("Content-Encoding") == "gzip":
+                content = gzip.decompress(e.read())
+            else:
+                content = e.read()
+
+            # Try to decode the content as UTF-8
+            content_str = content.decode("utf-8")
+        except UnicodeDecodeError:
+            # If UTF-8 decoding fails, try to decode as ISO-8859-1
+            content_str = content.decode("iso-8859-1")
+
+        self.logger.debug(f"Response content: {content_str}")
+
+        if e.code == 403:
+            error_msg = "Received a 403 Forbidden error."
+            raise ProviderError(error_msg)
+        elif e.code == 429:
+            try:
+                error_data = json.loads(content_str)
+                resets_at_unix = json.loads(error_data["error"]["message"])["resetsAt"]
+                resets_at_local = datetime.fromtimestamp(
+                    resets_at_unix, tz=timezone.utc
+                ).astimezone()
+                formatted_time = resets_at_local.strftime("%a %b %d %Y %H:%M:%S %Z%z")
+                error_msg = f"Message limit exceeded. Try again after {formatted_time}"
+            except (KeyError, json.JSONDecodeError) as parse_error:
+                error_msg = f"HTTP 429: Too Many Requests. Failed to parse error response: {parse_error}"
+            self.logger.error(error_msg)
+            raise ProviderError(error_msg)
+        else:
+            error_msg = f"API request failed with status code {e.code}: {content_str}"
+            self.logger.error(error_msg)
+            raise ProviderError(error_msg)
+
+    def _make_request_v1(self, method, endpoint, data=None, organization_id=None):
+        """Make a request to the v1 API (not under /api prefix)."""
+        # For v1 endpoints, we use the root URL without the /api prefix
+        base_url = self.base_url.replace("/api", "")
+
+        # Add required Anthropic headers for v1 API
+        extra_headers = {
+            "anthropic-version": "2023-06-01",
+        }
+
+        # Add organization header if provided
+        if organization_id:
+            extra_headers["x-organization-uuid"] = organization_id
+
+        return self._make_request_internal(
+            method, endpoint, data, base_url, extra_headers
+        )
+
+    def _make_request_stream(self, method, endpoint, data=None):
+        url = f"{self.base_url}{endpoint}"
+        session_key, _ = self.config.get_session_key("claude.ai")
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0",
+            "Content-Type": "application/json",
+            "Accept": "text/event-stream",
+            "Cookie": f"sessionKey={session_key}",
+        }
+
+        req = urllib.request.Request(url, method=method, headers=headers)
+        if data:
+            req.data = json.dumps(data).encode("utf-8")
+
+        try:
+            return urllib.request.urlopen(req)
+        except urllib.error.HTTPError as e:
+            self.handle_http_error(e)
+        except urllib.error.URLError as e:
+            raise ProviderError(f"API request failed: {str(e)}")
+
+    def _make_request_stream_v1(self, method, endpoint, organization_id=None):
+        """Make a streaming request to the v1 API."""
+        # For v1 endpoints, use root URL without /api prefix
+        base_url = self.base_url.replace("/api", "")
+        url = f"{base_url}{endpoint}"
+
+        session_key, _ = self.config.get_session_key("claude.ai")
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0",
+            "Accept": "text/event-stream",
+            "anthropic-version": "2023-06-01",
+            "Cookie": f"sessionKey={session_key}",
+        }
+
+        # Add organization header if provided
+        if organization_id:
+            headers["x-organization-uuid"] = organization_id
+
+        req = urllib.request.Request(url, method=method, headers=headers)
+
+        try:
+            return urllib.request.urlopen(req)
+        except urllib.error.HTTPError as e:
+            self.handle_http_error(e)
+        except urllib.error.URLError as e:
+            raise ProviderError(f"API request failed: {str(e)}")

ctxsync/session_key_manager.py

@@ -0,0 +1,129 @@
+import subprocess
+import base64
+import logging
+from pathlib import Path
+from cryptography.fernet import Fernet
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+
+class SessionKeyManager:
+    def __init__(self, ssh_key_path=None):
+        self.logger = logging.getLogger(__name__)
+        # Allow config-provided ssh_key_path to guide key discovery
+        self.ssh_key_path = self._find_ssh_key(ssh_key_path)
+
+    def _find_ssh_key(self, configured_path=None):
+        """
+        Locate an SSH private key for session encryption.
+        Priority:
+          1. If configured_path points to a specific file, check it first
+          2. If configured_path is a directory, search it alongside ~/.ssh
+          3. Fall back to ~/.ssh with default key names
+          4. Prompt the user as a last resort
+        """
+        default_ssh_dir = Path.home() / ".ssh"
+        key_names = ["id_ed25519", "id_ecdsa"]
+        search_dirs = [default_ssh_dir]
+
+        if configured_path:
+            configured = Path(configured_path)
+
+            if configured.is_file():
+                # Config points directly to a key file — use it immediately
+                return str(configured)
+
+            if configured.is_dir():
+                # Config is a directory — search it in addition to ~/.ssh
+                if configured != default_ssh_dir:
+                    search_dirs.insert(0, configured)
+            else:
+                # Path doesn't exist — warn and fall through to defaults
+                self.logger.warning(
+                    "Configured ssh_key_path not found: %s", configured_path
+                )
+
+        # Search all candidate directories for supported key names
+        for search_dir in search_dirs:
+            for key_name in key_names:
+                key_path = search_dir / key_name
+                if key_path.exists():
+                    return str(key_path)
+
+        # If no supported key is found, prompt the user to generate an Ed25519 key
+        self.logger.warning(
+            "* No supported SSH key found. RSA keys are no longer supported."
+        )
+        self.logger.warning(
+            "* Please generate an Ed25519 key using the following command:"
+        )
+        self.logger.warning('  ssh-keygen -t ed25519 -C "your_email@example.com"')
+        self.logger.warning(
+            "* If you have NOT specified a custom ssh_key_path in config,"
+        )
+        self.logger.warning("* have created a key, and are still seeing this message,")
+        self.logger.warning(
+            "  be sure to name your key 'id_ed25519' or 'id_ecdsa' so it's found automatically."
+        )
+        self.logger.warning(
+            "* Or set ssh_key_path with the full key name in your .ctxsync/config.local.json"
+        )
+        return input("Enter the full path to your new Ed25519 private key: ")
+
+    def _get_key_type(self):
+        try:
+            result = subprocess.run(
+                ["ssh-keygen", "-l", "-f", self.ssh_key_path],
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            output = result.stdout.lower()
+            if "ecdsa" in output:
+                return "ecdsa"
+            elif "ed25519" in output:
+                return "ed25519"
+            else:
+                raise ValueError(f"Unsupported key type for {self.ssh_key_path}")
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"Failed to determine key type: {e}")
+            raise RuntimeError(
+                "Failed to determine SSH key type. Make sure the key file is valid and accessible."
+            )
+
+    def _derive_key_from_ssh_key(self):
+        with open(self.ssh_key_path, "rb") as key_file:
+            ssh_key_data = key_file.read()
+
+        kdf = PBKDF2HMAC(
+            algorithm=hashes.SHA256(),
+            length=32,
+            salt=b"ctxsync",  # Using a fixed salt; consider using a secure random salt in production
+            iterations=100000,
+        )
+        key = base64.urlsafe_b64encode(kdf.derive(ssh_key_data))
+        return key
+
+    def encrypt_session_key(self, provider, session_key):
+        self._get_key_type()
+        return self._encrypt_symmetric(session_key)
+
+    def _encrypt_symmetric(self, session_key):
+        key = self._derive_key_from_ssh_key()
+        f = Fernet(key)
+        encrypted_session_key = f.encrypt(session_key.encode()).decode()
+        return encrypted_session_key, "symmetric"
+
+    def decrypt_session_key(self, provider, encryption_method, encrypted_session_key):
+        if not encrypted_session_key or not encryption_method:
+            return None
+
+        if encryption_method == "symmetric":
+            return self._decrypt_symmetric(encrypted_session_key)
+        else:
+            raise ValueError(f"Unknown encryption method: {encryption_method}")
+
+    def _decrypt_symmetric(self, encrypted_session_key):
+        key = self._derive_key_from_ssh_key()
+        f = Fernet(key)
+        return f.decrypt(encrypted_session_key.encode()).decode()

ctxsync/syncmanager.py

@@ -0,0 +1,328 @@
+import functools
+import os
+import time
+import logging
+from datetime import datetime, timezone
+import io
+
+from tqdm import tqdm
+
+from ctxsync.utils import compute_md5_hash
+from ctxsync.exceptions import ProviderError
+from .compression import compress_content, decompress_content
+
+logger = logging.getLogger(__name__)
+
+
+def retry_on_403(max_retries=3, delay=1):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            self = args[0] if len(args) > 0 else None
+            for attempt in range(max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except ProviderError as e:
+                    if "403 Forbidden" in str(e) and attempt < max_retries - 1:
+                        if self and hasattr(self, "logger"):
+                            self.logger.warning(
+                                f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})"
+                            )
+                        else:
+                            logger.warning(
+                                f"Received 403 error. Retrying in {delay} seconds... (Attempt {attempt + 1}/{max_retries})"
+                            )
+                        time.sleep(delay)
+                    else:
+                        raise
+
+        return wrapper
+
+    return decorator
+
+
+class SyncManager:
+    def __init__(self, provider, config, local_path):
+        self.provider = provider
+        self.config = config
+        self.active_organization_id = config.get("active_organization_id")
+        self.active_project_id = config.get("active_project_id")
+        self.local_path = local_path
+        self.upload_delay = config.get("upload_delay", 0.5)
+        self.two_way_sync = config.get("two_way_sync", False)
+        self.max_retries = 3
+        self.retry_delay = 1
+        self.compression_algorithm = config.get("compression_algorithm", "none")
+        self.synced_files = {}
+
+    def sync(self, local_files, remote_files):
+        self.synced_files = {}  # Reset synced files at the start of sync
+        if self.compression_algorithm == "none":
+            self._sync_without_compression(local_files, remote_files)
+        else:
+            self._sync_with_compression(local_files, remote_files)
+
+    def _sync_without_compression(self, local_files, remote_files):
+        remote_files_to_delete = set(rf["file_name"] for rf in remote_files)
+        synced_files = set()
+
+        with tqdm(total=len(local_files), desc="Local → Remote") as pbar:
+            for local_file, local_checksum in local_files.items():
+                remote_file = next(
+                    (rf for rf in remote_files if rf["file_name"] == local_file), None
+                )
+                if remote_file:
+                    self.update_existing_file(
+                        local_file,
+                        local_checksum,
+                        remote_file,
+                        remote_files_to_delete,
+                        synced_files,
+                    )
+                else:
+                    self.upload_new_file(local_file, synced_files)
+                pbar.update(1)
+
+        self.update_local_timestamps(remote_files, synced_files)
+
+        if self.two_way_sync:
+            with tqdm(total=len(remote_files), desc="Local ← Remote") as pbar:
+                for remote_file in remote_files:
+                    self.sync_remote_to_local(
+                        remote_file, remote_files_to_delete, synced_files
+                    )
+                    pbar.update(1)
+
+        self.prune_remote_files(remote_files, remote_files_to_delete)
+
+    def _sync_with_compression(self, local_files, remote_files):
+        packed_content = self._pack_files(local_files)
+        compressed_content = compress_content(
+            packed_content, self.compression_algorithm
+        )
+
+        remote_file_name = (
+            f"ctxsync_packed_{datetime.now().strftime('%Y%m%d%H%M%S')}.dat"
+        )
+        self._upload_compressed_file(compressed_content, remote_file_name)
+
+        if self.two_way_sync:
+            remote_compressed_content = self._download_compressed_file()
+            if remote_compressed_content:
+                remote_packed_content = decompress_content(
+                    remote_compressed_content, self.compression_algorithm
+                )
+                self._unpack_files(remote_packed_content)
+
+        self._cleanup_old_remote_files(remote_files)
+
+    def _pack_files(self, local_files):
+        packed_content = io.StringIO()
+        for file_path, file_hash in local_files.items():
+            full_path = os.path.join(self.local_path, file_path)
+            with open(full_path, "r", encoding="utf-8") as f:
+                content = f.read()
+            packed_content.write(f"--- BEGIN FILE: {file_path} ---\n")
+            packed_content.write(content)
+            packed_content.write(f"\n--- END FILE: {file_path} ---\n")
+        return packed_content.getvalue()
+
+    @retry_on_403()
+    def _upload_compressed_file(self, compressed_content, file_name):
+        logger.debug(f"Uploading compressed file {file_name} to remote...")
+        self.provider.upload_file(
+            self.active_organization_id,
+            self.active_project_id,
+            file_name,
+            compressed_content,
+        )
+        time.sleep(self.upload_delay)
+
+    @retry_on_403()
+    def _download_compressed_file(self):
+        logger.debug("Downloading latest compressed file from remote...")
+        remote_files = self.provider.list_files(
+            self.active_organization_id, self.active_project_id
+        )
+        compressed_files = [
+            rf for rf in remote_files if rf["file_name"].startswith("ctxsync_packed_")
+        ]
+        if compressed_files:
+            latest_file = max(compressed_files, key=lambda x: x["file_name"])
+            return latest_file["content"]
+        return None
+
+    def _unpack_files(self, packed_content):
+        current_file = None
+        current_content = io.StringIO()
+
+        for line in packed_content.splitlines():
+            if line.startswith("--- BEGIN FILE:"):
+                if current_file:
+                    self._write_file(current_file, current_content.getvalue())
+                    current_content = io.StringIO()
+                current_file = line.split("--- BEGIN FILE:")[1].strip()
+            elif line.startswith("--- END FILE:"):
+                if current_file:
+                    self._write_file(current_file, current_content.getvalue())
+                    current_file = None
+                    current_content = io.StringIO()
+            else:
+                current_content.write(line + "\n")
+
+        if current_file:
+            self._write_file(current_file, current_content.getvalue())
+
+    def _write_file(self, file_path, content):
+        full_path = os.path.join(self.local_path, file_path)
+        os.makedirs(os.path.dirname(full_path), exist_ok=True)
+        with open(full_path, "w", encoding="utf-8") as f:
+            f.write(content)
+
+    def _cleanup_old_remote_files(self, remote_files):
+        for remote_file in remote_files:
+            if remote_file["file_name"].startswith("ctxsync_packed_"):
+                self.provider.delete_file(
+                    self.active_organization_id,
+                    self.active_project_id,
+                    remote_file["uuid"],
+                )
+
+    @retry_on_403()
+    def update_existing_file(
+        self,
+        local_file,
+        local_checksum,
+        remote_file,
+        remote_files_to_delete,
+        synced_files,
+    ):
+        remote_content = remote_file["content"]
+        remote_checksum = compute_md5_hash(remote_content)
+        if local_checksum != remote_checksum:
+            logger.debug(f"Updating {local_file} on remote...")
+            with tqdm(total=2, desc=f"Updating {local_file}", leave=False) as pbar:
+                self.provider.delete_file(
+                    self.active_organization_id,
+                    self.active_project_id,
+                    remote_file["uuid"],
+                )
+                pbar.update(1)
+                with open(
+                    os.path.join(self.local_path, local_file), "r", encoding="utf-8"
+                ) as file:
+                    content = file.read()
+                self.provider.upload_file(
+                    self.active_organization_id,
+                    self.active_project_id,
+                    local_file,
+                    content,
+                )
+                pbar.update(1)
+            time.sleep(self.upload_delay)
+            synced_files.add(local_file)
+        remote_files_to_delete.remove(local_file)
+
+    @retry_on_403()
+    def upload_new_file(self, local_file, synced_files):
+        logger.debug(f"Uploading new file {local_file} to remote...")
+        with open(
+            os.path.join(self.local_path, local_file), "r", encoding="utf-8"
+        ) as file:
+            content = file.read()
+        with tqdm(total=1, desc=f"Uploading {local_file}", leave=False) as pbar:
+            self.provider.upload_file(
+                self.active_organization_id, self.active_project_id, local_file, content
+            )
+            pbar.update(1)
+        time.sleep(self.upload_delay)
+        synced_files.add(local_file)
+
+    def update_local_timestamps(self, remote_files, synced_files):
+        for remote_file in remote_files:
+            if remote_file["file_name"] in synced_files:
+                local_file_path = os.path.join(
+                    self.local_path, remote_file["file_name"]
+                )
+                if os.path.exists(local_file_path):
+                    remote_timestamp = datetime.fromisoformat(
+                        remote_file["created_at"].replace("Z", "+00:00")
+                    ).timestamp()
+                    os.utime(local_file_path, (remote_timestamp, remote_timestamp))
+                    logger.debug(f"Updated timestamp on local file {local_file_path}")
+
+    def sync_remote_to_local(self, remote_file, remote_files_to_delete, synced_files):
+        local_file_path = os.path.join(self.local_path, remote_file["file_name"])
+        if os.path.exists(local_file_path):
+            self.update_existing_local_file(
+                local_file_path, remote_file, remote_files_to_delete, synced_files
+            )
+        else:
+            self.create_new_local_file(
+                local_file_path, remote_file, remote_files_to_delete, synced_files
+            )
+
+    def update_existing_local_file(
+        self, local_file_path, remote_file, remote_files_to_delete, synced_files
+    ):
+        local_mtime = datetime.fromtimestamp(
+            os.path.getmtime(local_file_path), tz=timezone.utc
+        )
+        remote_mtime = datetime.fromisoformat(
+            remote_file["created_at"].replace("Z", "+00:00")
+        )
+        if remote_mtime > local_mtime:
+            logger.debug(
+                f"Updating local file {remote_file['file_name']} from remote..."
+            )
+            content = remote_file["content"]
+            with open(local_file_path, "w", encoding="utf-8") as file:
+                file.write(content)
+            synced_files.add(remote_file["file_name"])
+            if remote_file["file_name"] in remote_files_to_delete:
+                remote_files_to_delete.remove(remote_file["file_name"])
+
+    def create_new_local_file(
+        self, local_file_path, remote_file, remote_files_to_delete, synced_files
+    ):
+        logger.debug(
+            f"Creating new local file {remote_file['file_name']} from remote..."
+        )
+        content = remote_file["content"]
+        with tqdm(
+            total=1, desc=f"Creating {remote_file['file_name']}", leave=False
+        ) as pbar:
+            with open(local_file_path, "w", encoding="utf-8") as file:
+                file.write(content)
+            pbar.update(1)
+        synced_files.add(remote_file["file_name"])
+        if remote_file["file_name"] in remote_files_to_delete:
+            remote_files_to_delete.remove(remote_file["file_name"])
+
+    def prune_remote_files(self, remote_files, remote_files_to_delete):
+        if not self.config.get("prune_remote_files"):
+            logger.info("Remote pruning is not enabled.")
+            return
+
+        for file_to_delete in list(remote_files_to_delete):
+            self.delete_remote_files(file_to_delete, remote_files)
+
+    @retry_on_403()
+    def delete_remote_files(self, file_to_delete, remote_files):
+        logger.debug(f"Deleting {file_to_delete} from remote...")
+        remote_file = next(
+            rf for rf in remote_files if rf["file_name"] == file_to_delete
+        )
+        with tqdm(total=1, desc=f"Deleting {file_to_delete}", leave=False) as pbar:
+            self.provider.delete_file(
+                self.active_organization_id, self.active_project_id, remote_file["uuid"]
+            )
+            pbar.update(1)
+        time.sleep(self.upload_delay)
+
+    def embedding(self, local_files):
+        packed_content = self._pack_files(local_files)
+        compressed_content = compress_content(
+            packed_content, self.compression_algorithm
+        )
+        return compressed_content