compair-core 0.3.1__tar.gz → 0.3.3__tar.gz

{compair_core-0.3.1 → compair_core-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compair-core
-Version: 0.3.1
+Version: 0.3.3
 Summary: Open-source foundation of the Compair collaboration platform.
 Author: RocketResearch, Inc.
 License: MIT
@@ -96,8 +96,8 @@ See `compair_core/server/settings.py` for the full settings surface.
 ```bash
 python -m venv .venv
 source .venv/bin/activate
-pip install -e .[dev]
-uvicorn compair_core.server.app:create_app --factory --reload
+pip install -e ".[dev]"
+uvicorn compair.server.app:create_app --factory --reload
 ```
 
 The API will be available at http://127.0.0.1:8000 and supports the Swagger UI at `/docs`.
@@ -106,6 +106,8 @@ The API will be available at http://127.0.0.1:8000 and supports the Swagger UI a
 
 Core currently ships with a syntax sanity check (`python -m compileall ...`). You can add pytest or other tooling as needed.
 
+Release and packaging steps are documented in `docs/maintainers.md`.
+
 ## Reporting Issues
 
 Please open GitHub issues or PRs against this repository. If you are a Compair Cloud customer, reach out through your support channel for issues related to premium features.

{compair_core-0.3.1 → compair_core-0.3.3}/README.md

@@ -62,8 +62,8 @@ See `compair_core/server/settings.py` for the full settings surface.
 ```bash
 python -m venv .venv
 source .venv/bin/activate
-pip install -e .[dev]
-uvicorn compair_core.server.app:create_app --factory --reload
+pip install -e ".[dev]"
+uvicorn compair.server.app:create_app --factory --reload
 ```
 
 The API will be available at http://127.0.0.1:8000 and supports the Swagger UI at `/docs`.
@@ -72,6 +72,8 @@ The API will be available at http://127.0.0.1:8000 and supports the Swagger UI a
 
 Core currently ships with a syntax sanity check (`python -m compileall ...`). You can add pytest or other tooling as needed.
 
+Release and packaging steps are documented in `docs/maintainers.md`.
+
 ## Reporting Issues
 
 Please open GitHub issues or PRs against this repository. If you are a Compair Cloud customer, reach out through your support channel for issues related to premium features.

compair_core-0.3.3/compair/__init__.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import logging
+import os
+
+from sqlalchemy import Engine, create_engine
+from sqlalchemy.orm import sessionmaker
+
+edition = os.getenv("COMPAIR_EDITION", "core").lower()
+
+_cloud_available = False
+if edition == "cloud":
+    try:  # Import cloud overrides if the private package is installed
+        from compair_cloud import (
+            bootstrap as cloud_bootstrap,
+            celery_app as cloud_celery_app,
+            default_groups as cloud_default_groups,
+            embeddings as cloud_embeddings,
+            feedback as cloud_feedback,
+            logger as cloud_logger,
+            main as cloud_main,
+            models as cloud_models,
+            tasks as cloud_tasks,
+            utils as cloud_utils,
+        )  # type: ignore
+
+        _cloud_available = True
+    except ImportError:
+        _cloud_available = False
+
+if _cloud_available:
+    from compair_cloud.default_groups import initialize_default_groups  # type: ignore
+    embeddings = cloud_embeddings
+    feedback = cloud_feedback
+    logger = cloud_logger
+    main = cloud_main
+    models = cloud_models
+    tasks = cloud_tasks
+    utils = cloud_utils
+    initialize_database_override = getattr(cloud_bootstrap, "initialize_database", None)
+else:
+    from . import embeddings, feedback, logger, main, models, tasks, utils
+    from .default_groups import initialize_default_groups
+    initialize_database_override = None
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def _handle_engine() -> Engine:
+    db = os.getenv("DB")
+    db_user = os.getenv("DB_USER")
+    db_passw = os.getenv("DB_PASSW")
+    db_url = os.getenv("DB_URL")
+
+    if all([db, db_user, db_passw, db_url]):
+        return create_engine(
+            f"postgresql+psycopg2://{db_user}:{db_passw}@{db_url}/{db}",
+            pool_size=10,
+            max_overflow=0,
+        )
+
+    sqlite_dir = os.getenv("COMPAIR_SQLITE_DIR", "/data")
+    os.makedirs(sqlite_dir, exist_ok=True)
+    sqlite_path = os.path.join(sqlite_dir, os.getenv("COMPAIR_SQLITE_NAME", "compair.db"))
+    return create_engine(f"sqlite:///{sqlite_path}", connect_args={"check_same_thread": False})
+
+
+engine = _handle_engine()
+
+
+def initialize_database() -> None:
+    models.Base.metadata.create_all(engine)
+    if initialize_database_override:
+        initialize_database_override(engine)
+
+
+initialize_database()
+
+Session = sessionmaker(engine)
+embedder = embeddings.Embedder()
+reviewer = feedback.Reviewer()
+
+with Session() as session:
+    initialize_default_groups(session)
+
+__all__ = ["embeddings", "feedback", "main", "models", "utils", "Session"]

compair_core-0.3.3/compair/celery_app.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+try:
+    from compair_cloud.celery_app import celery_app  # type: ignore
+except (ImportError, ModuleNotFoundError):
+
+    class _NoopCelery:
+        def __init__(self) -> None:
+            self.conf = SimpleNamespace(beat_schedule={})
+
+        def task(self, func=None, *args, **kwargs):
+            def decorator(fn):
+                return fn
+
+            return decorator(func) if func else decorator
+
+        def send_task(self, *args, **kwargs):
+            raise RuntimeError("Celery is not available in the Compair Core edition.")
+
+    celery_app = _NoopCelery()

compair_core-0.3.3/compair/default_groups.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from sqlalchemy.orm import Session
+
+try:
+    from compair_cloud.default_groups import initialize_default_groups as cloud_initialize_default_groups  # type: ignore
+except (ImportError, ModuleNotFoundError):
+    cloud_initialize_default_groups = None
+
+
+def initialize_default_groups(session: Session) -> None:
+    """Core builds do not seed any default groups by default."""
+    if cloud_initialize_default_groups:
+        cloud_initialize_default_groups(session)

compair_core-0.3.3/compair/embeddings.py

@@ -0,0 +1,66 @@
+import hashlib
+import os
+from typing import List
+
+import requests
+
+from .logger import log_event
+
+try:
+    from compair_cloud.embeddings import Embedder as CloudEmbedder  # type: ignore
+    from compair_cloud.embeddings import create_embedding as cloud_create_embedding  # type: ignore
+except (ImportError, ModuleNotFoundError):
+    CloudEmbedder = None
+    cloud_create_embedding = None
+
+
+class Embedder:
+    def __init__(self) -> None:
+        self.edition = os.getenv("COMPAIR_EDITION", "core").lower()
+        self._cloud_impl = None
+        if self.edition == "cloud" and CloudEmbedder is not None:
+            self._cloud_impl = CloudEmbedder()
+
+        if self._cloud_impl is None:
+            self.model = os.getenv("COMPAIR_LOCAL_EMBED_MODEL", "hash-embedding")
+            self.dimension = int(os.getenv("COMPAIR_LOCAL_EMBED_DIM", "384"))
+            base_url = os.getenv("COMPAIR_LOCAL_MODEL_URL", "http://local-model:9000")
+            route = os.getenv("COMPAIR_LOCAL_EMBED_ROUTE", "/embed")
+            self.endpoint = f"{base_url.rstrip('/')}{route}"
+
+    @property
+    def is_cloud(self) -> bool:
+        return self._cloud_impl is not None
+
+
+def _hash_embedding(text: str, dimension: int) -> List[float]:
+    """Generate a deterministic embedding using repeated SHA-256 hashing."""
+    if not text:
+        text = " "
+    digest = hashlib.sha256(text.encode("utf-8", "ignore")).digest()
+    vector: List[float] = []
+    while len(vector) < dimension:
+        for byte in digest:
+            vector.append((byte / 255.0) * 2 - 1)
+            if len(vector) == dimension:
+                break
+        digest = hashlib.sha256(digest).digest()
+    return vector
+
+
+def create_embedding(embedder: Embedder, text: str, user=None) -> list[float]:
+    if embedder.is_cloud and cloud_create_embedding is not None:
+        return cloud_create_embedding(embedder._cloud_impl, text, user=user)
+
+    # Local/core path
+    try:
+        response = requests.post(embedder.endpoint, json={"text": text}, timeout=15)
+        response.raise_for_status()
+        data = response.json()
+        embedding = data.get("embedding") or data.get("vector")
+        if embedding:
+            return embedding
+    except Exception as exc:
+        log_event("local_embedding_failed", error=str(exc))
+
+    return _hash_embedding(text, embedder.dimension)

compair_core-0.3.3/compair/feedback.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import os
+import requests
+from typing import Any
+
+from .logger import log_event
+from .models import Document, User
+
+try:
+    from compair_cloud.feedback import Reviewer as CloudReviewer  # type: ignore
+    from compair_cloud.feedback import get_feedback as cloud_get_feedback  # type: ignore
+except (ImportError, ModuleNotFoundError):
+    CloudReviewer = None  # type: ignore
+    cloud_get_feedback = None  # type: ignore
+
+
+class Reviewer:
+    """Edition-aware wrapper that falls back to the local feedback endpoint."""
+
+    def __init__(self) -> None:
+        self.edition = os.getenv("COMPAIR_EDITION", "core").lower()
+        self._cloud_impl = None
+        if self.edition == "cloud" and CloudReviewer is not None:
+            self._cloud_impl = CloudReviewer()
+        else:
+            self.client = None
+            self.model = os.getenv("COMPAIR_LOCAL_GENERATION_MODEL", "local-feedback")
+            base_url = os.getenv("COMPAIR_LOCAL_MODEL_URL", "http://local-model:9000")
+            route = os.getenv("COMPAIR_LOCAL_GENERATION_ROUTE", "/generate")
+            self.endpoint = f"{base_url.rstrip('/')}{route}"
+
+    @property
+    def is_cloud(self) -> bool:
+        return self._cloud_impl is not None
+
+
+def _fallback_feedback(text: str, references: list[Any]) -> str:
+    if not references:
+        return "NONE"
+    top_ref = references[0]
+    snippet = getattr(top_ref, "content", "") or ""
+    snippet = snippet.replace("\n", " ").strip()[:200]
+    if not snippet:
+        return "NONE"
+    return f"Check alignment with this reference: {snippet}"
+
+
+def get_feedback(
+    reviewer: Reviewer,
+    doc: Document,
+    text: str,
+    references: list[Any],
+    user: User,
+) -> str:
+    if reviewer.is_cloud and cloud_get_feedback is not None:
+        return cloud_get_feedback(reviewer._cloud_impl, doc, text, references, user)  # type: ignore[arg-type]
+
+    payload = {
+        "document": text,
+        "references": [getattr(ref, "content", "") for ref in references],
+        "length_instruction": {
+            "Brief": "1–2 short sentences",
+            "Detailed": "A couple short paragraphs",
+            "Verbose": "As thorough as reasonably possible without repeating information",
+        }.get(user.preferred_feedback_length, "1–2 short sentences"),
+    }
+
+    try:
+        response = requests.post(reviewer.endpoint, json=payload, timeout=30)
+        response.raise_for_status()
+        data = response.json()
+        feedback = data.get("feedback")
+        if feedback:
+            return feedback
+    except Exception as exc:  # pragma: no cover - network failures stay graceful
+        log_event("local_feedback_failed", error=str(exc))
+
+    return _fallback_feedback(text, references)

compair_core-0.3.3/compair/logger.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Callable
+
+try:
+    from compair_cloud.logger import log_event as cloud_log_event  # type: ignore
+except (ImportError, ModuleNotFoundError):
+    cloud_log_event: Callable[..., None] | None = None
+
+
+if not logging.getLogger().handlers:
+    logging.basicConfig(level=logging.INFO)
+
+_LOGGER = logging.getLogger("compair.core")
+
+
+def log_event(message: str, **fields: Any) -> None:
+    """Emit a structured log entry for the core edition."""
+    if cloud_log_event:
+        cloud_log_event(message, **fields)
+        return
+
+    try:
+        payload = json.dumps({"message": message, **fields}, default=str)
+    except TypeError:
+        payload = json.dumps({"message": message}, default=str)
+    _LOGGER.info(payload)

compair_core-0.3.3/compair/main.py

@@ -0,0 +1,240 @@
+from __future__ import annotations
+
+import logging
+import os
+from datetime import datetime, timedelta, timezone
+from typing import Mapping
+
+import Levenshtein
+from sqlalchemy import select
+from sqlalchemy.orm.attributes import get_history
+from sqlalchemy.orm import Session as SASession
+
+from .embeddings import create_embedding, Embedder
+from .feedback import get_feedback, Reviewer
+from .models import Chunk, Document, Feedback, Group, Note, Reference, User
+from .utils import chunk_text, log_activity
+
+
+def process_document(
+    user: User,
+    session: SASession,
+    embedder: Embedder,
+    reviewer: Reviewer,
+    doc: Document,
+    generate_feedback: bool = True,
+) -> Mapping[str, int]:
+    new = False
+
+    prev_content = get_history(doc, "content").deleted
+    prev_chunks: list[str] = []
+    if prev_content:
+        prev_chunks = chunk_text(prev_content[-1])
+
+    feedback_limit = int(os.getenv("COMPAIR_CORE_FEEDBACK_LIMIT", "5"))
+    time_cutoff = datetime.now(timezone.utc) - timedelta(hours=24)
+
+    recent_feedback_count = session.query(Feedback).filter(
+        Feedback.source_chunk_id.in_(
+            session.query(Chunk.chunk_id).filter(Chunk.document_id == doc.document_id)
+        ),
+        Feedback.timestamp >= time_cutoff,
+    ).count()
+
+    content = doc.content
+    chunks = chunk_text(content)
+    new_chunks = list(set(chunks) - set(prev_chunks))
+
+    prioritized_chunk_indices: list[int] = []
+    if generate_feedback:
+        prioritized_chunk_indices = detect_significant_edits(prev_chunks=prev_chunks, new_chunks=new_chunks)
+
+    num_chunks_can_generate_feedback = max((feedback_limit - recent_feedback_count), 0)
+    indices_to_generate_feedback = prioritized_chunk_indices[:num_chunks_can_generate_feedback]
+
+    for i, chunk in enumerate(new_chunks):
+        should_generate_feedback = i in indices_to_generate_feedback
+        process_text(
+            session=session,
+            embedder=embedder,
+            reviewer=reviewer,
+            doc=doc,
+            text=chunk,
+            generate_feedback=should_generate_feedback,
+        )
+
+    remove_chunks = set(prev_chunks) - set(chunks)
+    for chunk in remove_chunks:
+        remove_text(session=session, text=chunk, document_id=doc.document_id)
+
+    if doc.groups:
+        log_activity(
+            session=session,
+            user_id=doc.author_id,
+            group_id=doc.groups[0].group_id,
+            action="update",
+            object_id=doc.document_id,
+            object_name=doc.title,
+            object_type="document",
+        )
+
+    session.commit()
+    return {"new": new}
+
+
+def detect_significant_edits(
+    prev_chunks: list[str],
+    new_chunks: list[str],
+    threshold: float = 0.5,
+) -> list[int]:
+    prev_set = set(prev_chunks)
+    new_set = set(new_chunks)
+
+    unchanged_chunks = prev_set & new_set
+    new_chunks_to_check = new_set - unchanged_chunks
+
+    process_chunks: list[str] = []
+    for new_chunk in new_chunks_to_check:
+        best_match = max((Levenshtein.ratio(new_chunk, prev_chunk) for prev_chunk in prev_chunks), default=0.0)
+        if best_match < threshold:
+            process_chunks.append(new_chunk)
+
+    prioritized_chunks = prioritize_chunks(process_chunks)
+    return [new_chunks.index(new_chunk) for new_chunk in prioritized_chunks if new_chunk in new_chunks]
+
+
+def prioritize_chunks(chunks: list[str], limit: int | None = None) -> list[str]:
+    limit = limit or len(chunks)
+    indexed_chunks = [(chunk, idx) for idx, chunk in enumerate(chunks)]
+    indexed_chunks.sort(key=lambda x: (-len(x[0]), x[1]))
+    return [chunk for chunk, _ in indexed_chunks[:limit]]
+
+
+def process_text(
+    session: SASession,
+    embedder: Embedder,
+    reviewer: Reviewer,
+    doc: Document,
+    text: str,
+    generate_feedback: bool = True,
+    note: Note | None = None,
+) -> None:
+    logger = logging.getLogger(__name__)
+    chunk_hash = hash(text)
+
+    chunk_type = "note" if note else "document"
+    note_id = note.note_id if note else None
+
+    existing_chunks = session.query(Chunk).filter(
+        Chunk.hash == str(chunk_hash),
+        Chunk.document_id == doc.document_id,
+        Chunk.chunk_type == chunk_type,
+        Chunk.note_id == note_id,
+    )
+
+    user = session.query(User).filter(User.user_id == doc.author_id).first()
+    if existing_chunks.first():
+        for chunk in existing_chunks:
+            if chunk.embedding is None:
+                embedding = create_embedding(embedder, text, user=user)
+                existing_chunks.update({"embedding": embedding})
+        session.commit()
+    else:
+        chunk = Chunk(
+            hash=chunk_hash,
+            document_id=doc.document_id,
+            note_id=note_id,
+            chunk_type=chunk_type,
+            content=text,
+        )
+        embedding = create_embedding(embedder, text, user=user)
+        chunk.embedding = embedding
+        session.add(chunk)
+        session.commit()
+        existing_chunk = chunk
+    existing_chunk = session.query(Chunk).filter(
+        Chunk.document_id == doc.document_id,
+        Chunk.hash == str(chunk_hash),
+        Chunk.chunk_type == chunk_type,
+        Chunk.note_id == note_id,
+    ).first()
+
+    if generate_feedback and existing_chunk:
+        doc_group_ids = [g.group_id for g in doc.groups]
+        references = (
+            session.query(Chunk)
+            .join(Chunk.document)
+            .join(Document.groups)
+            .filter(
+                Document.is_published.is_(True),
+                Document.document_id != doc.document_id,
+                Chunk.chunk_type == "document",
+                Group.group_id.in_(doc_group_ids),
+            )
+            .order_by(Chunk.embedding.cosine_distance(existing_chunk.embedding))
+            .limit(3)
+            .all()
+        )
+
+        sql_references: list[Reference] = []
+        for ref_chunk in references:
+            sql_references.append(
+                Reference(
+                    source_chunk_id=existing_chunk.chunk_id,
+                    reference_type="document",
+                    reference_document_id=ref_chunk.document_id,
+                    reference_note_id=None,
+                )
+            )
+
+        if sql_references:
+            session.add_all(sql_references)
+            session.commit()
+
+        feedback = get_feedback(reviewer, doc, text, references, user)
+        if feedback != "NONE":
+            sql_feedback = Feedback(
+                source_chunk_id=existing_chunk.chunk_id,
+                feedback=feedback,
+                model=reviewer.model,
+            )
+            session.add(sql_feedback)
+            session.commit()
+
+
+def remove_text(session: SASession, text: str, document_id: str) -> None:
+    chunks = session.query(Chunk).filter(
+        Chunk.document_id == document_id,
+        Chunk.content == text,
+    )
+    chunks.delete(synchronize_session=False)
+    session.commit()
+
+
+def get_all_chunks_for_document(session: SASession, doc: Document) -> list[Chunk]:
+    doc_chunks = session.query(Chunk).filter(Chunk.document_id == doc.document_id).all()
+    note_chunks: list[Chunk] = []
+    notes = session.query(Note).filter(Note.document_id == doc.document_id).all()
+    for note in notes:
+        note_text_chunks = chunk_text(note.content)
+        for text in note_text_chunks:
+            chunk_hash = hash(text)
+            existing = session.query(Chunk).filter(
+                Chunk.hash == str(chunk_hash),
+                Chunk.document_id == doc.document_id,
+                Chunk.content == text,
+            ).first()
+            if not existing:
+                embedding = create_embedding(Embedder(), text, user=doc.author_id)
+                note_chunk = Chunk(
+                    hash=str(chunk_hash),
+                    document_id=doc.document_id,
+                    content=text,
+                    embedding=embedding,
+                )
+                session.add(note_chunk)
+                session.commit()
+                note_chunks.append(note_chunk)
+            else:
+                note_chunks.append(existing)
+    return doc_chunks + note_chunks