welearn-database 0.2.1__tar.gz → 0.2.3__tar.gz

{welearn_database-0.2.1 → welearn_database-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: welearn-database
-Version: 0.2.1
+Version: 0.2.3
 Summary: All stuff related to relationnal database from the WeLearn project
 License: cc-by-sa-nc
 Author: Théo

{welearn_database-0.2.1 → welearn_database-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "welearn-database"
-version = "0.2.1"
+version = "0.2.3"
 description = "All stuff related to relationnal database from the WeLearn project"
 authors = [
     {name = "Théo",email = "theo.nardin@cri-paris.org"}

{welearn_database-0.2.1 → welearn_database-0.2.3}/welearn_database/data/enumeration.py

@@ -18,6 +18,12 @@ class Counter(Enum):
 
 class DbSchemaEnum(StrEnum):
     GRAFANA = auto()
+    AGENT_RELATED = auto()
     CORPUS_RELATED = auto()
     DOCUMENT_RELATED = auto()
-    USER_RELATED = auto()
+    USER_RELATED = auto()
+
+class ContextType(StrEnum):
+    INTRODUCTION = auto()
+    TARGET = auto()
+    SUBJECT = auto()

{welearn_database-0.2.1 → welearn_database-0.2.3}/welearn_database/data/models/__init__.py

@@ -2,10 +2,16 @@ from datetime import datetime
 from typing import Any
 
 from sqlalchemy import types
-from sqlalchemy.dialects.postgresql import TIMESTAMP
+from sqlalchemy.dialects.postgresql import ARRAY, TIMESTAMP
+from sqlalchemy.ext.compiler import compiles
 from sqlalchemy.orm import DeclarativeBase
 
 
+@compiles(ARRAY, "sqlite")
+def compile_binary_sqlite(type_, compiler, **kw):
+    return "TEXT"
+
+
 class Base(DeclarativeBase):
     type_annotation_map = {
         dict[str, Any]: types.JSON,
@@ -18,12 +24,17 @@ class Base(DeclarativeBase):
         Raise an exception if attempting to assign to an atribute of a "read-only" object
         Transient attributes need to be prefixed with "_t_"
         """
-        if (getattr(self, '__read_only__', False)
-          and name != "_sa_instance_state"
-          and not name.startswith("_t_")):
-            raise ValueError("Trying to assign to %s of a read-only object %s" % (name, self))
+        if (
+            getattr(self, "__read_only__", False)
+            and name != "_sa_instance_state"
+            and not name.startswith("_t_")
+        ):
+            raise ValueError(
+                "Trying to assign to %s of a read-only object %s" % (name, self)
+            )
         super(Base, self).__setattr__(name, value)
 
-from .document_related import *
+
 from .corpus_related import *
-from .user_related import *
+from .document_related import *
+from .user_related import *

welearn_database-0.2.3/welearn_database/data/models/agent_related.py

@@ -0,0 +1,78 @@
+from typing import Any, Dict, Optional
+
+from sqlalchemy import Index, Text
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy import text
+from sqlalchemy.types import LargeBinary, Integer
+
+from welearn_database.data.enumeration import DbSchemaEnum
+from welearn_database.data.models import Base
+
+
+schema_name = DbSchemaEnum.AGENT_RELATED.value
+
+
+class CheckpointBlobs(Base):
+    __tablename__ = "checkpoint_blobs"
+    __table_args__ = (
+        Index("checkpoint_blobs_thread_id_idx", "thread_id"),
+        {"schema": schema_name},
+    )
+
+    thread_id: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False)
+    checkpoint_ns: Mapped[str] = mapped_column(
+        Text, server_default=text("''::text"), primary_key=True, nullable=False
+    )
+    channel: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False)
+    version_: Mapped[str] = mapped_column("version", Text, primary_key=True, nullable=False)
+    type_: Mapped[str] = mapped_column("type", Text, nullable=False)
+    blob: Mapped[Optional[bytes]] = mapped_column(LargeBinary, nullable=True)
+
+
+class CheckpointMigrations(Base):
+    __tablename__ = "checkpoint_migrations"
+    __table_args__ = {"schema": schema_name},
+
+    v: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+
+
+class CheckpointWrites(Base):
+    __tablename__ = "checkpoint_writes"
+    __table_args__ = (
+        Index("checkpoint_writes_thread_id_idx", "thread_id"),
+        {"schema": schema_name},
+    )
+
+    thread_id: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False)
+    checkpoint_ns: Mapped[str] = mapped_column(
+        Text, server_default=text("''::text"), primary_key=True, nullable=False
+    )
+    checkpoint_id: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False)
+    task_id: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False)
+    idx: Mapped[int] = mapped_column(Integer, primary_key=True, nullable=False)
+
+    channel: Mapped[str] = mapped_column(Text, nullable=False)
+    type_: Mapped[Optional[str]] = mapped_column("type", Text, nullable=True)
+    blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
+    task_path: Mapped[str] = mapped_column(
+        Text, server_default=text("''::text"), nullable=False
+    )
+
+
+class Checkpoints(Base):
+    __tablename__ = "checkpoints"
+    __table_args__ = {"schema": schema_name},
+
+    thread_id: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False)
+    checkpoint_ns: Mapped[str] = mapped_column(
+        Text, server_default=text("''::text"), primary_key=True, nullable=False
+    )
+    checkpoint_id: Mapped[str] = mapped_column(Text, primary_key=True, nullable=False)
+
+    parent_checkpoint_id: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
+    type_: Mapped[Optional[str]] = mapped_column("type", Text, nullable=True)
+    checkpoint: Mapped[Dict[str, Any]] = mapped_column(JSONB, nullable=False)
+    metadata_: Mapped[Dict[str, Any]] = mapped_column("metadata",
+        JSONB, server_default=text("'{}'::jsonb"), nullable=False
+    )

{welearn_database-0.2.1 → welearn_database-0.2.3}/welearn_database/data/models/document_related.py

@@ -4,16 +4,21 @@ from urllib.parse import urlparse
 from uuid import UUID
 from zlib import adler32
 
-from sqlalchemy import types, ForeignKey, UniqueConstraint, func, LargeBinary
-from sqlalchemy.dialects.postgresql import TIMESTAMP, ENUM
+from sqlalchemy import ForeignKey, Integer, LargeBinary, UniqueConstraint, func, types
+from sqlalchemy.dialects.postgresql import ARRAY, ENUM, TIMESTAMP
 from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy.orm import mapped_column, Mapped, relationship, validates
-
-from welearn_database.modules.text_cleaning import clean_text
-from welearn_database.data.enumeration import DbSchemaEnum, Step, Counter
-from welearn_database.data.models.corpus_related import Corpus, NClassifierModel, BiClassifierModel, EmbeddingModel
-from . import Base
+from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
+
+from welearn_database.data.enumeration import ContextType, Counter, DbSchemaEnum, Step
+from welearn_database.data.models import Base
+from welearn_database.data.models.corpus_related import (
+    BiClassifierModel,
+    Corpus,
+    EmbeddingModel,
+    NClassifierModel,
+)
 from welearn_database.exceptions import InvalidURLScheme
+from welearn_database.modules.text_cleaning import clean_text
 
 schema_name = DbSchemaEnum.DOCUMENT_RELATED.value
 
@@ -34,6 +39,7 @@ class WeLearnDocument(Base):
     :cvar updated_at: The timestamp when the document was last updated.
     :cvar corpus: The relationship to the Corpus object.
     """
+
     __tablename__ = "welearn_document"
     __table_args__ = (
         UniqueConstraint("url", name="welearn_document_url_key"),
@@ -46,8 +52,8 @@ class WeLearnDocument(Base):
     url: Mapped[str] = mapped_column(nullable=False)
     title: Mapped[str | None]
     lang: Mapped[str | None]
-    _description: Mapped[str]
-    _full_content: Mapped[str]
+    _description: Mapped[str | None]
+    _full_content: Mapped[str | None]
     details: Mapped[dict[str, Any] | None]
     _trace: Mapped[int | None] = mapped_column(types.BIGINT)
     corpus_id: Mapped[UUID] = mapped_column(
@@ -83,9 +89,7 @@ class WeLearnDocument(Base):
         parsed_url = urlparse(url=value)
         accepted_scheme = ["https"]
         if parsed_url.scheme not in accepted_scheme or len(parsed_url.netloc) == 0:
-            raise InvalidURLScheme(
-                "There is an error on the URL form : %s", value
-            )
+            raise InvalidURLScheme("There is an error on the URL form : %s", value)
         return value
 
     @validates("_full_content")
@@ -98,7 +102,7 @@ class WeLearnDocument(Base):
         :raises ValueError: If the full content is too short.
         """
         if not value:
-            raise ValueError("Full content cannot be empty or None")
+            return value
         if len(value) < 25:
             raise ValueError(f"Content is too short : {len(value)}")
         return value
@@ -118,7 +122,7 @@ class WeLearnDocument(Base):
     @description.setter
     def description(self, description):
         if not description:
-            raise ValueError("Description cannot be empty or None")
+            self._description = description
         self._description = clean_text(description)
 
     @hybrid_property
@@ -136,6 +140,7 @@ class ProcessState(Base):
     :cvar operation_order: A bigint representing the order of operations for the process state.
     :cvar document: The relationship to the WeLearnDocument object.
     """
+
     __tablename__ = "process_state"
     __table_args__ = {"schema": schema_name}
 
@@ -173,7 +178,7 @@ class Keyword(Base):
     __tablename__ = "keyword"
     __table_args__ = (
         UniqueConstraint("keyword", name="keyword_unique"),
-        {"schema": schema_name}
+        {"schema": schema_name},
     )
 
     id: Mapped[UUID] = mapped_column(
@@ -314,7 +319,6 @@ class AnalyticCounter(Base):
     document: Mapped["WeLearnDocument"] = relationship()
 
 
-
 class CorpusEmbeddingModel(Base):
     __tablename__ = "corpus_embedding_model"
     __table_args__ = (
@@ -451,6 +455,55 @@ class Sdg(Base):
     n_classifier_model: Mapped["NClassifierModel"] = relationship()
     slice: Mapped["DocumentSlice"] = relationship()
 
+
+class ContextDocument(Base):
+    __tablename__ = "context_document"
+
+    id = mapped_column(
+        types.Uuid,
+        primary_key=True,
+        server_default="gen_random_uuid()",
+        nullable=False,
+    )
+    url: Mapped[str]
+    title: Mapped[str]
+    full_content: Mapped[str]
+    embedding_model_id = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.embedding_model.id"),
+    )
+    sdg_related: Mapped[list[int]] = mapped_column(ARRAY(Integer))
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default="NOW()",
+    )
+    embedding: Mapped[bytes] = mapped_column(LargeBinary)
+
+    updated_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        onupdate=func.localtimestamp(),
+    )
+
+    context_type: Mapped[str] = mapped_column(
+        ENUM(
+            *(e.value.lower() for e in ContextType),
+            name="context_type",
+            schema="document_related",
+        ),
+        nullable=False,
+    )
+    embedding_model = relationship("EmbeddingModel", foreign_keys=[embedding_model_id])
+
+    __table_args__ = (
+        UniqueConstraint("url", name="meta_document_url_key"),
+        {"schema": "document_related"},
+    )
+
+
 # Views
 class QtyDocumentInQdrant(Base):
     __tablename__ = "qty_document_in_qdrant"