welearn-database 1.4.5.dev1__tar.gz → 1.5.2__tar.gz

{welearn_database-1.4.5.dev1 → welearn_database-1.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: welearn-database
-Version: 1.4.5.dev1
+Version: 1.5.2
 Summary: All stuff related to relationnal database from the WeLearn project
 License: cc-by-sa-nc
 Author: Théo

{welearn_database-1.4.5.dev1 → welearn_database-1.5.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "welearn-database"
-version = "1.4.5.dev1"
+version = "1.5.2"
 description = "All stuff related to relationnal database from the WeLearn project"
 authors = [
     {name = "Théo",email = "theo.nardin@cri-paris.org"}

welearn_database-1.5.2/welearn_database/alembic/versions/5aad97149044_add_histo_table_for_materialized_views.py

@@ -0,0 +1,71 @@
+"""add_histo_table_for_materialized_views
+
+Revision ID: 5aad97149044
+Revises: b84462ca3800
+Create Date: 2026-06-17 11:31:55.292978
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import text
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "5aad97149044"
+down_revision: Union[str, None] = "b84462ca3800"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "historical_qty_document_per_corpus",
+        sa.Column(
+            "id", sa.Uuid(), server_default=text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("source_name", sa.String(), nullable=False),
+        sa.Column("count", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), server_default="NOW()", nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="document_related",
+    )
+
+    op.create_table(
+        "historical_qty_document_in_qdrant_per_corpus",
+        sa.Column(
+            "id", sa.Uuid(), server_default=text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("source_name", sa.String(), nullable=False),
+        sa.Column("count", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), server_default="NOW()", nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="document_related",
+    )
+
+    op.create_table(
+        "historical_qty_document_in_qdrant",
+        sa.Column(
+            "id", sa.Uuid(), server_default=text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("count", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), server_default="NOW()", nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="document_related",
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("historical_qty_document_per_corpus", schema="document_related")
+    op.drop_table(
+        "historical_qty_document_in_qdrant_per_corpus", schema="document_related"
+    )
+    op.drop_table("historical_qty_document_in_qdrant", schema="document_related")

welearn_database-1.5.2/welearn_database/alembic/versions/6d4346fad6f4_view_used_feature_per_session.py

@@ -0,0 +1,88 @@
+"""view_used_feature_per_session
+
+Revision ID: 6d4346fad6f4
+Revises: 5aad97149044
+Create Date: 2026-06-17 11:54:52.441998
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "6d4346fad6f4"
+down_revision: Union[str, None] = "5aad97149044"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("""
+        CREATE OR REPLACE VIEW grafana.used_feature_per_session
+        AS WITH matching_features_endpoint AS (
+SELECT
+	endpoint_name,
+	feature_name
+FROM
+	(
+VALUES
+	('/api/v1/search/by_document', 'search'),
+	('/api/v1/qna/chat/answer', 'chat'),
+	('/api/v1/qna/chat/agent', 'chat'),
+	('/api/v1/tutor/syllabus', 'syllabus'),
+	('/api/v1/user/bookmarks/' || ':' || 'document_id', 'bookmark'),
+	('/api/v1/user/' || ':' || 'user_id' || '/bookmarks/' || ':' || 'document_id', 'bookmark')
+	) AS t(endpoint_name, feature_name)
+),
+session_feature_pair AS (
+SELECT
+	DISTINCT
+	er.session_id,
+	mfe.feature_name
+FROM
+	user_related.endpoint_request er
+CROSS JOIN matching_features_endpoint mfe
+ORDER BY
+	er.session_id
+),
+actual_count AS (
+SELECT
+	er.session_id,
+	mfe.feature_name,
+	COUNT(1) AS cnt
+FROM
+	user_related.endpoint_request er
+INNER JOIN
+	user_related."session" s ON
+	s.id = er.session_id
+INNER JOIN
+	matching_features_endpoint mfe ON
+	mfe.endpoint_name = er.endpoint_name
+GROUP BY
+	er.session_id,
+	mfe.feature_name 
+)
+SELECT 
+	s.inferred_user_id,
+	sfp.session_id,
+	sfp.feature_name,
+	COALESCE(ac.cnt, 0) AS cnt,
+	COALESCE(ac.cnt, 0) > 0 AS is_feature_used,
+	s.created_at AS session_created_at
+FROM
+	session_feature_pair sfp
+LEFT JOIN actual_count ac ON
+	ac.feature_name = sfp.feature_name
+	AND ac.session_id = sfp.session_id
+INNER JOIN user_related."session" s ON
+	s.id = sfp.session_id
+ORDER BY
+	session_created_at
+        """)
+
+
+def downgrade() -> None:
+    op.execute("""
+    DROP VIEW IF EXISTS grafana.used_feature_per_session;
+    """)

welearn_database-1.5.2/welearn_database/alembic/versions/95ffe7afa64c_add_index_idx_welearn_document_keyword_.py

@@ -0,0 +1,35 @@
+"""add_index_idx_welearn_document_keyword_document_id
+
+Revision ID: 95ffe7afa64c
+Revises: f5e91a5c950d
+Create Date: 2026-09-16 15:44:06.977345
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "95ffe7afa64c"
+down_revision: Union[str, None] = "f5e91a5c950d"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_index(
+        "idx_welearn_document_keyword_document_id",
+        "welearn_document_keyword",
+        ["welearn_document_id"],
+        schema="document_related",
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "idx_welearn_document_keyword_document_id",
+        table_name="document_keyword",
+        schema="document_related",
+    )

welearn_database-1.5.2/welearn_database/alembic/versions/b84462ca3800_update_grafana_process_state_view.py

@@ -0,0 +1,45 @@
+"""update grafana process state view
+
+Revision ID: b84462ca3800
+Revises: 8b780aea403a
+Create Date: 2026-06-04 16:56:18.693751
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b84462ca3800"
+down_revision: Union[str, None] = "8b780aea403a"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("""
+        CREATE OR REPLACE VIEW grafana.process_state
+        AS SELECT process_state.id,
+        process_state.document_id,
+        process_state.title,
+        process_state.created_at,
+        process_state.operation_order,
+        corpus.source_name
+           FROM document_related.process_state
+             JOIN document_related.welearn_document ON welearn_document.id = process_state.document_id
+             JOIN corpus_related.corpus ON welearn_document.corpus_id = corpus.id;
+        """)
+
+
+def downgrade() -> None:
+    op.execute("""
+        CREATE OR REPLACE VIEW grafana.process_state
+        AS SELECT process_state.id,
+        process_state.document_id,
+        process_state.title,
+        process_state.created_at,
+        process_state.operation_order,
+           FROM document_related.process_state;
+        """)

welearn_database-1.5.2/welearn_database/alembic/versions/f5e91a5c950d_add_analytic_form.py

@@ -0,0 +1,67 @@
+"""add analytic form
+
+Revision ID: f5e91a5c950d
+Revises: 6d4346fad6f4
+Create Date: 2026-09-02 16:32:55.188400
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "f5e91a5c950d"
+down_revision: Union[str, None] = "6d4346fad6f4"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "analytic_form",
+        sa.Column(
+            "id", sa.Uuid(), server_default=sa.func.gen_random_uuid(), nullable=False
+        ),
+        sa.Column("form_name", sa.String(), nullable=False),
+        sa.Column("question", sa.String(), nullable=False),
+        sa.Column("answer", sa.String(), nullable=False),
+        sa.Column(
+            "answer_type",
+            postgresql.ENUM(
+                "text",
+                "checkbox",
+                "predefined_text",
+                name="answer_type",
+                schema="user_related",
+            ),
+            nullable=False,
+        ),
+        sa.Column("session_id", sa.Uuid(), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=False),
+            nullable=False,
+            server_default="NOW()",
+        ),
+        sa.ForeignKeyConstraint(
+            ["session_id"],
+            ["user_related.session.id"],
+            name="analytic_form_session_id_fkey",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="user_related",
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "analytic_form_session_id_fkey",
+        "analytic_form",
+        schema="user_related",
+        type_="foreignkey",
+    )
+    op.drop_table("analytic_form", schema="user_related")
+    op.execute("DROP TYPE IF EXISTS user_related.answer_type")

welearn_database-1.5.2/welearn_database/data/details_dict.py

@@ -0,0 +1,30 @@
+from dataclasses import asdict, is_dataclass
+
+from sqlalchemy import types
+
+
+class DetailsDict(types.TypeDecorator):
+    """
+    Convert dataclass instances into dictionaries for JSON storage.
+    """
+
+    impl = types.JSON
+    cache_ok = True
+
+    @staticmethod
+    def _is_dataclass_instance(obj):
+        return is_dataclass(obj) and not isinstance(obj, type)
+
+    def _inner_serialize_dataclass(self, value):
+        match value:
+            case list():
+                return [self._inner_serialize_dataclass(item) for item in value]
+            case dict():
+                return {k: self._inner_serialize_dataclass(v) for k, v in value.items()}
+        if self._is_dataclass_instance(value):
+            return asdict(value)
+        return value
+
+    def process_bind_param(self, value, dialect):
+        # Serialize recursively without mutating the original object stored on the ORM instance.
+        return self._inner_serialize_dataclass(value)

{welearn_database-1.4.5.dev1 → welearn_database-1.5.2}/welearn_database/data/enumeration.py

@@ -43,3 +43,9 @@ class ExternalIdType(StrEnum):
 class FilterType(StrEnum):
     SDG = auto()
     SOURCE = auto()
+
+
+class QuestionType(StrEnum):
+    TEXT = auto()
+    CHECKBOX = auto()
+    PREDEFINED_TEXT = auto()

{welearn_database-1.4.5.dev1 → welearn_database-1.5.2}/welearn_database/data/models/document_related.py

@@ -16,6 +16,7 @@ from sqlalchemy import (
 from sqlalchemy.dialects.postgresql import ARRAY, ENUM, TIMESTAMP
 from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
 
+from welearn_database.data.details_dict import DetailsDict
 from welearn_database.data.enumeration import (
     ContextType,
     Counter,
@@ -44,7 +45,7 @@ class WeLearnDocument(Base):
     """
     This class represents a document in the WeLearn system.
     :cvar id: The unique identifier of the document.
-    :cvar doi: The unique DOI identifier if it exist for this document
+    :cvar doi: The unique DOI identifier if it exists for this document
     :cvar external_id: ID use by the document provider for identify it (ex: PubMed ID, ArXiv ID, OAI PMH identifier etc.)
     :cvar external_id_type: The type of the external ID, represented as an enumeration
     :cvar url: The URL of the document.
@@ -83,7 +84,7 @@ class WeLearnDocument(Base):
     lang: Mapped[str | None]
     description: Mapped[str | None]
     full_content: Mapped[str | None]
-    details: Mapped[dict[str, Any] | None]
+    details: Mapped[dict[str, Any] | None] = mapped_column(DetailsDict)
     trace: Mapped[int | None] = mapped_column(types.BIGINT)
     corpus_id: Mapped[UUID] = mapped_column(
         types.Uuid,
@@ -478,6 +479,65 @@ class ContextDocument(Base):
     )
 
 
+class HistoricalQtyDocumentPerCorpus(Base):
+    __tablename__ = "historical_qty_document_per_corpus"
+    __table_args__ = {"schema": schema_name}
+
+    id = mapped_column(
+        types.Uuid,
+        primary_key=True,
+        server_default=GEN_RANDOM_UUID,
+        nullable=False,
+    )
+    source_name: Mapped[str] = mapped_column()
+    count: Mapped[int]
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default=NOW,
+    )
+
+
+class HistoricalQtyDocumentInQdrantPerCorpus(Base):
+    __tablename__ = "historical_qty_document_in_qdrant_per_corpus"
+    __table_args__ = {"schema": schema_name}
+
+    id = mapped_column(
+        types.Uuid,
+        primary_key=True,
+        server_default=GEN_RANDOM_UUID,
+        nullable=False,
+    )
+    source_name: Mapped[str] = mapped_column()
+    count: Mapped[int]
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default=NOW,
+    )
+
+
+class HistoricalQtyDocumentInQdrant(Base):
+    __tablename__ = "historical_qty_document_in_qdrant"
+    __table_args__ = {"schema": schema_name}
+
+    id = mapped_column(
+        types.Uuid,
+        primary_key=True,
+        server_default=GEN_RANDOM_UUID,
+        nullable=False,
+    )
+    count: Mapped[int]
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default=NOW,
+    )
+
+
 # Views
 class QtyDocumentInQdrant(Base):
     __tablename__ = "qty_document_in_qdrant"

{welearn_database-1.4.5.dev1 → welearn_database-1.5.2}/welearn_database/data/models/grafana.py

@@ -136,3 +136,18 @@ class Session(Base):
     created_at: Mapped[datetime] = mapped_column()
     end_at: Mapped[datetime | None] = mapped_column(nullable=True)
     host: Mapped[str | None] = mapped_column(nullable=True)
+
+
+class UsedFeaturePerSession(Base):
+    __tablename__ = "used_feature_per_session"
+    __table_args__ = {"schema": schema_name}
+    __read_only__ = True
+
+    inferred_user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), primary_key=True
+    )
+    session_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
+    feature_name: Mapped[str] = mapped_column(primary_key=True)
+    cnt: Mapped[int] = mapped_column()
+    is_feature_used: Mapped[bool] = mapped_column()
+    session_created_at: Mapped[datetime] = mapped_column()

{welearn_database-1.4.5.dev1 → welearn_database-1.5.2}/welearn_database/data/models/user_related.py

@@ -5,7 +5,7 @@ from sqlalchemy import ForeignKey, func, types
 from sqlalchemy.dialects.postgresql import ENUM, TIMESTAMP
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
-from welearn_database.data.enumeration import DbSchemaEnum, FilterType
+from welearn_database.data.enumeration import DbSchemaEnum, FilterType, QuestionType
 from welearn_database.data.models.document_related import WeLearnDocument
 
 from . import Base
@@ -271,3 +271,35 @@ class FilterUsedInQuery(Base):
     filter_value: Mapped[str]
 
     chat_message: Mapped["ChatMessage"] = relationship()
+
+
+class AnalyticForm(Base):
+    __tablename__ = "analytic_form"
+    __table_args__ = {"schema": DbSchemaEnum.USER_RELATED.value}
+
+    id: Mapped[UUID] = mapped_column(
+        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+    )
+    form_name: Mapped[str] = mapped_column(nullable=False)
+    question: Mapped[str] = mapped_column(nullable=False)
+    answer: Mapped[str] = mapped_column(nullable=False)
+    answer_type: Mapped[str] = mapped_column(
+        ENUM(
+            *(e.value.lower() for e in QuestionType),
+            name="answer_type",
+            schema=DbSchemaEnum.USER_RELATED.value,
+        ),
+        nullable=False,
+    )
+    session_id: Mapped[UUID] = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.USER_RELATED.value}.session.id"),
+        nullable=False,
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default="NOW()",
+    )
+    session = relationship("Session", foreign_keys=[session_id])