welearn-database 1.4.5.dev0__tar.gz → 1.5.1__tar.gz

{welearn_database-1.4.5.dev0 → welearn_database-1.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: welearn-database
-Version: 1.4.5.dev0
+Version: 1.5.1
 Summary: All stuff related to relationnal database from the WeLearn project
 License: cc-by-sa-nc
 Author: Théo

{welearn_database-1.4.5.dev0 → welearn_database-1.5.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "welearn-database"
-version = "1.4.5.dev0"
+version = "1.5.1"
 description = "All stuff related to relationnal database from the WeLearn project"
 authors = [
     {name = "Théo",email = "theo.nardin@cri-paris.org"}

welearn_database-1.5.1/welearn_database/alembic/versions/5aad97149044_add_histo_table_for_materialized_views.py

@@ -0,0 +1,71 @@
+"""add_histo_table_for_materialized_views
+
+Revision ID: 5aad97149044
+Revises: b84462ca3800
+Create Date: 2026-06-17 11:31:55.292978
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import text
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "5aad97149044"
+down_revision: Union[str, None] = "b84462ca3800"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "historical_qty_document_per_corpus",
+        sa.Column(
+            "id", sa.Uuid(), server_default=text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("source_name", sa.String(), nullable=False),
+        sa.Column("quantity", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), server_default="NOW()", nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="document_related",
+    )
+
+    op.create_table(
+        "historical_qty_document_in_qdrant_per_corpus",
+        sa.Column(
+            "id", sa.Uuid(), server_default=text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("source_name", sa.String(), nullable=False),
+        sa.Column("quantity", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), server_default="NOW()", nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="document_related",
+    )
+
+    op.create_table(
+        "historical_qty_document_in_qdrant",
+        sa.Column(
+            "id", sa.Uuid(), server_default=text("gen_random_uuid()"), nullable=False
+        ),
+        sa.Column("quantity", sa.Integer(), nullable=False),
+        sa.Column(
+            "created_at", postgresql.TIMESTAMP(), server_default="NOW()", nullable=False
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        schema="document_related",
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("historical_qty_document_per_corpus", schema="document_related")
+    op.drop_table(
+        "historical_qty_document_in_qdrant_per_corpus", schema="document_related"
+    )
+    op.drop_table("historical_qty_document_in_qdrant", schema="document_related")

welearn_database-1.5.1/welearn_database/alembic/versions/6d4346fad6f4_view_used_feature_per_session.py

@@ -0,0 +1,88 @@
+"""view_used_feature_per_session
+
+Revision ID: 6d4346fad6f4
+Revises: 5aad97149044
+Create Date: 2026-06-17 11:54:52.441998
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "6d4346fad6f4"
+down_revision: Union[str, None] = "5aad97149044"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("""
+        CREATE OR REPLACE VIEW grafana.used_feature_per_session
+        AS WITH matching_features_endpoint AS (
+SELECT
+	endpoint_name,
+	feature_name
+FROM
+	(
+VALUES
+	('/api/v1/search/by_document', 'search'),
+	('/api/v1/qna/chat/answer', 'chat'),
+	('/api/v1/qna/chat/agent', 'chat'),
+	('/api/v1/tutor/syllabus', 'syllabus'),
+	('/api/v1/user/bookmarks/:document_id', 'bookmark'),
+	('/api/v1/user/:user_id/bookmarks/:document_id', 'bookmark')
+	) AS t(endpoint_name, feature_name)
+),
+session_feature_pair AS (
+SELECT
+	DISTINCT
+	er.session_id,
+	mfe.feature_name
+FROM
+	user_related.endpoint_request er
+CROSS JOIN matching_features_endpoint mfe
+ORDER BY
+	er.session_id
+),
+actual_count AS (
+SELECT
+	er.session_id,
+	mfe.feature_name,
+	COUNT(1) AS cnt
+FROM
+	user_related.endpoint_request er
+INNER JOIN
+	user_related."session" s ON
+	s.id = er.session_id
+INNER JOIN
+	matching_features_endpoint mfe ON
+	mfe.endpoint_name = er.endpoint_name
+GROUP BY
+	er.session_id,
+	mfe.feature_name 
+)
+SELECT 
+	s.inferred_user_id,
+	sfp.session_id,
+	sfp.feature_name,
+	COALESCE(ac.cnt, 0) AS cnt,
+	COALESCE(ac.cnt, 0) > 0 AS is_feature_used,
+	s.created_at AS session_created_at
+FROM
+	session_feature_pair sfp
+LEFT JOIN actual_count ac ON
+	ac.feature_name = sfp.feature_name
+	AND ac.session_id = sfp.session_id
+INNER JOIN user_related."session" s ON
+	s.id = sfp.session_id
+ORDER BY
+	session_created_at
+        """)
+
+
+def downgrade() -> None:
+    op.execute("""
+    DROP VIEW IF EXISTS grafana.used_feature_per_session;
+    """)

{welearn_database-1.4.5.dev0 → welearn_database-1.5.1}/welearn_database/alembic/versions/8b780aea403a_add_university_data.py

@@ -20,6 +20,9 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
+    # op.execute("""
+    # CREATE TYPE user_related.university_role AS ENUM ('student', 'teacher', 'staff')
+    # """)
     op.add_column(
         "inferred_user",
         sa.Column("university_title", sa.String(), nullable=True),
@@ -29,13 +32,7 @@ def upgrade() -> None:
         "inferred_user",
         sa.Column(
             "role",
-            postgresql.ENUM(
-                "student",
-                "teacher",
-                "staff",
-                name="university_role",
-                schema="user_related",
-            ),
+            sa.String(),
             nullable=True,
         ),
         schema="user_related",
@@ -45,4 +42,3 @@ def upgrade() -> None:
 def downgrade() -> None:
     op.drop_column("inferred_user", "university_title", schema="user_related")
     op.drop_column("inferred_user", "role", schema="user_related")
-    op.execute("DROP TYPE user_related.university_role")

welearn_database-1.5.1/welearn_database/alembic/versions/b84462ca3800_update_grafana_process_state_view.py

@@ -0,0 +1,45 @@
+"""update grafana process state view
+
+Revision ID: b84462ca3800
+Revises: 8b780aea403a
+Create Date: 2026-06-04 16:56:18.693751
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b84462ca3800"
+down_revision: Union[str, None] = "8b780aea403a"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("""
+        CREATE OR REPLACE VIEW grafana.process_state
+        AS SELECT process_state.id,
+        process_state.document_id,
+        process_state.title,
+        process_state.created_at,
+        process_state.operation_order,
+        corpus.source_name
+           FROM document_related.process_state
+             JOIN document_related.welearn_document ON welearn_document.id = process_state.document_id
+             JOIN corpus_related.corpus ON welearn_document.corpus_id = corpus.id;
+        """)
+
+
+def downgrade() -> None:
+    op.execute("""
+        CREATE OR REPLACE VIEW grafana.process_state
+        AS SELECT process_state.id,
+        process_state.document_id,
+        process_state.title,
+        process_state.created_at,
+        process_state.operation_order,
+           FROM document_related.process_state;
+        """)

welearn_database-1.5.1/welearn_database/data/details_dict.py

@@ -0,0 +1,30 @@
+from dataclasses import asdict, is_dataclass
+
+from sqlalchemy import types
+
+
+class DetailsDict(types.TypeDecorator):
+    """
+    Convert dataclass instances into dictionaries for JSON storage.
+    """
+
+    impl = types.JSON
+    cache_ok = True
+
+    @staticmethod
+    def _is_dataclass_instance(obj):
+        return is_dataclass(obj) and not isinstance(obj, type)
+
+    def _inner_serialize_dataclass(self, value):
+        match value:
+            case list():
+                return [self._inner_serialize_dataclass(item) for item in value]
+            case dict():
+                return {k: self._inner_serialize_dataclass(v) for k, v in value.items()}
+        if self._is_dataclass_instance(value):
+            return asdict(value)
+        return value
+
+    def process_bind_param(self, value, dialect):
+        # Serialize recursively without mutating the original object stored on the ORM instance.
+        return self._inner_serialize_dataclass(value)

{welearn_database-1.4.5.dev0 → welearn_database-1.5.1}/welearn_database/data/enumeration.py

@@ -43,9 +43,3 @@ class ExternalIdType(StrEnum):
 class FilterType(StrEnum):
     SDG = auto()
     SOURCE = auto()
-
-
-class UniversityRole(StrEnum):
-    STUDENT = auto()
-    TEACHER = auto()
-    STAFF = auto()

{welearn_database-1.4.5.dev0 → welearn_database-1.5.1}/welearn_database/data/models/document_related.py

@@ -16,6 +16,7 @@ from sqlalchemy import (
 from sqlalchemy.dialects.postgresql import ARRAY, ENUM, TIMESTAMP
 from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
 
+from welearn_database.data.details_dict import DetailsDict
 from welearn_database.data.enumeration import (
     ContextType,
     Counter,
@@ -44,7 +45,7 @@ class WeLearnDocument(Base):
     """
     This class represents a document in the WeLearn system.
     :cvar id: The unique identifier of the document.
-    :cvar doi: The unique DOI identifier if it exist for this document
+    :cvar doi: The unique DOI identifier if it exists for this document
     :cvar external_id: ID use by the document provider for identify it (ex: PubMed ID, ArXiv ID, OAI PMH identifier etc.)
     :cvar external_id_type: The type of the external ID, represented as an enumeration
     :cvar url: The URL of the document.
@@ -83,7 +84,7 @@ class WeLearnDocument(Base):
     lang: Mapped[str | None]
     description: Mapped[str | None]
     full_content: Mapped[str | None]
-    details: Mapped[dict[str, Any] | None]
+    details: Mapped[dict[str, Any] | None] = mapped_column(DetailsDict)
     trace: Mapped[int | None] = mapped_column(types.BIGINT)
     corpus_id: Mapped[UUID] = mapped_column(
         types.Uuid,
@@ -478,6 +479,65 @@ class ContextDocument(Base):
     )
 
 
+class HistoricalQtyDocumentPerCorpus(Base):
+    __tablename__ = "historical_qty_document_per_corpus"
+    __table_args__ = {"schema": schema_name}
+
+    id = mapped_column(
+        types.Uuid,
+        primary_key=True,
+        server_default=GEN_RANDOM_UUID,
+        nullable=False,
+    )
+    source_name: Mapped[str] = mapped_column()
+    quantity: Mapped[int]
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default=NOW,
+    )
+
+
+class HistoricalQtyDocumentInQdrantPerCorpus(Base):
+    __tablename__ = "historical_qty_document_in_qdrant_per_corpus"
+    __table_args__ = {"schema": schema_name}
+
+    id = mapped_column(
+        types.Uuid,
+        primary_key=True,
+        server_default=GEN_RANDOM_UUID,
+        nullable=False,
+    )
+    source_name: Mapped[str] = mapped_column()
+    quantity: Mapped[int]
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default=NOW,
+    )
+
+
+class HistoricalQtyDocumentInQdrant(Base):
+    __tablename__ = "historical_qty_document_in_qdrant"
+    __table_args__ = {"schema": schema_name}
+
+    id = mapped_column(
+        types.Uuid,
+        primary_key=True,
+        server_default=GEN_RANDOM_UUID,
+        nullable=False,
+    )
+    quantity: Mapped[int]
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default=NOW,
+    )
+
+
 # Views
 class QtyDocumentInQdrant(Base):
     __tablename__ = "qty_document_in_qdrant"

{welearn_database-1.4.5.dev0 → welearn_database-1.5.1}/welearn_database/data/models/grafana.py

@@ -136,3 +136,18 @@ class Session(Base):
     created_at: Mapped[datetime] = mapped_column()
     end_at: Mapped[datetime | None] = mapped_column(nullable=True)
     host: Mapped[str | None] = mapped_column(nullable=True)
+
+
+class UsedFeaturePerSession(Base):
+    __tablename__ = "used_feature_per_session"
+    __table_args__ = {"schema": schema_name}
+    __read_only__ = True
+
+    inferred_user_id: Mapped[uuid.UUID] = mapped_column(
+        UUID(as_uuid=True), primary_key=True
+    )
+    session_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
+    feature_name: Mapped[str] = mapped_column(primary_key=True)
+    cnt: Mapped[int] = mapped_column()
+    is_feature_used: Mapped[bool] = mapped_column()
+    session_created_at: Mapped[datetime] = mapped_column()

{welearn_database-1.4.5.dev0 → welearn_database-1.5.1}/welearn_database/data/models/user_related.py

@@ -3,12 +3,11 @@ from uuid import UUID
 
 from sqlalchemy import ForeignKey, func, types
 from sqlalchemy.dialects.postgresql import ENUM, TIMESTAMP
-from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 
-from welearn_database.data.enumeration import DbSchemaEnum, FilterType, UniversityRole
+from welearn_database.data.enumeration import DbSchemaEnum, FilterType
 from welearn_database.data.models.document_related import WeLearnDocument
 
-from ...exceptions import EarlyEnumerationVerificationError
 from . import Base
 
 schema_name = DbSchemaEnum.USER_RELATED.value
@@ -218,14 +217,7 @@ class InferredUser(Base):
     )
     origin_referrer: Mapped[str | None]
     university_title: Mapped[str | None]
-    role: Mapped[str] = mapped_column(
-        ENUM(
-            *(r.value.lower() for r in UniversityRole),
-            name="university_role",
-            schema="user_related",
-        ),
-        nullable=True,
-    )
+    role: Mapped[str | None]
     created_at: Mapped[datetime] = mapped_column(
         TIMESTAMP(timezone=False),
         nullable=False,
@@ -233,13 +225,6 @@ class InferredUser(Base):
         server_default="NOW()",
     )
 
-    @validates("role")
-    def validate_role(self, key, value):
-        valid_roles = {r.value.lower() for r in UniversityRole}
-        if value not in valid_roles:
-            raise EarlyEnumerationVerificationError(f"Invalid role: {value}")
-        return value
-
 
 class EndpointRequest(Base):
     __tablename__ = "endpoint_request"