welearn-database 1.4.0.dev0__tar.gz → 1.4.4__tar.gz

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: welearn-database
-Version: 1.4.0.dev0
+Version: 1.4.4
 Summary: All stuff related to relationnal database from the WeLearn project
 License: cc-by-sa-nc
 Author: Théo
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: alembic (>=1.16.5,<2.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.11,<3.0.0)
-Requires-Dist: python-dotenv (>=1.1.1,<2.0.0)
+Requires-Dist: python-dotenv (>=1.2.2,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0.43,<3.0.0)
 Description-Content-Type: text/markdown
 

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "welearn-database"
-version = "1.4.0.dev0"
+version = "1.4.4"
 description = "All stuff related to relationnal database from the WeLearn project"
 authors = [
     {name = "Théo",email = "theo.nardin@cri-paris.org"}
@@ -11,17 +11,17 @@ requires-python = ">=3.10"
 dependencies = [
     "sqlalchemy (>=2.0.43,<3.0.0)",
     "alembic (>=1.16.5,<2.0.0)",
-    "python-dotenv (>=1.1.1,<2.0.0)",
+    "python-dotenv (>=1.2.2,<2.0.0)",
     "psycopg2-binary (>=2.9.11,<3.0.0)",
 ]
 
 [tool.poetry]
 
 [tool.poetry.group.dev.dependencies]
-mypy = "^1.16.0"
-bandit = "^1.8.6"
-isort = "^6.1.0"
-black = "^25.9.0"
+mypy = "^2.1.0"
+bandit = "^1.9.4"
+isort = "^8.0.1"
+black = "26.3.1"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/welearn_database/alembic/env.py

@@ -79,8 +79,16 @@ def run_migrations_online() -> None:
     if "prod" in os.getenv("PG_HOST").lower():
         print("Connecting to production database for migrations!")
         input("Press Enter to continue...")
-    elif "dev" in os.getenv("PG_HOST" "").lower():
-        print("Connecting to development database for migrations!")
+    elif (
+        "dev" in os.getenv("PG_HOST" "").lower()
+        and "staging" in os.getenv("PG_DB" "").lower()
+    ):
+        print("Connecting to staging database for migrations!")
+    elif (
+        "dev" in os.getenv("PG_HOST" "").lower()
+        and not "staging" in os.getenv("PG_DB" "").lower()
+    ):
+        print("Connecting to dev database for migrations!")
     else:
         print("Connecting to unknown database for migrations!")
 

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/welearn_database/alembic/versions/16ff997426d3_remove_error_retrieval_unique_constraint.py

@@ -9,9 +9,8 @@ Create Date: 2025-06-02 14:23:49.689745
 from typing import Sequence, Union
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = "16ff997426d3"
@@ -21,9 +20,10 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    op.drop_index(
+    op.drop_constraint(
         "error_retrieval_document_id_http_error_code_idx",
         "error_retrieval",
+        type_="unique",
         schema="document_related",
     )
 

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/welearn_database/alembic/versions/4c7161819e5a_grafana_views.py

@@ -18,6 +18,8 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade():
+    op.execute("CREATE SCHEMA IF NOT EXISTS grafana;")
+
     op.execute(
         """
     CREATE OR REPLACE VIEW grafana.corpus
@@ -162,6 +164,21 @@ AS SELECT session.id,
     op.execute(
         """
 CREATE OR REPLACE VIEW grafana.document_latest_state
+AS SELECT DISTINCT ON (ps.document_id) ps.id,
+    ps.document_id,
+    wd.corpus_id,
+    wd.lang,
+    ps.title,
+    ps.created_at,
+    ps.operation_order
+   FROM document_related.process_state ps
+     JOIN document_related.welearn_document wd ON ps.document_id = wd.id
+  ORDER BY ps.document_id, ps.operation_order DESC;
+    """
+    )
+    op.execute(
+        """
+CREATE OR REPLACE VIEW grafana.test_document_latest_state
 AS SELECT DISTINCT ON (ps.document_id) ps.id,
     ps.document_id,
     wd.corpus_id,

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/welearn_database/alembic/versions/821173cf9c5d_initial_migration.py

@@ -8,10 +8,10 @@ Create Date: 2025-01-27 18:10:55.045457
 
 from typing import Sequence, Union
 
+import sqlalchemy
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = "821173cf9c5d"
@@ -22,9 +22,22 @@ depends_on: Union[str, Sequence[str], None] = None
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("CREATE SCHEMA IF NOT EXISTS document_related;")
+    op.execute("CREATE SCHEMA IF NOT EXISTS corpus_related;")
+    op.execute("CREATE SCHEMA IF NOT EXISTS user_related;")
+
+    op.execute(
+        "CREATE SEQUENCE IF NOT EXISTS document_related.process_state_operation_order_seq"
+    )
+
     op.create_table(
         "bi_classifier_model",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("title", sa.String(), nullable=False),
         sa.Column("binary_treshold", sa.NUMERIC(), nullable=False),
         sa.Column("lang", sa.String(), nullable=False),
@@ -36,7 +49,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "corpus",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("source_name", sa.String(), nullable=False),
         sa.Column("is_fix", sa.Boolean(), nullable=False),
         sa.Column("binary_treshold", sa.NUMERIC(), nullable=False),
@@ -46,7 +64,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "embedding_model",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("title", sa.String(), nullable=False),
         sa.Column("lang", sa.String(), nullable=False),
         sa.PrimaryKeyConstraint("id"),
@@ -54,7 +77,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "n_classifier_model",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("title", sa.String(), nullable=False),
         sa.Column("lang", sa.String(), nullable=False),
         sa.Column("treshold_sdg_1", sa.NUMERIC(), nullable=False),
@@ -81,7 +109,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "keyword",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("keyword", sa.String(), nullable=False),
         sa.Column(
             "created_at", postgresql.TIMESTAMP(), server_default="NOW()", nullable=False
@@ -92,7 +125,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "user_profile",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("username", sa.String(), nullable=False),
         sa.Column("email", sa.String(), nullable=False),
         sa.Column("password_digest", sa.LargeBinary(), nullable=False),
@@ -173,7 +211,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "welearn_document",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("url", sa.String(), nullable=False),
         sa.Column("title", sa.String(), nullable=True),
         sa.Column("lang", sa.String(), nullable=True),
@@ -198,7 +241,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "chat_message",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("textual_content", sa.String(), nullable=False),
         sa.Column("user_id", sa.Uuid(), nullable=False),
         sa.Column(
@@ -210,13 +258,19 @@ def upgrade() -> None:
         sa.ForeignKeyConstraint(
             ["user_id"],
             ["user_related.user_profile.id"],
+            name="message_user_id_fkey",
         ),
         sa.PrimaryKeyConstraint("id"),
         schema="user_related",
     )
     op.create_table(
         "analytic_counter",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("document_id", sa.Uuid(), nullable=False),
         sa.Column("counter_name", sa.Enum("HIT", name="counter"), nullable=False),
         sa.Column("counter_value", sa.Integer(), nullable=False),
@@ -236,7 +290,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "document_slice",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("document_id", sa.Uuid(), nullable=False),
         sa.Column("embedding", sa.LargeBinary(), nullable=True),
         sa.Column("body", sa.String(), nullable=True),
@@ -257,7 +316,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "error_retrieval",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("document_id", sa.Uuid(), nullable=False),
         sa.Column("http_error_code", sa.Integer(), nullable=True),
         sa.Column(
@@ -282,7 +346,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "process_state",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("document_id", sa.Uuid(), nullable=False),
         sa.Column(
             "title",
@@ -308,7 +377,9 @@ def upgrade() -> None:
         sa.Column(
             "operation_order",
             sa.BIGINT(),
-            server_default="nextval('document_related.process_state_operation_order_seq",
+            server_default=sqlalchemy.text(
+                "nextval('document_related.process_state_operation_order_seq')"
+            ),
             nullable=False,
         ),
         sa.ForeignKeyConstraint(
@@ -321,7 +392,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "welearn_document_keyword",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("welearn_document_id", sa.Uuid(), nullable=False),
         sa.Column("keyword_id", sa.Uuid(), nullable=False),
         sa.ForeignKeyConstraint(
@@ -343,7 +419,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "bookmark",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("document_id", sa.Uuid(), nullable=False),
         sa.Column("user_id", sa.Uuid(), nullable=False),
         sa.Column(
@@ -365,7 +446,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "returned_document",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("message_id", sa.Uuid(), nullable=False),
         sa.Column("document_id", sa.Uuid(), nullable=False),
         sa.ForeignKeyConstraint(
@@ -382,7 +468,12 @@ def upgrade() -> None:
     )
     op.create_table(
         "sdg",
-        sa.Column("id", sa.Uuid(), server_default="gen_random_uuid()", nullable=False),
+        sa.Column(
+            "id",
+            sa.Uuid(),
+            server_default=sqlalchemy.func.gen_random_uuid(),
+            nullable=False,
+        ),
         sa.Column("sdg_number", sa.Integer(), nullable=False),
         sa.Column("slice_id", sa.Uuid(), nullable=False),
         sa.Column(
@@ -406,19 +497,19 @@ def upgrade() -> None:
         sa.PrimaryKeyConstraint("id"),
         schema="document_related",
     )
-    op.drop_table("alembic_version")
+    # op.drop_table("alembic_version")
     # ### end Alembic commands ###
 
 
 def downgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table(
-        "alembic_version",
-        sa.Column(
-            "version_num", sa.VARCHAR(length=32), autoincrement=False, nullable=False
-        ),
-        sa.PrimaryKeyConstraint("version_num", name="alembic_version_pkc"),
-    )
+    # op.create_table(
+    #     "alembic_version",
+    #     sa.Column(
+    #         "version_num", sa.VARCHAR(length=32), autoincrement=False, nullable=False
+    #     ),
+    #     sa.PrimaryKeyConstraint("version_num", name="alembic_version_pkc"),
+    # )
     op.drop_table("sdg", schema="document_related")
     op.drop_table("returned_document", schema="user_related")
     op.drop_table("bookmark", schema="user_related")

welearn_database-1.4.4/welearn_database/alembic/versions/b049924f7067_modify_corpus_name_embedding_model_lang_.py

@@ -0,0 +1,93 @@
+"""modify corpus_name_embedding_model_lang view
+
+Revision ID: b049924f7067
+Revises: f8602200fa99
+Create Date: 2026-03-31 16:09:12.085443
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "b049924f7067"
+down_revision: Union[str, None] = "f8602200fa99"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        DROP MATERIALIZED VIEW corpus_related.corpus_name_embedding_model_lang;
+        """
+    )
+    op.execute(
+        """
+
+CREATE MATERIALIZED VIEW corpus_related.corpus_name_embedding_model_lang
+TABLESPACE pg_default
+AS WITH ranked AS (
+SELECT
+    c.source_name,
+    cem.corpus_id,
+    cem.embedding_model_id,
+    em.title,
+    em.lang,
+    cem.used_since,
+    c.category_id,
+    ROW_NUMBER() OVER (
+      PARTITION BY cem.corpus_id,
+    em.lang
+ORDER BY
+    cem.used_since DESC
+    ) AS rn
+FROM
+    corpus_related.corpus_embedding_model cem
+JOIN corpus_related.corpus c ON
+    c.id = cem.corpus_id
+JOIN corpus_related.embedding_model em ON
+    em.id = cem.embedding_model_id
+WHERE
+    c.is_active
+)
+    SELECT
+        source_name,
+        corpus_id,
+        embedding_model_id,
+        title,
+        lang,
+        used_since,
+        category_id
+    FROM
+        ranked
+    WHERE
+        rn = 1
+        
+WITH DATA;
+    """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+        DROP MATERIALIZED VIEW corpus_related.corpus_name_embedding_model_lang;
+        """
+    )
+    op.execute(
+        """
+        CREATE MATERIALIZED VIEW corpus_related.corpus_name_embedding_model_lang
+        TABLESPACE pg_default
+        AS SELECT corpus.source_name,
+            embedding_model.title,
+            embedding_model.lang
+           FROM corpus_related.corpus
+             JOIN corpus_related.corpus_embedding_model ON corpus_embedding_model.corpus_id = corpus.id
+             JOIN corpus_related.embedding_model ON embedding_model.id = corpus_embedding_model.embedding_model_id
+          WHERE corpus.is_active
+        WITH DATA;
+        """
+    )

welearn_database-1.4.4/welearn_database/alembic/versions/f1ce0ad2845b_add_doi_column_to_document.py

@@ -0,0 +1,30 @@
+"""add doi column to document
+
+Revision ID: f1ce0ad2845b
+Revises: f8602200fa99
+Create Date: 2026-04-29 15:39:51.079086
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "f1ce0ad2845b"
+down_revision: Union[str, None] = "b049924f7067"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "welearn_document",
+        sa.Column("doi", sa.String(), nullable=True, unique=True),
+        schema="document_related",
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("welearn_document", "doi", schema="document_related")

welearn_database-1.4.4/welearn_database/alembic/versions/f8602200fa99_update_session_view.py

@@ -0,0 +1,42 @@
+"""update session view
+
+Revision ID: f8602200fa99
+Revises: 9b4f1da0c1f2
+Create Date: 2026-03-12 12:34:29.240684
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "f8602200fa99"
+down_revision: Union[str, None] = "9b4f1da0c1f2"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+CREATE OR REPLACE VIEW grafana."session"
+AS SELECT *
+   FROM user_related.session;
+    """
+    )
+
+
+def downgrade() -> None:
+    op.execute(
+        """
+CREATE OR REPLACE VIEW grafana."session"
+AS SELECT session.id,
+    session.inferred_user_id,
+    session.created_at,
+    session.end_at,
+    session.host
+   FROM user_related.session;
+    """
+    )

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/welearn_database/data/models/corpus_related.py

@@ -106,9 +106,13 @@ class CorpusNameEmbeddingModelLang(Base):
     __tablename__ = "corpus_name_embedding_model_lang"
     __table_args__ = {"schema": schema_name}
     __read_only__ = True
-    source_name: Mapped[str] = mapped_column(primary_key=True)
+    source_name: Mapped[str] = mapped_column()
+    corpus_id: Mapped[UUID] = mapped_column(primary_key=True)
+    embedding_model_id: Mapped[UUID]
     title: Mapped[str]
-    lang: Mapped[str]
+    lang: Mapped[str] = mapped_column(primary_key=True)
+    used_since: Mapped[datetime]
+    category_id: Mapped[UUID]
 
 
 class CorpusEmbeddingModel(Base):

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/welearn_database/data/models/document_related.py

@@ -1,3 +1,4 @@
+import re
 from datetime import datetime
 from typing import Any
 from urllib.parse import urlparse
@@ -30,16 +31,23 @@ from welearn_database.data.models.corpus_related import (
     EmbeddingModel,
     NClassifierModel,
 )
-from welearn_database.exceptions import InvalidURLScheme
+from welearn_database.exceptions import ContentIsTooShort, InvalidDOI, InvalidURLScheme
 from welearn_database.modules.text_cleaning import clean_text
+from welearn_database.regular_expression import DOI_VALIDATION_REGEX
 
 schema_name = DbSchemaEnum.DOCUMENT_RELATED.value
 
+NOW = "NOW()"
+GEN_RANDOM_UUID = "gen_random_uuid()"
+
 
 class WeLearnDocument(Base):
     """
     This class represents a document in the WeLearn system.
     :cvar id: The unique identifier of the document.
+    :cvar doi: The unique DOI identifier if it exist for this document
+    :cvar external_id: ID use by the document provider for identify it (ex: PubMed ID, ArXiv ID, OAI PMH identifier etc.)
+    :cvar external_id_type: The type of the external ID, represented as an enumeration
     :cvar url: The URL of the document.
     :cvar title: The title of the document.
     :cvar lang: The language of the document.
@@ -60,8 +68,9 @@ class WeLearnDocument(Base):
     )
 
     id: Mapped[UUID] = mapped_column(
-        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+        types.Uuid, primary_key=True, nullable=False, server_default=GEN_RANDOM_UUID
     )
+    doi: Mapped[str | None]
     external_id: Mapped[str | None]
     external_id_type: Mapped[str | None] = mapped_column(
         ENUM(
@@ -86,13 +95,13 @@ class WeLearnDocument(Base):
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
     updated_at: Mapped[datetime] = mapped_column(
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
         onupdate=func.localtimestamp(),
     )
 
@@ -127,7 +136,7 @@ class WeLearnDocument(Base):
             return value
         cleaned = clean_text(value)
         if len(value) < 25:
-            raise ValueError(f"Content is too short : {len(value)}")
+            raise ContentIsTooShort(f"Content is too short : {len(value)}")
 
         # Hash compute and db storage
         self.trace = adler32(cleaned.encode("utf-8"))
@@ -145,6 +154,16 @@ class WeLearnDocument(Base):
             return value
         return clean_text(value)
 
+    @validates("doi")
+    def validate_doi(self, key, value):
+        """"""
+        if not value:
+            return value
+
+        if not re.match(DOI_VALIDATION_REGEX, value):
+            raise InvalidDOI(f"DOI is not valid : {value}")
+        return value
+
 
 class ProcessState(Base):
     """
@@ -161,7 +180,7 @@ class ProcessState(Base):
     __table_args__ = {"schema": schema_name}
 
     id: Mapped[UUID] = mapped_column(
-        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+        types.Uuid, primary_key=True, nullable=False, server_default=GEN_RANDOM_UUID
     )
     document_id: Mapped[UUID] = mapped_column(
         types.Uuid,
@@ -180,7 +199,7 @@ class ProcessState(Base):
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
     operation_order = mapped_column(
         types.BIGINT,
@@ -198,14 +217,14 @@ class Keyword(Base):
     )
 
     id: Mapped[UUID] = mapped_column(
-        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+        types.Uuid, primary_key=True, nullable=False, server_default=GEN_RANDOM_UUID
     )
     keyword: Mapped[str]
     created_at: Mapped[datetime] = mapped_column(
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
 
 
@@ -220,7 +239,7 @@ class WeLearnDocumentKeyword(Base):
         {"schema": schema_name},
     )
     id: Mapped[UUID] = mapped_column(
-        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+        types.Uuid, primary_key=True, nullable=False, server_default=GEN_RANDOM_UUID
     )
     welearn_document_id: Mapped[UUID] = mapped_column(
         types.Uuid,
@@ -242,7 +261,7 @@ class ErrorRetrieval(Base):
     __table_args__ = ({"schema": schema_name},)
 
     id: Mapped[UUID] = mapped_column(
-        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+        types.Uuid, primary_key=True, nullable=False, server_default=GEN_RANDOM_UUID
     )
 
     document_id: Mapped[UUID] = mapped_column(
@@ -258,13 +277,13 @@ class ErrorRetrieval(Base):
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
     updated_at: Mapped[datetime] = mapped_column(
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
         onupdate=func.localtimestamp(),
     )
     error_info: Mapped[str]
@@ -280,7 +299,7 @@ class ErrorDataQuality(Base):
         types.Uuid,
         primary_key=True,
         nullable=False,
-        server_default="gen_random_uuid()",
+        server_default=GEN_RANDOM_UUID,
     )
     document_id: Mapped[UUID] = mapped_column(
         types.Uuid,
@@ -304,7 +323,7 @@ class ErrorDataQuality(Base):
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
     document: Mapped["WeLearnDocument"] = relationship(cascade="all, delete")
     slice: Mapped["DocumentSlice"] = relationship(cascade="all, delete")
@@ -315,7 +334,7 @@ class DocumentSlice(Base):
     __table_args__ = {"schema": schema_name}
 
     id: Mapped[UUID] = mapped_column(
-        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+        types.Uuid, primary_key=True, nullable=False, server_default=GEN_RANDOM_UUID
     )
     document_id: Mapped[UUID] = mapped_column(
         types.Uuid,
@@ -345,7 +364,7 @@ class AnalyticCounter(Base):
     __table_args__ = {"schema": schema_name}
 
     id: Mapped[UUID] = mapped_column(
-        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+        types.Uuid, primary_key=True, nullable=False, server_default=GEN_RANDOM_UUID
     )
     document_id: Mapped[UUID] = mapped_column(
         types.Uuid,
@@ -361,13 +380,13 @@ class AnalyticCounter(Base):
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
     updated_at: Mapped[datetime] = mapped_column(
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
         onupdate=func.localtimestamp(),
     )
     document: Mapped["WeLearnDocument"] = relationship()
@@ -381,7 +400,7 @@ class Sdg(Base):
         types.Uuid,
         primary_key=True,
         nullable=False,
-        server_default="gen_random_uuid()",
+        server_default=GEN_RANDOM_UUID,
     )
     slice_id = mapped_column(
         types.Uuid,
@@ -396,7 +415,7 @@ class Sdg(Base):
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
 
     bi_classifier_model_id = mapped_column(
@@ -418,7 +437,7 @@ class ContextDocument(Base):
     id = mapped_column(
         types.Uuid,
         primary_key=True,
-        server_default="gen_random_uuid()",
+        server_default=GEN_RANDOM_UUID,
         nullable=False,
     )
     url: Mapped[str]
@@ -433,7 +452,7 @@ class ContextDocument(Base):
         TIMESTAMP(timezone=False),
         nullable=False,
         default=func.localtimestamp(),
-        server_default="NOW()",
+        server_default=NOW,
     )
     embedding: Mapped[bytes] = mapped_column(LargeBinary)
 
@@ -484,4 +503,4 @@ class QtyDocumentPerCorpus(Base):
     __read_only__ = True
 
     source_name: Mapped[str] = mapped_column(primary_key=True)
-    count: Mapped[int]
+    count: Mapped[int]

{welearn_database-1.4.0.dev0 → welearn_database-1.4.4}/welearn_database/database_utils.py

@@ -1,7 +1,5 @@
 import logging
-import math
 import os
-from typing import Any, List
 
 from sqlalchemy import URL, create_engine
 from sqlalchemy.orm import sessionmaker

welearn_database-1.4.4/welearn_database/exceptions.py

@@ -0,0 +1,29 @@
+class InvalidURLScheme(Exception):
+    """
+    Scheme detected in URL is not accepted
+    """
+
+    def __init__(self, msg="URL schema is not accepted", *args):
+        super().__init__(msg, *args)
+
+
+class InvalidDOI(Exception):
+    """
+    Scheme detected in DOI is not accepted
+    """
+
+    def __init__(self, msg="DOI schema is not accepted", *args):
+        super().__init__(msg, *args)
+
+
+class ContentIsTooShort(Exception):
+    """
+    The string used as content is too short, it should be at least 25 characters long
+    """
+
+    def __init__(
+        self,
+        msg="Content is too short, it should be at least 25 characters long",
+        *args,
+    ):
+        super().__init__(msg, *args)

welearn_database-1.4.4/welearn_database/regular_expression.py

@@ -0,0 +1,11 @@
+# Regular expressions for data cleaning and preprocessing in the WeLearn Database project.
+
+# description: Matches backline characters (newline, tab, carriage return) for removal or replacement.
+# example: "Hello\n\tWorld" -> matches "\n" and "\t"
+# limit: Does not match other whitespace characters like spaces or form feeds.
+BACKLINES_REGEX = r"([\n\t\r])"
+
+# description: Match DOI identifier without doi.org as prefix
+# example : 10.1590/s0103-90162002000200027 -> matches "10.1590/s0103-90162002000200027"
+# limit : can't be really used for extraction, can be roughly used for other things than validation
+DOI_VALIDATION_REGEX = r"^(10\.\d{4,5}\/[\S]+[^;,.\s])$"

welearn_database-1.4.0.dev0/welearn_database/exceptions.py

@@ -1,7 +0,0 @@
-class InvalidURLScheme(Exception):
-    """
-    Scheme detected in URL is not accepted
-    """
-
-    def __init__(self, msg="URL schema is not accepted", *args):
-        super().__init__(msg, *args)