welearn-database 1.0.0.dev1__tar.gz → 1.0.0.dev2__tar.gz

{welearn_database-1.0.0.dev1 → welearn_database-1.0.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: welearn-database
-Version: 1.0.0.dev1
+Version: 1.0.0.dev2
 Summary: All stuff related to relationnal database from the WeLearn project
 License: cc-by-sa-nc
 Author: Théo

{welearn_database-1.0.0.dev1 → welearn_database-1.0.0.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "welearn-database"
-version = "1.0.0.dev1"
+version = "1.0.0.dev2"
 description = "All stuff related to relationnal database from the WeLearn project"
 authors = [
     {name = "Théo",email = "theo.nardin@cri-paris.org"}

{welearn_database-1.0.0.dev1 → welearn_database-1.0.0.dev2}/welearn_database/data/enumeration.py

@@ -1,4 +1,4 @@
-from enum import Enum, StrEnum, auto
+from enum import Enum, auto, StrEnum
 
 
 class Step(Enum):
@@ -13,11 +13,9 @@ class Step(Enum):
     KEPT_FOR_TRACE = "kept_for_trace"
     DOCUMENT_IS_IRRETRIEVABLE = "document_is_irretrievable"
 
-
 class Counter(Enum):
     HIT = auto()
 
-
 class DbSchemaEnum(StrEnum):
     GRAFANA = auto()
     AGENT_RELATED = auto()
@@ -25,16 +23,7 @@ class DbSchemaEnum(StrEnum):
     DOCUMENT_RELATED = auto()
     USER_RELATED = auto()
 
-
 class ContextType(StrEnum):
     INTRODUCTION = auto()
     TARGET = auto()
-    SUBJECT = auto()
-
-
-class ExternalIdType(StrEnum):
-    DOI = auto()
-    API_ID = auto()
-    HANDLE = auto()
-    SLUG = auto()
-    QID = auto()
+    SUBJECT = auto()

{welearn_database-1.0.0.dev1 → welearn_database-1.0.0.dev2}/welearn_database/data/models/corpus_related.py

@@ -1,15 +1,17 @@
 from datetime import datetime
 from uuid import UUID
 
-from sqlalchemy import types, ForeignKey, func
+from sqlalchemy import ForeignKey, UniqueConstraint, func, types
 from sqlalchemy.dialects.postgresql import TIMESTAMP
-from sqlalchemy.orm import mapped_column, Mapped
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 
-from . import Base
 from welearn_database.data.enumeration import DbSchemaEnum
 
+from . import Base
+
 schema_name = DbSchemaEnum.CORPUS_RELATED.value
 
+
 class Corpus(Base):
     __tablename__ = "corpus"
     __table_args__ = {"schema": schema_name}
@@ -26,6 +28,7 @@ class Corpus(Base):
         ForeignKey(f"{schema_name}.category.id"),
     )
 
+
 class Category(Base):
     __tablename__ = "category"
     __table_args__ = {"schema": schema_name}
@@ -97,11 +100,109 @@ class NClassifierModel(Base):
         server_default="NOW()",
     )
 
+
 class CorpusNameEmbeddingModelLang(Base):
     __tablename__ = "corpus_name_embedding_model_lang"
     __table_args__ = {"schema": schema_name}
     __read_only__ = True
-    source_name : Mapped[str]= mapped_column(primary_key=True)
+    source_name: Mapped[str] = mapped_column(primary_key=True)
     title: Mapped[str]
     lang: Mapped[str]
 
+
+class CorpusEmbeddingModel(Base):
+    __tablename__ = "corpus_embedding_model"
+    __table_args__ = (
+        UniqueConstraint(
+            "corpus_id",
+            "embedding_model_id",
+            name="unique_corpus_embedding_association",
+        ),
+        {"schema": schema_name},
+    )
+
+    corpus_id = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.corpus.id"),
+        primary_key=True,
+    )
+    embedding_model_id = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.embedding_model.id"),
+        primary_key=True,
+    )
+
+    used_since: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default="NOW()",
+    )
+
+    embedding_model: Mapped["EmbeddingModel"] = relationship()
+    corpus: Mapped["Corpus"] = relationship()
+
+
+class CorpusNClassifierModel(Base):
+    __tablename__ = "corpus_n_classifier_model"
+    __table_args__ = (
+        UniqueConstraint(
+            "corpus_id",
+            "n_classifier_model_id",
+            name="unique_corpus_n_classifier_association",
+        ),
+        {"schema": schema_name},
+    )
+
+    corpus_id = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.corpus.id"),
+        primary_key=True,
+    )
+    n_classifier_model_id = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.n_classifier_model.id"),
+        primary_key=True,
+    )
+
+    used_since: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default="NOW()",
+    )
+
+    n_classifier_model: Mapped["NClassifierModel"] = relationship()
+    corpus: Mapped["Corpus"] = relationship()
+
+
+class CorpusBiClassifierModel(Base):
+    __tablename__ = "corpus_bi_classifier_model"
+    __table_args__ = (
+        UniqueConstraint(
+            "corpus_id",
+            "bi_classifier_model_id",
+            name="unique_corpus_bi_classifier_association",
+        ),
+        {"schema": schema_name},
+    )
+
+    corpus_id = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.corpus.id"),
+        primary_key=True,
+    )
+    bi_classifier_model_id = mapped_column(
+        types.Uuid,
+        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.bi_classifier_model.id"),
+        primary_key=True,
+    )
+    used_since: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default="NOW()",
+    )
+
+    bi_classifier_model: Mapped["BiClassifierModel"] = relationship()
+    corpus: Mapped["Corpus"] = relationship()

{welearn_database-1.0.0.dev1 → welearn_database-1.0.0.dev2}/welearn_database/data/models/document_related.py

@@ -8,13 +8,7 @@ from sqlalchemy import ForeignKey, Integer, LargeBinary, UniqueConstraint, func,
 from sqlalchemy.dialects.postgresql import ARRAY, ENUM, TIMESTAMP
 from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
 
-from welearn_database.data.enumeration import (
-    ContextType,
-    Counter,
-    DbSchemaEnum,
-    ExternalIdType,
-    Step,
-)
+from welearn_database.data.enumeration import ContextType, Counter, DbSchemaEnum, Step
 from welearn_database.data.models import Base
 from welearn_database.data.models.corpus_related import (
     BiClassifierModel,
@@ -54,14 +48,6 @@ class WeLearnDocument(Base):
     id: Mapped[UUID] = mapped_column(
         types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
     )
-    external_id: Mapped[str | None]
-    external_id_type: Mapped[str | None] = mapped_column(
-        ENUM(
-            *(e.value.lower() for e in ExternalIdType),
-            name="external_id_type",
-            schema="document_related",
-        ),
-    )
     url: Mapped[str] = mapped_column(nullable=False)
     title: Mapped[str | None]
     lang: Mapped[str | None]
@@ -264,6 +250,41 @@ class ErrorRetrieval(Base):
     document: Mapped["WeLearnDocument"] = relationship()
 
 
+class ErrorDataQuality(Base):
+    __tablename__ = "error_data_quality"
+    __table_args__ = ({"schema": schema_name},)
+
+    id: Mapped[UUID] = mapped_column(
+        types.Uuid, primary_key=True, nullable=False, server_default="gen_random_uuid()"
+    )
+    document_id: Mapped[UUID] = mapped_column(
+        types.Uuid,
+        ForeignKey(
+            f"{DbSchemaEnum.DOCUMENT_RELATED.value}.welearn_document.id",
+            name="error_data_quality_document_id_fkey",
+        ),
+        nullable=False,
+    )
+    slice_id: Mapped[UUID] = mapped_column(
+        types.Uuid,
+        ForeignKey(
+            f"{DbSchemaEnum.DOCUMENT_RELATED.value}.document_slice.id",
+            name="error_data_quality_slice_id_fkey",
+        ),
+        nullable=True,
+    )
+    error_raiser: Mapped[str]
+    error_info: Mapped[str]
+    created_at: Mapped[datetime] = mapped_column(
+        TIMESTAMP(timezone=False),
+        nullable=False,
+        default=func.localtimestamp(),
+        server_default="NOW()",
+    )
+    document: Mapped["WeLearnDocument"] = relationship()
+    slice: Mapped["DocumentSlice"] = relationship()
+
+
 class DocumentSlice(Base):
     __tablename__ = "document_slice"
     __table_args__ = {"schema": schema_name}
@@ -327,104 +348,6 @@ class AnalyticCounter(Base):
     document: Mapped["WeLearnDocument"] = relationship()
 
 
-class CorpusEmbeddingModel(Base):
-    __tablename__ = "corpus_embedding_model"
-    __table_args__ = (
-        UniqueConstraint(
-            "corpus_id",
-            "embedding_model_id",
-            name="unique_corpus_embedding_association",
-        ),
-        {"schema": schema_name},
-    )
-
-    corpus_id = mapped_column(
-        types.Uuid,
-        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.corpus.id"),
-        primary_key=True,
-    )
-    embedding_model_id = mapped_column(
-        types.Uuid,
-        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.embedding_model.id"),
-        primary_key=True,
-    )
-
-    used_since: Mapped[datetime] = mapped_column(
-        TIMESTAMP(timezone=False),
-        nullable=False,
-        default=func.localtimestamp(),
-        server_default="NOW()",
-    )
-
-    embedding_model: Mapped["EmbeddingModel"] = relationship()
-    corpus: Mapped["Corpus"] = relationship()
-
-
-class CorpusNClassifierModel(Base):
-    __tablename__ = "corpus_n_classifier_model"
-    __table_args__ = (
-        UniqueConstraint(
-            "corpus_id",
-            "n_classifier_model_id",
-            name="unique_corpus_n_classifier_association",
-        ),
-        {"schema": schema_name},
-    )
-
-    corpus_id = mapped_column(
-        types.Uuid,
-        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.corpus.id"),
-        primary_key=True,
-    )
-    n_classifier_model_id = mapped_column(
-        types.Uuid,
-        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.n_classifier_model.id"),
-        primary_key=True,
-    )
-
-    used_since: Mapped[datetime] = mapped_column(
-        TIMESTAMP(timezone=False),
-        nullable=False,
-        default=func.localtimestamp(),
-        server_default="NOW()",
-    )
-
-    n_classifier_model: Mapped["NClassifierModel"] = relationship()
-    corpus: Mapped["Corpus"] = relationship()
-
-
-class CorpusBiClassifierModel(Base):
-    __tablename__ = "corpus_bi_classifier_model"
-    __table_args__ = (
-        UniqueConstraint(
-            "corpus_id",
-            "bi_classifier_model_id",
-            name="unique_corpus_bi_classifier_association",
-        ),
-        {"schema": schema_name},
-    )
-
-    corpus_id = mapped_column(
-        types.Uuid,
-        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.corpus.id"),
-        primary_key=True,
-    )
-    bi_classifier_model_id = mapped_column(
-        types.Uuid,
-        ForeignKey(f"{DbSchemaEnum.CORPUS_RELATED.value}.bi_classifier_model.id"),
-        primary_key=True,
-    )
-    used_since: Mapped[datetime] = mapped_column(
-        TIMESTAMP(timezone=False),
-        nullable=False,
-        default=func.localtimestamp(),
-        server_default="NOW()",
-    )
-
-    bi_classifier_model: Mapped["BiClassifierModel"] = relationship()
-    corpus: Mapped["Corpus"] = relationship()
-
-
 class Sdg(Base):
     __tablename__ = "sdg"
     __table_args__ = {"schema": schema_name}