bdrc-db-models 2.0.2__py3-none-any.whl

BdrcDbModels/base.py

@@ -0,0 +1,25 @@
+import sqlalchemy
+from sqlalchemy.orm import declarative_base
+from sqlalchemy import Column, Integer, text
+from sqlalchemy.dialects.mysql import TIMESTAMP
+
+Base = declarative_base()
+
+class IdMixin:
+    """
+    Provides generic id in primary key
+    """
+    id = Column(Integer, primary_key=True, autoincrement=True)
+
+class TimestampMixin:
+    """
+    Injects created and update times
+    """
+    create_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+class IdTimestampMixin(IdMixin, TimestampMixin):
+    """
+    Provides generic id and create+update time columns
+    """
+    pass

BdrcDbModels/model_utils.py

@@ -0,0 +1,81 @@
+"""
+Some utilities. These used to be class members of the DrsDbContextBase,
+but the ORM classes were refactored into this library, so they could not be
+in the BdrcDbLib package (circular reference).
+
+Clients who use this will have to use them in the context of a DrsDbContext session.
+"""
+
+import logging
+from sqlalchemy import select
+from BdrcDbModels.models import Works, Volumes, GbDownload
+from BdrcDbLib.SqlAlchemy_get_or_create import get_or_create
+
+def get_some_works(session) -> list:
+    """
+    Test shell. See https://auth0.com/blog/sqlalchemy-orm-tutorial-for-python-developers/
+    for starters
+    :return:
+    """
+    return session.query(Works).filter(Works.WorkName == 'W1FPL2251').all()
+
+def get_work_by_name(session, work_name: str) -> Works:
+    """
+    Get work by Name
+    :param work_name: work to fetch
+    :return: MySQL ORM object
+    """
+    return session.query(Works).filter(Works.WorkName == work_name).first()
+
+def get_or_create_work(session, work_name: str) -> Works:
+    """
+    Get or create an object managed in this session
+    :param work_name: create or get by work name only
+    as you need
+    :return: object which matches the input filter, or a new object with the
+    **kwargs attributes
+    """
+    defaults = {"HOLLIS": '0'}
+    o, newly_made = get_or_create(session, Works, defaults, WorkName=work_name)
+    if newly_made:
+        logging.debug(f"Newly made {o}")
+    return o
+
+def get_or_create_volume(session, work_name: str, volume_name: str):
+    vol_work: Works = get_or_create_work(session, work_name)
+    o, newly_made = get_or_create(session, Volumes, label=volume_name, work=vol_work)
+    if newly_made:
+        logging.debug(f"Newly made {o}")
+    return o
+
+def get_downloads(session):
+    """
+    Gets all the downloads
+    """
+    statement = select(GbDownload).order_by(GbDownload.download_time)
+    return session.execute(statement).fetchall()
+
+def remove_volumes(session, labels: list[str]):
+    """
+    Removes a list of volumes identified by their label
+    !! Don't use in productionDon't generally use this!
+    :param labels: list of volumes to remove
+    :return:
+    """
+    vols2 = session.query(Volumes).filter(Volumes.label.in_(labels)).all()
+    for vv in vols2:
+        session.delete(vv)
+    session.commit()
+
+def remove_works(session, work_names: list[str]):
+    """
+    Removes a list of works identified by their WorkName.
+    !! Don't use in production
+    :param work_names:
+    :return:
+    """
+    works = session.query(Works).filter(Works.WorkName.in_(work_names)).all()
+    for w in works:
+        session.delete(w)
+    session.commit()
+

BdrcDbModels/models.py

@@ -0,0 +1,248 @@
+"""
+Sources:
+toth to https://stackoverflow.com/questions/33743379/sqlalchemy-timestamp-on-update-extra
+    create_time = Column(TIMESTAMP, nullable=False, server_default=sqlalchemy.func.now())
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+"""
+
+import sqlalchemy
+from sqlalchemy.orm import declarative_base, relationship
+from sqlalchemy import Column, DateTime, text
+from sqlalchemy import ForeignKey, String, Integer
+from sqlalchemy.dialects.mysql import (
+    BIGINT,
+    # BINARY,
+    # BIT,
+    # BLOB,
+    # BOOLEAN,
+    # CHAR,
+    # DATE,
+    # DATETIME,
+    # DECIMAL,
+    # DECIMAL,
+    # DOUBLE,
+    # ENUM,
+    # FLOAT,
+    # INTEGER,
+    # LONGBLOB,
+    LONGTEXT,
+    # MEDIUMBLOB,
+    # MEDIUMINT,
+    # MEDIUMTEXT,
+    # NCHAR,
+    # NUMERIC,
+    # NVARCHAR,
+    # REAL,
+    # SET,
+    # SMALLINT,
+    # TEXT,
+    # TIME,
+    TIMESTAMP,
+    # TINYBLOB,
+    # TINYINT,
+    # TINYTEXT,
+    # VARBINARY,
+    # VARCHAR,
+    # YEAR,
+)
+
+Base = declarative_base()
+
+class IdMixin:
+    """
+    Provides generic id in primary key
+    """
+    id = Column(Integer, primary_key=True, autoincrement=True)
+
+class TimestampMixin:
+    """
+    Injects created and update times
+    Tip o the hat to https://docs.sqlalchemy.org/en/13/orm/extensions/declarative/mixins.html
+    """
+
+    create_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+class IdTimestampMixin(IdMixin, TimestampMixin):
+    """
+    Provides generic id and create+update time columns
+    """
+
+class Works(Base):
+    __tablename__ = "Works"
+    workId = Column(Integer, primary_key=True, autoincrement=True)
+    WorkName = Column(String(45))
+    WorkSize = Column(BIGINT)
+    HOLLIS = Column(String(45))
+    WorkFileCount = Column(BIGINT)
+    WorkImageFileCount = Column(BIGINT)
+    WorkImageTotalFileSize = Column(BIGINT)
+    WorkNonImageFileCount = Column(BIGINT)
+    WorkNonImageTotalFileSize = Column(BIGINT)
+
+    volumes = relationship("Volumes", back_populates='work')
+    #    gb_metadata = relationship("GBMetadata", back_populates='work')
+
+    create_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+    def __repr__(self) -> str:
+        return f"id: {self.workId}:{self.WorkName}"
+
+
+class Volumes(Base):
+    __tablename__ = 'Volumes'
+    volumeId = Column(Integer, primary_key=True, autoincrement=True)
+    label = Column(String(45))
+    workId = Column(Integer, ForeignKey('Works.workId'))
+
+    work = relationship("Works", back_populates='volumes')
+    gb_downloads = relationship("GbDownload", back_populates='volume')
+    gb_content = relationship("GbContent", back_populates='volume')
+    gb_state = relationship('GbState', back_populates='volume')
+    gb_dist = relationship('GbDistribution', back_populates='volume')
+    gb_unpack = relationship('GbUnpack', back_populates='volume')
+
+    create_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+    def __repr__(self):
+        return f"<volume(ig):{self.label} id: {self.volumeId}>"
+
+
+class GbMetadata(Base):
+    """
+    xxx = GbMetaDataTrack(work_id = workId,upload_time = some_time)
+    Record the upload of a work's metadata to GB: columns
+    `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+    `update_time` timestamp NULL DEFAULT NULL,
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `work_id` int(11) DEFAULT NULL,
+    `upload_time` datetime NOT NULL,
+    `upload_result` int(11) DEFAULT NULL,
+    """
+    __tablename__ = 'GB_Metadata_Track'
+    id: Column = Column(Integer, primary_key=True, autoincrement=True)
+    work_id = Column(Integer, ForeignKey('Works.workId'))
+    upload_time = Column(TIMESTAMP)
+    upload_result = Column(Integer)
+
+    #    work = relationship('Works', back_populates='gb_metadata')
+
+    create_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+    def __repr__(self):
+        return f"<id: {self.id} upload time:{self.upload_time} work: {self.work_id}>"
+
+
+class GbContent(IdTimestampMixin, Base):
+    """
+    Records the steps in an **image group's** process. Captures the steps that **this service**
+    has performed
+    """
+    __tablename__ = 'GB_Content_Track'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    volume_id = Column(Integer, ForeignKey('Volumes.volumeId'))
+    job_step = Column(String(45))
+    step_time = Column(DateTime)
+    step_rc = Column(Integer)
+    gb_log = Column(LONGTEXT)
+
+    volume = relationship('Volumes', back_populates='gb_content')
+
+    create_time = Column(TIMESTAMP, nullable=False, server_default=sqlalchemy.func.now())
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+    def __repr__(self):
+        return f"<id: {self.id} step: {self.job_step} step time {self.step_time} volume:{self.volume}"
+
+
+class GbState(Base):
+    """
+    Records raw google book data as we poll for it. Captures the state of Google's processing
+    volume_id   int  FK Volumes.volumeId
+    job_state    varchar(45)    -- name of the page reporting this activity
+    state_date  datetime        -- observation datetime
+    gb_log      longtext        -- other columns in the row, as json dict. The programmer has to manually
+                                    determine the columns from the visual page - they do not come down in the
+                                    text version
+    """
+    __tablename__ = 'GB_Content_State'
+    volume_id = Column(Integer, ForeignKey('Volumes.volumeId'), primary_key=True)
+    job_state = Column(String(45), primary_key=True)
+    gb_log = Column(LONGTEXT)
+    state_date = Column(TIMESTAMP, primary_key=True)
+
+    volume = relationship('Volumes', back_populates='gb_state')
+
+    create_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+
+    def __repr__(self):
+        return f"<volume: {self.volume} state:{self.state}>"
+
+
+class GbDownload(Base):
+    __tablename__ = "GB_Downloads"
+    id: Column = Column(Integer, primary_key=True, autoincrement=True)
+    volume_id = Column(Integer, ForeignKey('Volumes.volumeId'))
+    download_object_name = Column(String(255))
+    download_path = Column(String(255))
+    download_time = Column(TIMESTAMP)
+    create_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))
+    update_time = Column(TIMESTAMP, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))
+    volume = relationship('Volumes', back_populates='gb_downloads')
+
+    def __repr__(self):
+        return f"<volume: {self.volume.label} path: {self.download_path} time: {self.download_time}>"
+
+
+class GbReadyTrack(IdTimestampMixin, Base):
+    """
+    Queue of items ready to be acted on
+    """
+    __tablename__ = 'GB_Ready_Track'
+    target_id = Column(Integer, comment='Id in specific tabel, varies with activity')
+    activity = Column(String(50), comment="Supported activities: download unpack")
+
+
+class GbUnpack(IdTimestampMixin, Base):
+    __tablename__ = "GB_Unpack"
+    volume_id = Column(Integer, ForeignKey('Volumes.volumeId'))
+    unpack_object_name = Column(String(255))
+    unpacked_path = Column(String(255))
+    unpack_time = Column(TIMESTAMP)
+    volume = relationship('Volumes', back_populates='gb_unpack')
+
+
+class GbDistribution(IdTimestampMixin, Base):
+    """
+    Records the steps in an **image group's** process. Captures the steps that **this service**
+    has performed
+    """
+    __tablename__ = 'GB_Distribution'
+
+    volume_id = Column(Integer, ForeignKey('Volumes.volumeId'))
+    dist_time = Column(TIMESTAMP)
+    src = Column(String(255))
+    dest = Column(String(255))
+
+    volume = relationship('Volumes', back_populates='gb_dist')
+
+    def __repr__(self):
+        return f"<id: {self.id} dist_time {self.dist_time} volume:{self.volume}>"
+
+
+class DipActivities(Base):
+    """
+    Legal DIP activities
+    """
+    __tablename__ = 'dip_activity_types'
+
+    id = Column('iddip_activity_types', Integer, primary_key=True, autoincrement=True)
+    label = Column('dip_activity_types_label', String(45))
+
+    def __repr__(self):
+        return f"<id: {self.id} label: {self.label}>"

BdrcDbModels/project_manager.py

@@ -0,0 +1,99 @@
+from sqlalchemy import Column, text, ForeignKey, String, Integer
+from sqlalchemy.dialects.mysql import LONGTEXT, TIMESTAMP
+from sqlalchemy.orm import relationship
+from .base import Base, IdTimestampMixin, TimestampMixin
+
+class ProjectTypes(IdTimestampMixin, Base):
+    """Types of projects, e.g. capture, digitization, distribution, analyis, etc."""
+    __tablename__ = "pm_project_types"
+    project_type_name = Column(String(45), nullable=False)
+    project_type_desc = Column(LONGTEXT)
+    project = relationship("Projects", back_populates="project_type")
+
+    def __repr__(self):
+        return f"<{self.id}: {self.project_type_name}>"
+
+
+class Projects(IdTimestampMixin, Base):
+    """Specific projects, e.g. Google books, DRS"""
+    __tablename__ = "pm_projects"
+    name = Column(String(45))
+    description = Column(LONGTEXT)
+
+    project_type_id = Column(Integer, ForeignKey('pm_project_types.id'))
+    project_type = relationship("ProjectTypes", back_populates="project")
+
+    m_type_id = Column(Integer, ForeignKey('pm_member_types.id'))
+    m_type = relationship("MemberTypes", back_populates="project")
+
+    member = relationship("ProjectMembers", back_populates="project")
+
+    def __repr__(self):
+        return f"<{self.id}: {self.name}-{self.project_type}-{self.m_type}>"
+
+
+class MemberTypes(IdTimestampMixin, Base):
+    """TUsusally, container level (work, volume).)"""
+    __tablename__ = "pm_member_types"
+    m_type = Column(String(45), nullable=False)
+    project = relationship('Projects', back_populates='m_type')
+    members = relationship("MemberTypes", back_populates="member_type")
+
+    def __repr__(self):
+        return f"<{self.id}: {self.m_type}>"
+
+
+class MemberStates(IdTimestampMixin, Base):
+    """States of project members (See States)"""
+    __tablename__ = "pm_member_states"
+    m_state_name = Column(String(45), nullable=False)
+    m_state_desc = Column(LONGTEXT)
+
+    project_member = relationship('ProjectMembers', back_populates="project_member_state")
+
+    def __repr__(self):
+        return f"<{self.id}: {self.m_state_name}>"
+
+
+class ProjectMembers(TimestampMixin, Base):
+    """An instance of a work or volume in a specific project"""
+    __tablename__ = "pm_project_members"
+    pm_id = Column(Integer, primary_key=True, autoincrement=True)
+    pm_type = Column(Integer, ForeignKey('pm_member_types.id'))
+
+    pm_work_id = Column(Integer, ForeignKey('Works.workId'))
+    pm_volume_id = Column(Integer, ForeignKey('Volumes.volumeId'))
+    
+    pm_project = Column(Integer, ForeignKey('pm_projects.id'))
+    project = relationship("Projects", back_populates="member")
+
+    pm_project_state_id = Column(Integer, ForeignKey('pm_member_states.id'))
+
+    member_type = relationship("MemberTypes", back_populates="members")
+    project_member_state = relationship('MemberStates', back_populates="project_member")
+
+    def __repr__(self):
+        return f"<{self.pm_id}: {self.project} {self.member_type}>"
+
+class Steps(IdTimestampMixin, Base):
+    """All possible steps."""
+    __tablename__ = "pm_steps"
+    s_name = Column(String(45), nullable=False)
+    s_desc = Column(LONGTEXT)
+
+
+class ProjectSteps(IdTimestampMixin, Base):
+    "A Step instance in a specific project"""
+    __tablename__ = "pm_project_steps"
+    ps_project = Column(ForeignKey('pm_projects.id'))
+    ps_step = Column(ForeignKey('pm_steps.id'))
+    project = relationship("Projects", back_populates="member")
+    step = relationship("Steps")
+
+
+class ProjectMemberSteps(TimestampMixin, Base):
+    """Lookup table to track steps completed for project members"""
+    __tablename__ = "pm_project_member_steps"
+    project_member_id = Column(ForeignKey('pm_project_members.pm_id'), primary_key=True)
+    project_step_id = Column(ForeignKey("pm_project_steps.id"), primary_key=True)
+    project_step_time = Column(TIMESTAMP, nullable=False, server_default=text('CURRENT_TIMESTAMP'))

bdrc_db_models-2.0.2.dist-info/METADATA

@@ -0,0 +1,282 @@
+Metadata-Version: 2.4
+Name: bdrc-db-models
+Version: 2.0.2
+Summary: More Secure Bdrc Db library
+Author-email: jimk <jimk@tbrc.org>, Buddhist Digital Resource Center <jimk@tbrc.org>
+License-Expression: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Requires-Python: >=3.9
+Description-Content-Type: text/x-rst
+Requires-Dist: bdrc-db-lib2>=2.0.3a
+Requires-Dist: pytest>=8.4.2
+
+==============
+Bdrc Db Models
+==============
+
+SqlAlchemy classes representing parts of the DRS database.
+
+Schema
+
+IdMixin
+=======
+
+Provides generic id in primary key
+
+**Attributes:**
+
+- id
+
+TimestampMixin
+==============
+
+Injects created and update times
+
+**Attributes:**
+
+- create_time
+- update_time
+
+IdTimestampMixin
+================
+
+Provides generic id and create+update time columns
+
+ProjectTypes
+============
+
+Types of projects, e.g. capture, digitization, distribution, analyis, etc.
+
+**Attributes:**
+
+- project_type_name
+- project_type_desc
+- project
+
+Projects
+========
+
+Specific projects, e.g. Google books, DRS
+
+**Attributes:**
+
+- name
+- description
+- project_type_id
+- project_type
+- m_type_id
+- m_type
+- member
+
+MemberTypes
+===========
+
+Ususally, container level (work, volume).
+
+**Attributes:**
+
+- m_type
+- project
+- members
+
+MemberStates
+============
+
+States of project members (See States)
+
+**Attributes:**
+
+- m_state_name
+- m_state_desc
+- project_member
+
+ProjectMembers
+==============
+
+An instance of a work or volume in a specific project
+
+**Attributes:**
+
+- pm_id
+- pm_type
+- pm_work_id
+- pm_volume_id
+- pm_project
+- project
+- pm_project_state_id
+- member_type
+- project_member_state
+
+Steps
+=====
+
+All possible steps.
+
+**Attributes:**
+
+- s_name
+- s_desc
+
+ProjectSteps
+============
+
+A Step instance in a specific project
+
+**Attributes:**
+
+- ps_project
+- ps_step
+- project
+- step
+
+ProjectMemberSteps
+==================
+
+Lookup table to track steps completed for project members
+
+**Attributes:**
+
+- project_member_id
+- project_step_id
+- project_step_time
+
+Works
+=====
+
+**Attributes:**
+
+- workId
+- WorkName
+- WorkSize
+- HOLLIS
+- WorkFileCount
+- WorkImageFileCount
+- WorkImageTotalFileSize
+- WorkNonImageFileCount
+- WorkNonImageTotalFileSize
+- volumes
+- create_time
+- update_time
+
+Volumes
+=======
+
+**Attributes:**
+
+- volumeId
+- label
+- workId
+- work
+- gb_downloads
+- gb_content
+- gb_state
+- gb_dist
+- gb_unpack
+- create_time
+- update_time
+
+GbMetadata
+==========
+
+Record the upload of a work's metadata to GB
+
+**Attributes:**
+
+- id
+- work_id
+- upload_time
+- upload_result
+- create_time
+- update_time
+
+GbContent
+=========
+
+Records the steps in an **image group's** process. Captures the steps that **this service** has performed
+
+**Attributes:**
+
+- id
+- volume_id
+- job_step
+- step_time
+- step_rc
+- gb_log
+- volume
+- create_time
+- update_time
+
+GbState
+=======
+
+Records raw google book data as we poll for it. Captures the state of Google's processing
+
+**Attributes:**
+
+- volume_id
+- job_state
+- gb_log
+- state_date
+- volume
+- create_time
+- update_time
+
+GbDownload
+==========
+
+**Attributes:**
+
+- id
+- volume_id
+- download_object_name
+- download_path
+- download_time
+- create_time
+- update_time
+- volume
+
+GbReadyTrack
+============
+
+Queue of items ready to be acted on
+
+**Attributes:**
+
+- target_id
+- activity
+
+GbUnpack
+========
+
+**Attributes:**
+
+- volume_id
+- unpack_object_name
+- unpacked_path
+- unpack_time
+- volume
+
+GbDistribution
+==============
+
+Records the steps in an **image group's** process. Captures the steps that **this service** has performed
+
+**Attributes:**
+
+- volume_id
+- dist_time
+- src
+- dest
+- volume
+
+DipActivities
+=============
+
+Legal DIP activities
+
+**Attributes:**
+
+- id
+- label

bdrc_db_models-2.0.2.dist-info/RECORD

@@ -0,0 +1,9 @@
+BdrcDbModels/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+BdrcDbModels/base.py,sha256=wleH1Z2OjT972Vji8GoZBRX15Siq_SKKp56-28ttcVw,722
+BdrcDbModels/model_utils.py,sha256=QKKu0oFTrfxJ7N4x8gqDijy3JCLPS7e6FXB5kRJ4Mvw,2621
+BdrcDbModels/models.py,sha256=6a4c9ve95Lm8l8SdxWkMgFOmMeaG3tbjUo5QcJ0Xm2g,8720
+BdrcDbModels/project_manager.py,sha256=LxiwSirieIOtqIjPsuw3X2QPWYB0Ua85rz-KrL9jTpQ,3888
+bdrc_db_models-2.0.2.dist-info/METADATA,sha256=XWyR40teVbRjdZpu7NylhS9PPnvrF4OL92hpHFMUkG8,3691
+bdrc_db_models-2.0.2.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+bdrc_db_models-2.0.2.dist-info/top_level.txt,sha256=q9XwX-TnCxFQStst5wsLxB8Hp0L0HQ5VNVaFjmYGVvU,13
+bdrc_db_models-2.0.2.dist-info/RECORD,,

bdrc_db_models-2.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

bdrc_db_models-2.0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+BdrcDbModels