cs-models 0.0.627__py3-none-any.whl → 0.0.847__py3-none-any.whl

cs_models/resources/OtherManager213F/schemas.py

@@ -0,0 +1,13 @@
+from marshmallow import Schema, fields
+from ...utils.utils import Safe13FInteger
+
+
+class Othermanager213FSchema(Schema):
+    id = fields.Integer(dump_only=True)
+    ACCESSION_NUMBER = fields.String(required=True)
+    SEQUENCENUMBER = Safe13FInteger(required=True)
+    CIK = fields.String()
+    FORM13FFILENUMBER = fields.String()
+    CRDNUMBER = fields.String()
+    SECFILENUMBER = fields.String()
+    NAME = fields.String(required=True)

cs_models/resources/PipelineCrawlSession/models.py

@@ -0,0 +1,67 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Float,
+    Text,
+    ForeignKey,
+    Enum,
+)
+from datetime import datetime
+
+from ...database import Base
+
+
+class PipelineCrawlSessionModel(Base):
+    __tablename__ = 'pipeline_crawl_sessions'
+
+    id = Column(Integer, primary_key=True)
+    company_sec_id = Column(
+        Integer,
+        ForeignKey('companies_sec.id'),
+        nullable=True,
+        index=True,
+    )
+    company_ous_id = Column(
+        Integer,
+        ForeignKey('companies_ous.id'),
+        nullable=True,
+        index=True,
+    )
+
+    # Session tracking
+    session_uuid = Column(String(36), nullable=False, unique=True, index=True)
+    crawl_date = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)
+
+    # Timing metrics
+    crawl_duration_seconds = Column(Float, nullable=True)
+    page_discovery_time_seconds = Column(Float, nullable=True)
+    extraction_time_seconds = Column(Float, nullable=True)
+
+    # Status tracking (MySQL ENUM)
+    status = Column(
+        Enum('started', 'discovering_pages', 'extracting_data', 'completed', 'failed', 'timeout'),
+        nullable=False,
+        default='started',
+        index=True,
+    )
+    error_message = Column(Text, nullable=True)
+
+    # Crawl statistics
+    total_pages_discovered = Column(Integer, nullable=True, default=0)
+    total_pages_crawled = Column(Integer, nullable=True, default=0)
+    total_screenshots_captured = Column(Integer, nullable=True, default=0)
+    total_drugs_extracted = Column(Integer, nullable=True, default=0)
+
+    # Crawl configuration
+    crawl_method = Column(String(50), nullable=True)
+    max_depth = Column(Integer, nullable=True, default=2)
+
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(
+        DateTime,
+        nullable=False,
+        default=datetime.utcnow,
+        onupdate=datetime.utcnow,
+    )

cs_models/resources/PipelineCrawlSession/schemas.py

@@ -0,0 +1,22 @@
+from marshmallow import Schema, fields
+
+
+class PipelineCrawlSessionSchema(Schema):
+    id = fields.Int(dump_only=True)
+    company_sec_id = fields.Int(allow_none=True)
+    company_ous_id = fields.Int(allow_none=True)
+    session_uuid = fields.Str(required=True)
+    crawl_date = fields.DateTime(required=True)
+    crawl_duration_seconds = fields.Float(allow_none=True)
+    page_discovery_time_seconds = fields.Float(allow_none=True)
+    extraction_time_seconds = fields.Float(allow_none=True)
+    status = fields.Str(required=True)
+    error_message = fields.Str(allow_none=True)
+    total_pages_discovered = fields.Int(allow_none=True)
+    total_pages_crawled = fields.Int(allow_none=True)
+    total_screenshots_captured = fields.Int(allow_none=True)
+    total_drugs_extracted = fields.Int(allow_none=True)
+    crawl_method = fields.Str(allow_none=True)
+    max_depth = fields.Int(allow_none=True)
+    created_at = fields.DateTime(dump_only=True)
+    updated_at = fields.DateTime(dump_only=True)

cs_models/resources/PipelineCrawledPage/models.py

@@ -0,0 +1,80 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Float,
+    Boolean,
+    ForeignKey,
+    Enum,
+    Index,
+    UniqueConstraint,
+)
+from datetime import datetime
+
+from ...database import Base
+
+
+class PipelineCrawledPageModel(Base):
+    __tablename__ = 'pipeline_crawled_pages'
+
+    id = Column(Integer, primary_key=True)
+    session_id = Column(
+        Integer,
+        ForeignKey('pipeline_crawl_sessions.id'),
+        nullable=False,
+        index=True,
+    )
+
+    # URL information
+    url = Column(String(1024), nullable=False)
+    url_hash = Column(String(64), nullable=False, index=True)
+    page_title = Column(String(512), nullable=True)
+
+    # Page content storage (S3)
+    html_content_s3_key = Column(String(512), nullable=True)
+    html_content_hash = Column(String(64), nullable=True)
+    html_content_length = Column(Integer, nullable=True)
+
+    # Screenshot storage (S3)
+    screenshot_s3_key = Column(String(512), nullable=True)
+    screenshot_hash = Column(String(64), nullable=True)
+    screenshot_width = Column(Integer, nullable=True)
+    screenshot_height = Column(Integer, nullable=True)
+    screenshot_file_size = Column(Integer, nullable=True)
+
+    # Page classification (MySQL ENUM)
+    page_type = Column(
+        Enum('pipeline', 'science', 'rd', 'products', 'clinical_trials', 'other'),
+        nullable=True,
+    )
+    relevance_score = Column(Float, nullable=True)
+
+    # Extraction metadata
+    has_drug_data = Column(Boolean, nullable=True, default=False)
+    extraction_method = Column(String(50), nullable=True)
+
+    # Crawl metadata
+    crawl_depth = Column(Integer, nullable=True, default=0)
+    parent_page_id = Column(Integer, ForeignKey('pipeline_crawled_pages.id'), nullable=True)
+    discovered_from_url = Column(String(1024), nullable=True)
+
+    # Technical metadata
+    http_status_code = Column(Integer, nullable=True)
+    content_type = Column(String(128), nullable=True)
+    response_time_ms = Column(Integer, nullable=True)
+
+    # Timestamps
+    crawled_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(
+        DateTime,
+        nullable=False,
+        default=datetime.utcnow,
+        onupdate=datetime.utcnow,
+    )
+
+    __table_args__ = (
+        Index('idx_session_url_hash', 'session_id', 'url_hash'),
+        UniqueConstraint('session_id', 'url_hash', name='uq_session_url'),
+    )

cs_models/resources/PipelineCrawledPage/schemas.py

@@ -0,0 +1,34 @@
+from marshmallow import Schema, fields
+
+
+class PipelineCrawledPageSchema(Schema):
+    id = fields.Int(dump_only=True)
+    session_id = fields.Int(required=True)
+    url = fields.Str(required=True)
+    url_hash = fields.Str(required=True)
+    page_title = fields.Str(allow_none=True)
+    html_content_s3_key = fields.Str(allow_none=True)
+    html_content_hash = fields.Str(allow_none=True)
+    html_content_length = fields.Int(allow_none=True)
+    screenshot_s3_key = fields.Str(allow_none=True)
+    screenshot_hash = fields.Str(allow_none=True)
+    screenshot_width = fields.Int(allow_none=True)
+    screenshot_height = fields.Int(allow_none=True)
+    screenshot_file_size = fields.Int(allow_none=True)
+    page_type = fields.Str(allow_none=True)
+    relevance_score = fields.Float(allow_none=True)
+    has_drug_data = fields.Bool(allow_none=True)
+    extraction_method = fields.Str(allow_none=True)
+    crawl_depth = fields.Int(allow_none=True)
+    parent_page_id = fields.Int(allow_none=True)
+    discovered_from_url = fields.Str(allow_none=True)
+    http_status_code = fields.Int(allow_none=True)
+    content_type = fields.Str(allow_none=True)
+    response_time_ms = fields.Int(allow_none=True)
+    crawled_at = fields.DateTime(required=True)
+    created_at = fields.DateTime(dump_only=True)
+    updated_at = fields.DateTime(dump_only=True)
+
+
+class PipelineCrawledPageResourceSchema(PipelineCrawledPageSchema):
+    pass

cs_models/resources/PipelineDrugPortfolio/models.py

@@ -0,0 +1,92 @@
+from sqlalchemy import (
+    Column,
+    BIGINT,
+    Integer,
+    String,
+    DateTime,
+    Float,
+    Text,
+    Boolean,
+    ForeignKey,
+    Enum,
+    Index,
+    JSON,
+)
+from datetime import datetime
+
+from ...database import Base
+
+
+class PipelineDrugPortfolioModel(Base):
+    __tablename__ = 'pipeline_drug_portfolio'
+
+    id = Column(BIGINT, primary_key=True)
+    session_id = Column(
+        Integer,
+        ForeignKey('pipeline_crawl_sessions.id'),
+        nullable=False,
+        index=True,
+    )
+    page_id = Column(
+        Integer,
+        ForeignKey('pipeline_crawled_pages.id'),
+        nullable=False,
+        index=True,
+    )
+
+    company_sec_id = Column(
+        Integer,
+        ForeignKey('companies_sec.id'),
+        nullable=True,
+        index=True,
+    )
+    company_ous_id = Column(
+        Integer,
+        ForeignKey('companies_ous.id'),
+        nullable=True,
+        index=True,
+    )
+
+    # Core drug information
+    drug_name = Column(String(512), nullable=False, index=True)
+    drug_name_normalized = Column(String(512), nullable=True, index=True)
+    synonyms = Column(JSON, nullable=True)
+
+    modality = Column(String(256), nullable=True)
+    targets = Column(JSON, nullable=True)
+    mechanism_of_action = Column(Text, nullable=True)
+
+    # Indications (JSON)
+    indications = Column(JSON, nullable=False)
+
+    # Additional information (JSON)
+    additional_info = Column(JSON, nullable=True)
+
+    # Extraction metadata (MySQL ENUM)
+    extraction_method = Column(
+        Enum('html_parsing', 'vision_model', 'hybrid', 'manual'),
+        nullable=False,
+    )
+    extraction_confidence = Column(Float, nullable=True)
+    raw_extraction_text = Column(Text, nullable=True)
+
+    # Data quality flags
+    is_verified = Column(Boolean, nullable=True, default=False)
+    is_duplicate = Column(Boolean, nullable=True, default=False, index=True)
+    needs_review = Column(Boolean, nullable=True, default=False, index=True)
+    review_notes = Column(Text, nullable=True)
+
+    # Timestamps
+    extracted_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow)
+    updated_at = Column(
+        DateTime,
+        nullable=False,
+        default=datetime.utcnow,
+        onupdate=datetime.utcnow,
+    )
+
+    __table_args__ = (
+        Index('idx_company_drug_name', 'company_sec_id', 'drug_name_normalized'),
+        Index('idx_session_drug', 'session_id', 'drug_name_normalized'),
+    )

cs_models/resources/PipelineDrugPortfolio/schemas.py

@@ -0,0 +1,31 @@
+from marshmallow import Schema, fields
+
+
+class PipelineDrugPortfolioSchema(Schema):
+    id = fields.Int(dump_only=True)
+    session_id = fields.Int(required=True)
+    page_id = fields.Int(required=True)
+    company_sec_id = fields.Int(allow_none=True)
+    company_ous_id = fields.Int(allow_none=True)
+    drug_name = fields.Str(required=True)
+    drug_name_normalized = fields.Str(allow_none=True)
+    synonyms = fields.Raw(allow_none=True)  # JSON array
+    modality = fields.Str(allow_none=True)
+    targets = fields.Raw(allow_none=True)  # JSON array
+    mechanism_of_action = fields.Str(allow_none=True)
+    indications = fields.Raw(required=True)  # JSON array
+    additional_info = fields.Raw(allow_none=True)  # JSON object
+    extraction_method = fields.Str(required=True)
+    extraction_confidence = fields.Float(allow_none=True)
+    raw_extraction_text = fields.Str(allow_none=True)
+    is_verified = fields.Bool(allow_none=True)
+    is_duplicate = fields.Bool(allow_none=True)
+    needs_review = fields.Bool(allow_none=True)
+    review_notes = fields.Str(allow_none=True)
+    extracted_at = fields.DateTime(required=True)
+    created_at = fields.DateTime(dump_only=True)
+    updated_at = fields.DateTime(dump_only=True)
+
+
+class PipelineDrugPortfolioResourceSchema(PipelineDrugPortfolioSchema):
+    pass

cs_models/resources/PipelineExtractionLog/models.py

@@ -0,0 +1,55 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    DateTime,
+    Text,
+    ForeignKey,
+    Enum,
+)
+from datetime import datetime
+
+from ...database import Base
+
+
+class PipelineExtractionLogModel(Base):
+    __tablename__ = 'pipeline_extraction_logs'
+
+    id = Column(Integer, primary_key=True)
+    session_id = Column(
+        Integer,
+        ForeignKey('pipeline_crawl_sessions.id'),
+        nullable=False,
+        index=True,
+    )
+    page_id = Column(
+        Integer,
+        ForeignKey('pipeline_crawled_pages.id'),
+        nullable=True,
+    )
+
+    # Log details (MySQL ENUM)
+    log_level = Column(
+        Enum('debug', 'info', 'warning', 'error'),
+        nullable=False,
+        default='info',
+        index=True,
+    )
+    message = Column(Text, nullable=False)
+
+    # Context
+    extraction_method = Column(String(50), nullable=True)
+    model_name = Column(String(128), nullable=True)
+    tokens_used = Column(Integer, nullable=True)
+    api_latency_ms = Column(Integer, nullable=True)
+
+    # Error tracking
+    exception_type = Column(String(256), nullable=True)
+    exception_message = Column(Text, nullable=True)
+    stack_trace = Column(Text, nullable=True)
+
+    # Retry tracking
+    attempt_number = Column(Integer, nullable=True, default=1)
+    max_attempts = Column(Integer, nullable=True, default=3)
+
+    created_at = Column(DateTime, nullable=False, default=datetime.utcnow, index=True)

cs_models/resources/PipelineExtractionLog/schemas.py

@@ -0,0 +1,23 @@
+from marshmallow import Schema, fields
+
+
+class PipelineExtractionLogSchema(Schema):
+    id = fields.Int(dump_only=True)
+    session_id = fields.Int(required=True)
+    page_id = fields.Int(allow_none=True)
+    log_level = fields.Str(required=True)
+    message = fields.Str(required=True)
+    extraction_method = fields.Str(allow_none=True)
+    model_name = fields.Str(allow_none=True)
+    tokens_used = fields.Int(allow_none=True)
+    api_latency_ms = fields.Int(allow_none=True)
+    exception_type = fields.Str(allow_none=True)
+    exception_message = fields.Str(allow_none=True)
+    stack_trace = fields.Str(allow_none=True)
+    attempt_number = fields.Int(allow_none=True)
+    max_attempts = fields.Int(allow_none=True)
+    created_at = fields.DateTime(dump_only=True)
+
+
+class PipelineExtractionLogResourceSchema(PipelineExtractionLogSchema):
+    pass

cs_models/resources/PromptIndices/models.py

@@ -0,0 +1,27 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    String,
+    Text,
+    DateTime,
+)
+from datetime import datetime
+from ...database import Base
+
+
+class PromptIndicesModel(Base):
+    __tablename__ = 'prompt_indices'
+
+    id = Column(Integer, primary_key=True)
+    prompt_index = Column(String(128), nullable=False)
+    text = Column(Text, nullable=False)
+    metadatas = Column(Text, nullable=False)
+    vector = Column(Text, nullable=False)
+    search_type = Column(String(50), nullable=True)
+    updated_at = Column(
+        DateTime,
+        nullable=False,
+        # https://stackoverflow.com/questions/58776476/why-doesnt-freezegun-work-with-sqlalchemy-default-values
+        default=lambda: datetime.utcnow(),
+        onupdate=lambda: datetime.utcnow(),
+    )

cs_models/resources/PromptIndices/schemas.py

@@ -0,0 +1,17 @@
+from marshmallow import (
+    Schema,
+    fields,
+    validate,
+)
+
+
+class PromptIndicesSchema(Schema):
+    not_blank = validate.Length(min=1, error='Field cannot be blank')
+
+    id = fields.Integer(dump_only=True)
+    prompt_index = fields.String(required=True)
+    text = fields.String(required=True)
+    metadatas = fields.String(required=True)
+    vector = fields.String(required=True)
+    search_type = fields.String(allow_none=True)
+    updated_at = fields.DateTime()

cs_models/resources/PubmedMeetingSellSideSignal/models.py

@@ -0,0 +1,64 @@
+from sqlalchemy import (
+    Column,
+    Integer,
+    Float,
+    DateTime,
+    ForeignKey,
+    UniqueConstraint,
+)
+from datetime import datetime
+
+from ...database import Base
+
+
+class PubmedMeetingSellSideSignalModel(Base):
+    __tablename__ = "pubmed_meeting_sell_side_signals"
+
+    id = Column(Integer, primary_key=True)
+
+    pubmed_id = Column(
+        Integer,
+        ForeignKey("pubmed.id"),
+        nullable=False,
+        index=True,
+    )
+    meeting_id = Column(
+        Integer,
+        ForeignKey("meetings.id"),
+        nullable=False,
+        index=True,
+    )
+    sell_side_source_id = Column(
+        Integer,
+        ForeignKey("sell_side_sources.id"),
+        nullable=False,
+        index=True,
+    )
+
+    # intensity
+    doc_count = Column(Integer, nullable=False)      # distinct notes
+    mention_count = Column(Integer, nullable=False)  # total mentions across those notes
+
+    # sentiment summary
+    avg_sentiment = Column(Float, nullable=True)
+    max_sentiment = Column(Float, nullable=True)
+
+    first_mention_at = Column(DateTime, nullable=True)
+    last_mention_at = Column(DateTime, nullable=True)
+
+    created_at = Column(DateTime, default=lambda: datetime.utcnow(), nullable=False)
+    updated_at = Column(
+        DateTime,
+        default=lambda: datetime.utcnow(),
+        onupdate=lambda: datetime.utcnow(),
+        nullable=False,
+    )
+
+    __table_args__ = (
+        UniqueConstraint(
+            "pubmed_id",
+            "meeting_id",
+            "sell_side_source_id",
+            name="uq_pubmed_meeting_source",
+        ),
+    )

cs_models/resources/PubmedMeetingSellSideSignal/schemas.py

@@ -0,0 +1,21 @@
+from marshmallow import Schema, fields, validate
+
+
+class PubmedMeetingSellSideSignalResourceSchema(Schema):
+    id = fields.Integer(dump_only=True)
+
+    pubmed_id = fields.Integer(required=True)
+    meeting_id = fields.Integer(required=True)
+    sell_side_source_id = fields.Integer(required=True)
+
+    doc_count = fields.Integer(required=True)
+    mention_count = fields.Integer(required=True)
+
+    avg_sentiment = fields.Float(allow_none=True)
+    max_sentiment = fields.Float(allow_none=True)
+
+    first_mention_at = fields.DateTime(allow_none=True)
+    last_mention_at = fields.DateTime(allow_none=True)
+
+    created_at = fields.DateTime(dump_only=True)
+    updated_at = fields.DateTime(dump_only=True)

cs_models/resources/PubmedMeetingUserDocument/models.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+
+from sqlalchemy import (
+    Column,
+    Text,
+    Integer,
+    DateTime,
+    ForeignKey,
+)
+
+from ...database import Base
+
+
+class PubmedMeetingUserDocumentModel(Base):
+    __tablename__ = "pubmed_meeting_user_documents"
+
+    id = Column(Integer, primary_key=True)
+    pubmed_id = Column(
+        Integer,
+        ForeignKey('pubmed.id'),
+        nullable=False,
+    )
+    meeting_id = Column(
+        Integer,
+        ForeignKey('meetings.id'),
+        nullable=False,
+    )
+    user_document_id = Column(
+        Integer,
+        ForeignKey('user_documents.id'),
+        nullable=False,
+    )
+    details = Column(Text, nullable=True)
+    updated_at = Column(
+        DateTime,
+        nullable=False,
+        # https://stackoverflow.com/questions/58776476/why-doesnt-freezegun-work-with-sqlalchemy-default-values
+        default=lambda: datetime.utcnow(),
+        onupdate=lambda: datetime.utcnow(),
+    )

cs_models/resources/PubmedMeetingUserDocument/schemas.py

@@ -0,0 +1,16 @@
+from marshmallow import (
+    Schema,
+    fields,
+    validate,
+)
+
+
+class PubmedMeetingUserDocumentResourceSchema(Schema):
+    not_blank = validate.Length(min=1, error='Field cannot be blank')
+
+    id = fields.Integer(dump_only=True)
+    pubmed_id = fields.Integer(required=True)
+    meeting_id = fields.Integer(required=True)
+    user_document_id = fields.Integer(required=True)
+    details = fields.String(allow_none=True)
+    updated_at = fields.DateTime()

cs_models/resources/PubmedUserDocument/models.py

@@ -0,0 +1,35 @@
+from datetime import datetime
+
+from sqlalchemy import (
+    Column,
+    Integer,
+    DateTime,
+    ForeignKey,
+    Boolean,
+)
+
+from ...database import Base
+
+
+class PubmedUserDocumentModel(Base):
+    __tablename__ = "pubmed_user_documents"
+
+    id = Column(Integer, primary_key=True)
+    pubmed_id = Column(
+        Integer,
+        ForeignKey('pubmed.id'),
+        nullable=False,
+    )
+    user_document_id = Column(
+        Integer,
+        ForeignKey('user_documents.id'),
+        nullable=False,
+    )
+    is_deleted = Column(Boolean, nullable=True)
+    updated_at = Column(
+        DateTime,
+        nullable=False,
+        # https://stackoverflow.com/questions/58776476/why-doesnt-freezegun-work-with-sqlalchemy-default-values
+        default=lambda: datetime.utcnow(),
+        onupdate=lambda: datetime.utcnow(),
+    )

cs_models/resources/PubmedUserDocument/schemas.py

@@ -0,0 +1,15 @@
+from marshmallow import (
+    Schema,
+    fields,
+    validate,
+)
+
+
+class PubmedUserDocumentResourceSchema(Schema):
+    not_blank = validate.Length(min=1, error='Field cannot be blank')
+
+    id = fields.Integer(dump_only=True)
+    pubmed_id = fields.Integer(required=True)
+    user_document_id = fields.Integer(required=True)
+    is_deleted = fields.Boolean(allow_none=True)
+    updated_at = fields.DateTime()