ingestr 0.13.75__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/linear/__init__.py

@@ -3,7 +3,12 @@ from typing import Any, Dict, Iterable, Iterator
 import dlt
 import pendulum
 
-from .helpers import _normalize_issue, _normalize_team, _paginate
+from .helpers import (
+    _create_paginated_resource,
+    _get_date_range,
+    _graphql,
+    normalize_dictionaries,
+)
 
 ISSUES_QUERY = """
 query Issues($cursor: String) {
@@ -14,55 +19,407 @@ query Issues($cursor: String) {
       description
       createdAt
       updatedAt
-      creator { id }
-      assignee { id}
-      state { id}
-      labels { nodes { id } }
-      cycle { id}
-      project { id }
-      subtasks: children { nodes { id title } }
-      comments(first: 250) { nodes { id body } }
+      archivedAt
+      addedToCycleAt
+      autoArchivedAt
+      autoClosedAt
+      boardOrder
+      branchName
+      canceledAt
+      completedAt
+      customerTicketCount
+      descriptionState
+      dueDate
+      estimate
+      identifier
+      integrationSourceType
+      labelIds
+      number
+      previousIdentifiers
       priority
-      attachments { nodes { id } }
-      subscribers { nodes { id } }
+      priorityLabel
+      prioritySortOrder
+      reactionData
+      slaBreachesAt
+      slaHighRiskAt
+      slaMediumRiskAt
+      slaStartedAt
+      slaType
+      snoozedUntilAt
+      sortOrder
+      startedAt
+      startedTriageAt
+      subIssueSortOrder
+      triagedAt
+      url
+      
+      creator { id }
+      assignee { id }
+      botActor { id name type }
+      cycle { id }
+      delegate { id }
+      externalUserCreator { id }
+      favorite { id }
+      lastAppliedTemplate { id }
+      parent { id }
+      projectMilestone { id }
+      recurringIssueTemplate { id }
+      snoozedBy { id }
+      sourceComment { id }
+      state { id }
+      
+      labels(first: 250) { 
+        nodes { 
+          id 
+        } 
+      }
     }
     pageInfo { hasNextPage endCursor }
   }
 }
 """
 
-PROJECTS_QUERY = """
-query Projects($cursor: String) {
-  projects(first: 50, after: $cursor) {
+ATTACHMENTS_QUERY = """
+query Attachments($cursor: String) {
+  attachments(first: 50, after: $cursor) {
     nodes {
       id
-      name
+      archivedAt
+      bodyData
+      createdAt
+      groupBySource
+      metadata
+      sourceType
+      subtitle
+      title
+      updatedAt
+      url
+      
+      creator { id }
+      externalUserCreator { id }
+      issue { id }
+      originalIssue { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+COMMENTS_QUERY = """
+query Comments($cursor: String) {
+  comments(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      body
+      bodyData
+      createdAt
+      editedAt
+      quotedText
+      reactionData 
+      resolvedAt
+      threadSummary
+      updatedAt
+      url
+      
+      botActor { id  }
+      documentContent { id }
+      externalThread { id }
+      externalUser { id }
+      initiativeUpdate { id }
+      issue { id }
+      parent { id }
+      post { id }
+      projectUpdate { id }
+      resolvingComment { id }
+      resolvingUser { id }
+      user { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+CYCLES_QUERY = """
+query Cycles($cursor: String) {
+  cycles(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      autoArchivedAt
+      completedAt
+      completedIssueCountHistory
+      completedScopeHistory
+      createdAt
       description
+      endsAt
+      inProgressScopeHistory
+      issueCountHistory
+      name
+      number
+      progress
+      scopeHistory
+      startsAt
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+DOCUMENTS_QUERY = """
+query Documents($cursor: String) {
+  documents(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
       createdAt
+      icon
+      slugId
+      title
       updatedAt
+      
+      creator { id }
+      updatedBy { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+EXTERNAL_USERS_QUERY = """
+query ExternalUsers($cursor: String) {
+  externalUsers(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      avatarUrl
+      createdAt
+      displayName
+      email
+      lastSeen
+      name
+      updatedAt
+      
+      organization { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+INITIATIVES_QUERY = """
+query Initiatives($cursor: String) {
+  initiatives(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
+      completedAt
+      content
+      createdAt
+      description
+      frequencyResolution
       health
-      priority
+      healthUpdatedAt
+      icon
+      name
+      slugId
+      sortOrder
+      startedAt
+      status
       targetDate
-      lead { id }
+      targetDateResolution
+      trashed
+      updateReminderFrequency
+      updateReminderFrequencyInWeeks
+      updateRemindersDay
+      updateRemindersHour
+      updatedAt
+      
+      creator { id }
+      documentContent { id }
+      integrationsSettings { id }
+      lastUpdate { id }
+      organization { id }
+      owner { id }
+      parentInitiative { id }
     }
     pageInfo { hasNextPage endCursor }
   }
 }
 """
 
-TEAMS_QUERY = """
-query Teams($cursor: String) {
-  teams(first: 50, after: $cursor) {
+
+INITIATIVE_TO_PROJECTS_QUERY = """
+query InitiativeToProjects($cursor: String) {
+  initiativeToProjects(first: 50, after: $cursor) {
     nodes {
       id
+      archivedAt
+      createdAt
+      sortOrder
+      updatedAt
+      
+      initiative { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+PROJECT_MILESTONES_QUERY = """
+query ProjectMilestones($cursor: String) {
+  projectMilestones(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      createdAt
+      currentProgress
+      description
+      descriptionState
       name
-      key
+      progress
+      progressHistory
+      sortOrder
+      status
+      targetDate
+      updatedAt
+      
+      documentContent { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+PROJECT_STATUSES_QUERY = """
+query ProjectStatuses($cursor: String) {
+  projectStatuses(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
+      createdAt
+      description
+      indefinite
+      name
+      position
+      type
+      updatedAt
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+INTEGRATIONS_QUERY = """
+query Integrations($cursor: String) {
+  integrations(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      createdAt
+      service
+      updatedAt
+      
+      creator { id }
+      organization { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+LABELS_QUERY = """
+query IssueLabels($cursor: String) {
+  issueLabels(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
+      createdAt
       description
+      name
+      updatedAt
+      
+      creator { id }
+      organization { id }
+      parent { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+ORGANIZATION_QUERY = """
+query Organization {
+  viewer {
+    organization {
+      id
+      name
+      createdAt
       updatedAt
+      archivedAt
+      logoUrl
+      allowMembersToInvite
+      allowedAuthServices
+      createdIssueCount
+      customerCount
+      customersEnabled
+      deletionRequestedAt
+      gitBranchFormat
+      gitLinkbackMessagesEnabled
+      gitPublicLinkbackMessagesEnabled
+      logoUrl
+      periodUploadVolume
+      previousUrlKeys
+      roadmapEnabled
+      samlEnabled
+      scimEnabled
+    }
+  }
+}
+"""
+
+
+PROJECT_UPDATES_QUERY = """
+query ProjectUpdates($cursor: String) {
+  projectUpdates(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      body
+      bodyData
       createdAt
-      memberships { nodes { id } }
-      members { nodes { id } }
-      projects { nodes { id } }
+      diffMarkdown
+      health
+      updatedAt
+      url
+      
+      user { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+TEAM_MEMBERSHIPS_QUERY = """
+query TeamMemberships($cursor: String) {
+  teamMemberships(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      createdAt
+      owner
+      sortOrder
+      updatedAt
+      
+      user { id }
     }
     pageInfo { hasNextPage endCursor }
   }
@@ -74,16 +431,168 @@ query Users($cursor: String) {
   users(first: 50, after: $cursor) {
     nodes {
       id
-      name
+      active
+      admin
+      archivedAt
+      avatarUrl
+      calendarHash
+      createdAt
+      createdIssueCount
+      description
+      disableReason
       displayName
       email
+      guest
+      inviteHash
+      lastSeen
+      name
+      statusEmoji
+      statusLabel
+      statusUntilAt
+      timezone
+      updatedAt
+      url
+      
+      organization { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+WORKFLOW_STATES_QUERY = """
+query WorkflowStates($cursor: String) {
+  workflowStates(first: 50, after: $cursor) {
+    nodes {
+      id
+      archivedAt
+      color
       createdAt
+      description
+      name
+      position
+      type
       updatedAt
     }
     pageInfo { hasNextPage endCursor }
   }
 }
 """
+PROJECTS_QUERY = """
+query Projects($cursor: String) {
+  projects(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      description
+      createdAt
+      updatedAt
+      archivedAt
+      completedAt
+      canceledAt
+      startedAt
+      
+      color
+      icon
+      slugId
+      url
+      
+      health
+      priority
+      priorityLabel
+      state
+      
+      targetDate
+      startDate
+      
+      progress
+      currentProgress
+      scope
+      
+      sortOrder
+      trashed
+      
+      creator { id }
+      lead { id }
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+TEAMS_QUERY = """
+query Teams($cursor: String) {
+  teams(first: 50, after: $cursor) {
+    nodes {
+      id
+      name
+      key
+      description
+      color
+      icon
+      private
+      archivedAt
+      createdAt
+      updatedAt
+      
+      organization { id }
+      parent { id }
+      
+      cyclesEnabled
+      cycleDuration
+      cycleStartDay
+      cycleCooldownTime
+      
+      issueCount
+      issueEstimationType
+      issueEstimationAllowZero
+      issueEstimationExtended
+      issueOrderingNoPriorityFirst
+      
+      autoArchivePeriod
+      autoClosePeriod
+      autoCloseChildIssues
+      autoCloseParentIssues
+      
+      groupIssueHistory
+      timezone
+      inviteHash
+      joinByDefault
+      
+      slackNewIssue
+      slackIssueComments
+      slackIssueStatuses
+      
+      triageEnabled
+      requirePriorityToLeaveTriage
+      upcomingCycleCount
+    }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+"""
+
+
+# Paginated resources configuration
+PAGINATED_RESOURCES = [
+    ("issues", ISSUES_QUERY, "issues"),
+    ("users", USERS_QUERY, "users"),
+    ("workflow_states", WORKFLOW_STATES_QUERY, "workflowStates"),
+    ("cycles", CYCLES_QUERY, "cycles"),
+    ("attachments", ATTACHMENTS_QUERY, "attachments"),
+    ("comments", COMMENTS_QUERY, "comments"),
+    ("documents", DOCUMENTS_QUERY, "documents"),
+    ("external_users", EXTERNAL_USERS_QUERY, "externalUsers"),
+    ("initiative", INITIATIVES_QUERY, "initiatives"),
+    ("integrations", INTEGRATIONS_QUERY, "integrations"),
+    ("labels", LABELS_QUERY, "issueLabels"),
+    ("project_updates", PROJECT_UPDATES_QUERY, "projectUpdates"),
+    ("team_memberships", TEAM_MEMBERSHIPS_QUERY, "teamMemberships"),
+    ("initiative_to_project", INITIATIVE_TO_PROJECTS_QUERY, "initiativeToProjects"),
+    ("project_milestone", PROJECT_MILESTONES_QUERY, "projectMilestones"),
+    ("project_status", PROJECT_STATUSES_QUERY, "projectStatuses"),
+    ("projects", PROJECTS_QUERY, "projects"),
+    ("teams", TEAMS_QUERY, "teams"),
+]
 
 
 @dlt.source(name="linear", max_table_nesting=0)
@@ -92,8 +601,8 @@ def linear_source(
     start_date: pendulum.DateTime,
     end_date: pendulum.DateTime | None = None,
 ) -> Iterable[dlt.sources.DltResource]:
-    @dlt.resource(name="issues", primary_key="id", write_disposition="merge")
-    def issues(
+    @dlt.resource(name="organization", primary_key="id", write_disposition="merge")
+    def organization(
         updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
             "updatedAt",
             initial_value=start_date.isoformat(),
@@ -102,96 +611,24 @@ def linear_source(
             range_end="closed",
         ),
     ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
-
-        for item in _paginate(api_key, ISSUES_QUERY, "issues"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
-                if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield _normalize_issue(item)
+        current_start_date, current_end_date = _get_date_range(updated_at, start_date)
 
-    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
-    def projects(
-        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "updatedAt",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
-
-        for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
+        data = _graphql(api_key, ORGANIZATION_QUERY)
+        if "viewer" in data and "organization" in data["viewer"]:
+            item = data["viewer"]["organization"]
+            if item and pendulum.parse(item["updatedAt"]) >= current_start_date:
                 if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+                    yield normalize_dictionaries(item)
 
-    @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
-    def teams(
-        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "updatedAt",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[Dict[str, Any]]:
-        print(start_date)
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-        print(current_start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
-
-        for item in _paginate(api_key, TEAMS_QUERY, "teams"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
-                if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield _normalize_team(item)
-
-    @dlt.resource(name="users", primary_key="id", write_disposition="merge")
-    def users(
-        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "updatedAt",
-            initial_value=start_date.isoformat(),
-            end_value=end_date.isoformat() if end_date else None,
-            range_start="closed",
-            range_end="closed",
-        ),
-    ) -> Iterator[Dict[str, Any]]:
-        if updated_at.last_value:
-            current_start_date = pendulum.parse(updated_at.last_value)
-        else:
-            current_start_date = pendulum.parse(start_date)
-
-        if updated_at.end_value:
-            current_end_date = pendulum.parse(updated_at.end_value)
-        else:
-            current_end_date = pendulum.now(tz="UTC")
-
-        for item in _paginate(api_key, USERS_QUERY, "users"):
-            if pendulum.parse(item["updatedAt"]) >= current_start_date:
-                if pendulum.parse(item["updatedAt"]) <= current_end_date:
-                    yield item
+    # Create paginated resources dynamically
+    paginated_resources = [
+        _create_paginated_resource(
+            resource_name, query, query_field, api_key, start_date, end_date
+        )
+        for resource_name, query, query_field in PAGINATED_RESOURCES
+    ]
 
-    return issues, projects, teams, users
+    return [
+        *paginated_resources,
+        organization,
+    ]