unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.1__py3-none-any.whl

unstructured_ingest/connector/jira.py

@@ -1,469 +0,0 @@
-import math
-import typing as t
-from collections import abc
-from dataclasses import dataclass, field
-from datetime import datetime
-from functools import cached_property
-from pathlib import Path
-
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
-from unstructured_ingest.interfaces import (
-    AccessConfig,
-    BaseConnectorConfig,
-    BaseSessionHandle,
-    BaseSingleIngestDoc,
-    BaseSourceConnector,
-    ConfigSessionHandleMixin,
-    IngestDocCleanupMixin,
-    IngestDocSessionHandleMixin,
-    SourceConnectorCleanupMixin,
-    SourceMetadata,
-)
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-
-if t.TYPE_CHECKING:
-    from atlassian import Jira
-
-
-@dataclass
-class JiraSessionHandle(BaseSessionHandle):
-    service: "Jira"
-
-
-@requires_dependencies(["atlassian"], extras="jira")
-def create_jira_object(url, user_email, api_token):
-    """
-    Creates a jira object for interacting with Jira Cloud.
-    Args:
-        url: URL to Jira Cloud organization
-        user_email: Email for the user with the permissions
-        api_token: API Token, generated for the user
-
-    Returns:
-        Jira object
-    """
-    from atlassian import Jira
-
-    jira = Jira(
-        url,
-        username=user_email,
-        password=api_token,
-    )
-
-    response = jira.get_permissions("BROWSE_PROJECTS")
-    permitted = response["permissions"]["BROWSE_PROJECTS"]["havePermission"]
-
-    if permitted:
-        return jira
-
-    else:
-        raise ValueError(
-            """The user with the provided *user_email* and the *api_token*
-                         is not permitted to browse projects for the jira organization
-                         for the provided *url*. Try checking user_email, api_token,
-                         and the url arguments.""",
-        )
-
-
-@dataclass
-class JiraAccessConfig(AccessConfig):
-    api_token: str = enhanced_field(sensitive=True)
-
-
-@dataclass
-class SimpleJiraConfig(ConfigSessionHandleMixin, BaseConnectorConfig):
-    """Connector config where:
-    user_email is the email to authenticate into Atlassian (Jira) Cloud,
-    api_token is the api token to authenticate into Atlassian (Jira) Cloud,
-    url is the URL pointing to the Atlassian (Jira) Cloud instance,
-    list_of_projects is a list of project that is aimed to be ingested.
-
-    Check ...
-    for more info on the api_token.
-    """
-
-    user_email: str
-    access_config: JiraAccessConfig
-    url: str
-    projects: t.Optional[t.List[str]] = None
-    boards: t.Optional[t.List[str]] = None
-    issues: t.Optional[t.List[str]] = None
-
-    def create_session_handle(
-        self,
-    ) -> JiraSessionHandle:
-        service = create_jira_object(
-            url=self.url, user_email=self.user_email, api_token=self.access_config.api_token
-        )
-        return JiraSessionHandle(service=service)
-
-
-@dataclass
-class JiraFileMeta:
-    """Metadata specifying:
-    project_id: id for the jira project that the issue locates in, and
-    issue_key: key for the issue that is being reached to.
-    """
-
-    project_id: str
-    board_id: t.Optional[str]
-    issue_key: str
-    issue_id: str
-
-
-# An implementation to obtain nested-defaultdict functionality.
-# Keys have default values in a recursive manner, allowing
-# limitless templates to parse an api response object.
-def nested_object_to_field_getter(object):
-    if isinstance(object, abc.Mapping):
-        new_object = {}
-        for k, v in object.items():
-            if isinstance(v, abc.Mapping):
-                new_object[k] = FieldGetter(nested_object_to_field_getter(v))
-            else:
-                new_object[k] = v
-        return FieldGetter(new_object)
-    else:
-        return object
-
-
-class FieldGetter(dict):
-    def __getitem__(self, key):
-        value = super().__getitem__(key) if key in self else None
-        if value is None:
-            value = FieldGetter({})
-        return value
-
-
-def form_templated_string(issue, parsed_fields, c_sep="|||", r_sep="\n\n\n"):
-    """Forms a template string via parsing the fields from the API response object on the issue
-    The template string will be saved to the disk, and then will be processed by partition."""
-    return r_sep.join(
-        [
-            _get_id_fields_for_issue(issue),
-            _get_project_fields_for_issue(parsed_fields),
-            _get_dropdown_fields_for_issue(parsed_fields),
-            _get_subtasks_for_issue(parsed_fields),
-            _get_comments_for_issue(parsed_fields),
-            _get_text_fields_for_issue(parsed_fields),
-        ],
-    )
-
-
-DEFAULT_C_SEP = " " * 5
-DEFAULT_R_SEP = "\n"
-
-
-def _get_id_fields_for_issue(issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-    id, key = issue["id"], issue["key"]
-    return f"IssueID_IssueKey:{id}{c_sep}{key}{r_sep}"
-
-
-def _get_project_fields_for_issue(issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-    if "project" in issue:
-        return (
-            f"""ProjectID_Key:{issue["project"]["key"]}{c_sep}{issue["project"]["name"]}{r_sep}"""
-        )
-    else:
-        return ""
-
-
-def _get_dropdown_fields_for_issue(issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-    return f"""
-    IssueType:{issue["issuetype"]["name"]}
-    {r_sep}
-    Status:{issue["status"]["name"]}
-    {r_sep}
-    Priority:{issue["priority"]}
-    {r_sep}
-    AssigneeID_Name:{issue["assignee"]["accountId"]}{c_sep}{issue["assignee"]["displayName"]}
-    {r_sep}
-    ReporterAdr_Name:{issue["reporter"]["emailAddress"]}{c_sep}{issue["reporter"]["displayName"]}
-    {r_sep}
-    Labels:{c_sep.join(issue["labels"])}
-    {r_sep}
-    Components:{c_sep.join([component["name"] for component in issue["components"]])}
-    {r_sep}
-    """
-
-
-def _get_subtasks_for_issue(issue):
-    return ""
-
-
-def _get_text_fields_for_issue(issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-    return f"""
-    {issue["summary"]}
-    {r_sep}
-    {issue["description"]}
-    {r_sep}
-    {c_sep.join([atch["self"] for atch in issue["attachment"]])}
-    {r_sep}
-    """
-
-
-def _get_comments_for_issue(issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-    return c_sep.join(
-        [_get_fields_for_comment(comment) for comment in issue["comment"]["comments"]],
-    )
-
-
-def _get_fields_for_comment(comment, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-    return f"{comment['author']['displayName']}{c_sep}{comment['body']}{r_sep}"
-
-
-def scroll_wrapper(func, results_key="results"):
-    def wrapper(*args, **kwargs):
-        """Wraps a function to obtain scroll functionality.
-        Function needs to be able to accept 'start' and 'limit' arguments."""
-        if "number_of_items_to_fetch" in kwargs:
-            number_of_items_to_fetch = kwargs["number_of_items_to_fetch"]
-            del kwargs["number_of_items_to_fetch"]
-        else:
-            number_of_items_to_fetch = 100
-
-        kwargs["limit"] = min(100, number_of_items_to_fetch)
-        kwargs["start"] = kwargs.get("start", 0)
-
-        all_results = []
-        num_iterations = math.ceil(number_of_items_to_fetch / kwargs["limit"])
-
-        for _ in range(num_iterations):
-            response = func(*args, **kwargs)
-            if isinstance(response, list):
-                all_results += func(*args, **kwargs)
-            elif isinstance(response, dict):
-                if results_key not in response:
-                    raise KeyError(
-                        "Response object has no known keys to \
-                                   access the results, such as 'results' or 'values'.",
-                    )
-                all_results += func(*args, **kwargs)[results_key]
-            kwargs["start"] += kwargs["limit"]
-
-        return all_results[:number_of_items_to_fetch]
-
-    return wrapper
-
-
-@dataclass
-class JiraIngestDoc(IngestDocSessionHandleMixin, IngestDocCleanupMixin, BaseSingleIngestDoc):
-    """Class encapsulating fetching a doc and writing processed results (but not
-    doing the processing).
-
-    Current implementation creates a Jira connection object
-    to fetch each doc, rather than creating a it for each thread.
-    """
-
-    connector_config: SimpleJiraConfig
-    file_meta: t.Optional[JiraFileMeta] = None
-    registry_name: str = "jira"
-
-    @cached_property
-    def record_locator(self):  # Values must be JSON-serializable
-        """A dictionary with any data necessary to uniquely identify the document on
-        the source system."""
-        return {
-            "base_url": self.connector_config.url,
-            "issue_key": self.file_meta.issue_key,
-        }
-
-    @cached_property
-    @SourceConnectionNetworkError.wrap
-    def issue(self):
-        """Gets issue data"""
-        jira = self.session_handle.service
-        return jira.issue(self.file_meta.issue_key)
-
-    @cached_property
-    def parsed_fields(self):
-        return nested_object_to_field_getter(self.issue["fields"])
-
-    @property
-    def grouping_folder_name(self):
-        if self.file_meta.board_id:
-            return self.file_meta.board_id
-        else:
-            return self.file_meta.project_id
-
-    @property
-    def filename(self):
-        download_file = f"{self.file_meta.issue_id}.txt"
-
-        return (
-            Path(self.read_config.download_dir) / self.grouping_folder_name / download_file
-        ).resolve()
-
-    @property
-    def _output_filename(self):
-        """Create output file path."""
-        output_file = f"{self.file_meta.issue_id}.json"
-
-        return (
-            Path(self.processor_config.output_dir) / self.grouping_folder_name / output_file
-        ).resolve()
-
-    @property
-    def version(self) -> t.Optional[str]:
-        return None
-
-    def update_source_metadata(self, **kwargs) -> None:
-        exists = bool(self.issue)
-        if not exists:
-            self.source_metadata = SourceMetadata(
-                exists=exists,
-            )
-            return
-
-        self.source_metadata = SourceMetadata(
-            date_created=datetime.strptime(
-                self.parsed_fields["created"],
-                "%Y-%m-%dT%H:%M:%S.%f%z",
-            ).isoformat(),
-            date_modified=datetime.strptime(
-                self.parsed_fields["updated"],
-                "%Y-%m-%dT%H:%M:%S.%f%z",
-            ).isoformat(),
-            source_url=f"{self.connector_config.url}/browse/{self.file_meta.issue_key}",
-            exists=exists,
-        )
-
-    @SourceConnectionError.wrap
-    @requires_dependencies(["atlassian"], extras="jira")
-    @BaseSingleIngestDoc.skip_if_file_exists
-    def get_file(self):
-        document = form_templated_string(self.issue, self.parsed_fields)
-        self.update_source_metadata()
-        self.filename.parent.mkdir(parents=True, exist_ok=True)
-
-        with open(self.filename, "w", encoding="utf8") as f:
-            f.write(document)
-
-
-@dataclass
-class JiraSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
-    """Fetches issues from projects in an Atlassian (Jira) Cloud instance."""
-
-    connector_config: SimpleJiraConfig
-    _jira: t.Optional["Jira"] = field(init=False, default=None)
-
-    @property
-    def jira(self) -> "Jira":
-        if self._jira is None:
-            try:
-                self._jira = self.connector_config.create_session_handle().service
-            except Exception as e:
-                logger.error(f"failed to validate connection: {e}", exc_info=True)
-                raise SourceConnectionError(f"failed to validate connection: {e}")
-        return self._jira
-
-    @requires_dependencies(["atlassian"], extras="jira")
-    def initialize(self):
-        _ = self.jira
-
-    def check_connection(self):
-        _ = self.jira
-
-    @requires_dependencies(["atlassian"], extras="jira")
-    def _get_all_project_ids(self):
-        """Fetches ids for all projects in a Jira domain."""
-        project_ids = [project["key"] for project in self.jira.projects()]
-        return project_ids
-
-    @requires_dependencies(["atlassian"], extras="jira")
-    def _get_issues_within_one_project(
-        self,
-        project_id: str,
-    ):
-        get_issues_with_scroll = scroll_wrapper(self.jira.get_all_project_issues)
-        results = get_issues_with_scroll(project=project_id, fields=["key"])
-
-        return [(issue["key"], issue["id"], None) for issue in results]
-
-    @requires_dependencies(["atlassian"], extras="jira")
-    def _get_issue_keys_within_projects(self, project_ids: t.Optional[t.List[str]] = None):
-        if project_ids is None:
-            # for when a component list is provided, without any projects
-            if bool(self.connector_config.boards or self.connector_config.issues):
-                return []
-            # for when no components are provided. all projects will be ingested
-            else:
-                return self._get_all_project_ids()
-
-        # for when a component list is provided, including some projects
-        issue_keys_all = [self._get_issues_within_one_project(project_id=id) for id in project_ids]
-
-        issue_keys_flattened = [
-            (issue_key, issue_id, None)
-            for issue_keys_project in issue_keys_all
-            for issue_key, issue_id, board_id in issue_keys_project
-        ]
-
-        return issue_keys_flattened
-
-    def _get_issues_within_one_board(self, board_id: str):
-        get_issues_with_scroll = scroll_wrapper(
-            self.jira.get_issues_for_board,
-            results_key="issues",
-        )
-        results = get_issues_with_scroll(board_id=board_id, fields=["key"], jql=None)
-
-        return [(issue["key"], issue["id"], board_id) for issue in results]
-
-    def _get_issue_keys_within_boards(self, board_ids):
-        if board_ids is None:
-            return []
-
-        issue_keys_all = [self._get_issues_within_one_board(board_id=id) for id in board_ids]
-
-        issue_keys_flattened = [
-            (issue_key, issue_id, board_id)
-            for issue_keys_board in issue_keys_all
-            for issue_key, issue_id, board_id in issue_keys_board
-        ]
-        return issue_keys_flattened
-
-    def get_issues_info(self, issues):
-        issues_info = [self.jira.get_issue(issue, ["key", "id"]) for issue in issues]
-        return [(info["key"], info["id"], None) for info in issues_info]
-
-    def get_issue_keys_for_given_components(self):
-        issues = []
-
-        if self.connector_config.projects:
-            issues += self._get_issue_keys_within_projects(self.connector_config.projects)
-        if self.connector_config.boards:
-            issues += self._get_issue_keys_within_boards(self.connector_config.boards)
-        if self.connector_config.issues:
-            issues += self.get_issues_info(self.connector_config.issues)
-
-        return issues
-
-    def get_ingest_docs(self):
-        """Fetches all issues in a project."""
-        if bool(
-            self.connector_config.projects
-            or self.connector_config.boards
-            or self.connector_config.issues,
-        ):
-            issue_keys_and_ids = self.get_issue_keys_for_given_components()
-        else:
-            # gets all issue ids from all projects
-            issue_keys_and_ids = self._get_issue_keys_within_projects()
-
-        return [
-            JiraIngestDoc(
-                connector_config=self.connector_config,
-                processor_config=self.processor_config,
-                read_config=self.read_config,
-                file_meta=JiraFileMeta(
-                    issue_id=issue_id,
-                    issue_key=issue_key,
-                    project_id=issue_key.split("-")[0],
-                    board_id=board_id,
-                ),
-            )
-            for issue_key, issue_id, board_id in issue_keys_and_ids
-        ]