unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/connector/sharepoint.py

@@ -1,573 +0,0 @@
-import json
-import os
-import typing as t
-from dataclasses import dataclass
-from html import unescape
-from pathlib import Path
-from urllib.parse import urlparse
-
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
-from unstructured_ingest.interfaces import (
-    AccessConfig,
-    BaseConnectorConfig,
-    BaseSingleIngestDoc,
-    BaseSourceConnector,
-    IngestDocCleanupMixin,
-    SourceConnectorCleanupMixin,
-    SourceMetadata,
-)
-from unstructured_ingest.interfaces import PermissionsConfig as SharepointPermissionsConfig
-from unstructured_ingest.logger import logger
-from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.utils.string_and_date_utils import ensure_isoformat_datetime
-
-if t.TYPE_CHECKING:
-    from office365.sharepoint.client_context import ClientContext
-    from office365.sharepoint.files.file import File
-    from office365.sharepoint.publishing.pages.page import SitePage
-
-MAX_MB_SIZE = 512_000_000
-CONTENT_LABELS = ["CanvasContent1", "LayoutWebpartsContent1", "TimeCreated"]
-
-
-@dataclass
-class SharepointAccessConfig(AccessConfig):
-    client_cred: str = enhanced_field(repr=False, sensitive=True)
-
-
-@dataclass
-class SimpleSharepointConfig(BaseConnectorConfig):
-    access_config: SharepointAccessConfig
-    client_id: str
-    site: str
-    path: str
-    process_pages: bool = enhanced_field(default=True, init=False)
-    recursive: bool = False
-    files_only: bool = False
-    permissions_config: t.Optional[SharepointPermissionsConfig] = None
-
-    def __post_init__(self):
-        if not (self.client_id and self.access_config.client_cred and self.site):
-            raise ValueError(
-                "Please provide one of the following mandatory values:"
-                "\n--client-id\n--client-cred\n--site",
-            )
-        self.process_pages = not self.files_only
-
-    @requires_dependencies(["office365"], extras="sharepoint")
-    def get_site_client(self, site_url: str = "") -> "ClientContext":
-        from office365.runtime.auth.client_credential import ClientCredential
-        from office365.sharepoint.client_context import ClientContext
-
-        try:
-            site_client = ClientContext(site_url or self.site).with_credentials(
-                ClientCredential(self.client_id, self.access_config.client_cred),
-            )
-        except Exception:
-            logger.error("Couldn't set Sharepoint client.")
-            raise
-        return site_client
-
-    def get_permissions_client(self):
-        try:
-            permissions_connector = SharepointPermissionsConnector(self.permissions_config)
-            assert permissions_connector.access_token
-            return permissions_connector
-        except Exception as e:
-            logger.error("Couldn't obtain Sharepoint permissions ingestion access token:", e)
-
-
-@dataclass
-class SharepointIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
-    connector_config: SimpleSharepointConfig
-    site_url: str
-    server_path: str
-    is_page: bool
-    file_path: str
-    registry_name: str = "sharepoint"
-
-    def __post_init__(self):
-        self.extension = Path(self.file_path).suffix if not self.is_page else ".html"
-        self.extension = ".html" if self.extension == ".aspx" else self.extension
-        if not self.extension:
-            raise ValueError("Unsupported file without extension.")
-
-        self._set_download_paths()
-
-    def _set_download_paths(self) -> None:
-        """Parses the folder structure from the source and creates the download and output paths"""
-        download_path = Path(f"{self.read_config.download_dir}")
-        output_path = Path(f"{self.processor_config.output_dir}")
-        parent = Path(self.file_path).with_suffix(self.extension)
-        self.download_dir = (download_path / parent.parent).resolve()
-        self.download_filepath = (download_path / parent).resolve()
-        output_filename = str(parent) + ".json"
-        self.output_dir = (output_path / parent.parent).resolve()
-        self.output_filepath = (output_path / output_filename).resolve()
-
-    @property
-    def filename(self):
-        return Path(self.download_filepath).resolve()
-
-    @property
-    def _output_filename(self):
-        return Path(self.output_filepath).resolve()
-
-    @property
-    def record_locator(self) -> t.Optional[t.Dict[str, t.Any]]:
-        return {
-            "server_path": self.server_path,
-            "site_url": self.site_url,
-        }
-
-    @SourceConnectionNetworkError.wrap
-    @requires_dependencies(["office365"], extras="sharepoint")
-    def _fetch_file(self, properties_only: bool = False):
-        """Retrieves the actual page/file from the Sharepoint instance"""
-        from office365.runtime.client_request_exception import ClientRequestException
-
-        site_client = self.connector_config.get_site_client(self.site_url)
-
-        try:
-            if self.is_page:
-                file = site_client.web.get_file_by_server_relative_path("/" + self.server_path)
-                file = file.listItemAllFields.select(CONTENT_LABELS).get().execute_query()
-            else:
-                file = site_client.web.get_file_by_server_relative_url(self.server_path)
-                if properties_only:
-                    file = file.get().execute_query()
-        except ClientRequestException as e:
-            if e.response.status_code == 404:
-                return None
-            raise
-        return file
-
-    def _fetch_page(self):
-        site_client = self.connector_config.get_site_client(self.site_url)
-        try:
-            page = (
-                site_client.site_pages.pages.get_by_url(self.server_path)
-                .expand(["FirstPublished", "Modified", "Version"])
-                .get()
-                .execute_query()
-            )
-        except Exception as e:
-            logger.error(f"Failed to retrieve page {self.server_path} from site {self.site_url}")
-            logger.error(e)
-            return None
-        return page
-
-    def update_permissions_data(self):
-        def parent_name_matches(parent_type, permissions_filename, ingest_doc_filepath):
-            permissions_filename = permissions_filename.split("_SEP_")
-            ingest_doc_filepath = ingest_doc_filepath.split("/")
-
-            if parent_type == "sites":
-                return permissions_filename[0] == ingest_doc_filepath[1]
-
-            elif parent_type == "SitePages" or parent_type == "Shared Documents":
-                return True
-
-        permissions_data = None
-        permissions_dir = Path(self.processor_config.output_dir) / "permissions_data"
-
-        if permissions_dir.is_dir():
-            parent_type = self.file_path.split("/")[0]
-
-            if parent_type == "sites":
-                read_dir = permissions_dir / "sites"
-            elif parent_type == "SitePages" or parent_type == "Shared Documents":
-                read_dir = permissions_dir / "other"
-            else:
-                read_dir = permissions_dir / "other"
-
-            for filename in os.listdir(read_dir):
-                permissions_docname = os.path.splitext(filename)[0].split("_SEP_")[1]
-                ingestdoc_docname = self.file_path.split("/")[-1]
-
-                if ingestdoc_docname == permissions_docname and parent_name_matches(
-                    parent_type=parent_type,
-                    permissions_filename=filename,
-                    ingest_doc_filepath=self.file_path,
-                ):
-                    with open(read_dir / filename) as f:
-                        permissions_data = json.loads(f.read())
-
-        return permissions_data
-
-    def update_source_metadata(self, **kwargs):
-        if self.is_page:
-            page = self._fetch_page()
-            if page is None:
-                self.source_metadata = SourceMetadata(
-                    exists=False,
-                )
-                return
-            self.source_metadata = SourceMetadata(
-                date_created=page.get_property("FirstPublished", None),
-                date_modified=page.get_property("Modified", None),
-                version=page.get_property("Version", ""),
-                source_url=page.absolute_url,
-                exists=True,
-                permissions_data=(
-                    self.update_permissions_data()
-                    if self.connector_config.permissions_config
-                    else None
-                ),
-            )
-            return
-
-        file = self._fetch_file(True)
-        if file is None:
-            self.source_metadata = SourceMetadata(
-                exists=False,
-            )
-            return
-        self.source_metadata = SourceMetadata(
-            date_created=ensure_isoformat_datetime(timestamp=file.time_created),
-            date_modified=ensure_isoformat_datetime(timestamp=file.time_last_modified),
-            version=file.major_version,
-            source_url=file.properties.get("LinkingUrl", None),
-            exists=True,
-            permissions_data=(
-                self.update_permissions_data() if self.connector_config.permissions_config else None
-            ),
-        )
-
-    def _download_page(self):
-        """Formats and saves locally page content"""
-        content = self._fetch_file()
-        self.update_source_metadata()
-        pld = (content.properties.get("LayoutWebpartsContent1", "") or "") + (
-            content.properties.get("CanvasContent1", "") or ""
-        )
-        if pld != "":
-            pld = unescape(pld)
-        else:
-            logger.info(
-                f"Page {self.server_path} has no retrievable content. \
-                    Dumping empty doc.",
-            )
-            pld = "<div></div>"
-
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-        if not self.download_dir.is_dir():
-            logger.debug(f"creating directory: {self.download_dir}")
-            self.download_dir.mkdir(parents=True, exist_ok=True)
-        with self.filename.open(mode="w") as f:
-            f.write(pld)
-        logger.info(f"file downloaded: {self.filename}")
-
-    def _download_file(self):
-        file = self._fetch_file()
-        self.update_source_metadata()
-        fsize = file.length
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-
-        if not self.download_dir.is_dir():
-            logger.debug(f"creating directory: {self.download_dir}")
-            self.download_dir.mkdir(parents=True, exist_ok=True)
-
-        if fsize > MAX_MB_SIZE:
-            logger.info(f"downloading file with size: {fsize} bytes in chunks")
-            with self.filename.open(mode="wb") as f:
-                file.download_session(f, chunk_size=1024 * 1024 * 100).execute_query()
-        else:
-            with self.filename.open(mode="wb") as f:
-                file.download(f).execute_query()
-        logger.info(f"file downloaded: {self.filename}")
-
-    @BaseSingleIngestDoc.skip_if_file_exists
-    @SourceConnectionError.wrap
-    @requires_dependencies(["office365"])
-    def get_file(self):
-        if self.is_page:
-            self._download_page()
-        else:
-            self._download_file()
-        return
-
-
-@dataclass
-class SharepointSourceConnector(SourceConnectorCleanupMixin, BaseSourceConnector):
-    connector_config: SimpleSharepointConfig
-
-    def check_connection(self):
-        try:
-            site_client = self.connector_config.get_site_client()
-            site_client.site_pages.pages.get().execute_query()
-        except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
-            raise SourceConnectionError(f"failed to validate connection: {e}")
-
-    @requires_dependencies(["office365"], extras="sharepoint")
-    def _list_files(self, folder, recursive) -> t.List["File"]:
-        from office365.runtime.client_request_exception import ClientRequestException
-
-        try:
-            objects = folder.expand(["Files", "Folders"]).get().execute_query()
-            files = list(objects.files)
-            if not recursive:
-                return files
-            for f in objects.folders:
-                if "/Forms" in f.serverRelativeUrl:
-                    continue
-                files += self._list_files(f, recursive)
-            return files
-        except ClientRequestException as e:
-            if e.response.status_code != 404:
-                logger.info("Caught an error while processing documents %s", e.response.text)
-            return []
-
-    def _prepare_ingest_doc(self, obj: t.Union["File", "SitePage"], base_url, is_page=False):
-        if is_page:
-            file_path = obj.get_property("Url", "")
-            server_path = file_path if file_path[0] != "/" else file_path[1:]
-            if (url_path := (urlparse(base_url).path)) and (url_path != "/"):
-                file_path = url_path[1:] + "/" + file_path
-        else:
-            server_path = obj.serverRelativeUrl
-            file_path = obj.serverRelativeUrl[1:]
-
-        return SharepointIngestDoc(
-            processor_config=self.processor_config,
-            read_config=self.read_config,
-            connector_config=self.connector_config,
-            site_url=base_url,
-            server_path=server_path,
-            is_page=is_page,
-            file_path=file_path,
-        )
-
-    @requires_dependencies(["office365"], extras="sharepoint")
-    def _list_pages(self, site_client) -> list:
-        from office365.runtime.client_request_exception import ClientRequestException
-
-        try:
-            site_pages = site_client.site_pages.pages.get().execute_query()
-        except ClientRequestException as e:
-            logger.info(
-                "Caught an error while retrieving site pages from %s \n%s",
-                site_client.base_url,
-                e.response.text,
-            )
-            return []
-
-        return [self._prepare_ingest_doc(page, site_client.base_url, True) for page in site_pages]
-
-    def _ingest_site_docs(self, site_client) -> t.List["SharepointIngestDoc"]:
-        root_folder = site_client.web.get_folder_by_server_relative_path(self.connector_config.path)
-        files = self._list_files(root_folder, self.connector_config.recursive)
-        if not files:
-            logger.info(
-                f"No processable files at path {self.connector_config.path}\
-                for site {site_client.base_url}",
-            )
-        output = []
-        for file in files:
-            try:
-                output.append(self._prepare_ingest_doc(file, site_client.base_url))
-            except ValueError as e:
-                logger.error("Unable to process file %s", file.properties["Name"])
-                logger.error(e)
-        if self.connector_config.process_pages:
-            page_output = self._list_pages(site_client)
-            if not page_output:
-                logger.info(f"couldn't process pages for site {site_client.base_url}")
-            output = output + page_output
-        return output
-
-    def initialize(self):
-        pass
-
-    def get_ingest_docs(self):
-        base_site_client = self.connector_config.get_site_client()
-
-        if not all(
-            getattr(self.connector_config.permissions_config, attr, False)
-            for attr in ["application_id", "client_cred", "tenant"]
-        ):
-            logger.info(
-                "Permissions config is not fed with 'application_id', 'client_cred' and 'tenant'."
-                "Skipping permissions ingestion.",
-            )
-        else:
-            permissions_client = self.connector_config.get_permissions_client()
-            if permissions_client:
-                permissions_client.write_all_permissions(self.processor_config.output_dir)
-
-        if not base_site_client.is_tenant:
-            return self._ingest_site_docs(base_site_client)
-        tenant = base_site_client.tenant
-        tenant_sites = tenant.get_site_properties_from_sharepoint_by_filters().execute_query()
-        tenant_sites = {s.url for s in tenant_sites if (s.url is not None)}
-        ingest_docs: t.List[SharepointIngestDoc] = []
-        for site_url in tenant_sites:
-            logger.info(f"processing docs for site: {site_url}")
-            site_client = self.connector_config.get_site_client(site_url)
-            ingest_docs = ingest_docs + self._ingest_site_docs(site_client)
-        return ingest_docs
-
-
-@dataclass
-class SharepointPermissionsConnector:
-    def __init__(self, permissions_config):
-        self.permissions_config: SharepointPermissionsConfig = permissions_config
-        self.initialize()
-
-    def initialize(self):
-        self.access_token: str = self.get_access_token()
-
-    @requires_dependencies(["requests"], extras="sharepoint")
-    def get_access_token(self) -> str:
-        import requests
-
-        url = (
-            f"https://login.microsoftonline.com/{self.permissions_config.tenant}/oauth2/v2.0/token"
-        )
-        headers = {"Content-Type": "application/x-www-form-urlencoded"}
-        data = {
-            "client_id": self.permissions_config.application_id,
-            "scope": "https://graph.microsoft.com/.default",
-            "client_secret": self.permissions_config.client_cred,
-            "grant_type": "client_credentials",
-        }
-        response = requests.post(url, headers=headers, data=data)
-        return response.json()["access_token"]
-
-    def validated_response(self, response):
-        if response.status_code == 200:
-            return response.json()
-        else:
-            logger.info(f"request failed with status code {response.status_code}:")
-            logger.info(response.text)
-
-    @requires_dependencies(["requests"], extras="sharepoint")
-    def get_sites(self):
-        import requests
-
-        url = "https://graph.microsoft.com/v1.0/sites"
-        params = {
-            "$select": "webUrl, id",
-        }
-
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-        }
-
-        response = requests.get(url, params=params, headers=headers)
-        return self.validated_response(response)
-
-    @requires_dependencies(["requests"], extras="sharepoint")
-    def get_drives(self, site):
-        import requests
-
-        url = f"https://graph.microsoft.com/v1.0/sites/{site}/drives"
-
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-        }
-
-        response = requests.get(url, headers=headers)
-
-        return self.validated_response(response)
-
-    @requires_dependencies(["requests"], extras="sharepoint")
-    def get_drive_items(self, site, drive_id):
-        import requests
-
-        url = f"https://graph.microsoft.com/v1.0/sites/{site}/drives/{drive_id}/root/children"
-
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-        }
-
-        response = requests.get(url, headers=headers)
-
-        return self.validated_response(response)
-
-    def extract_site_name_from_weburl(self, weburl):
-        split_path = urlparse(weburl).path.lstrip("/").split("/")
-
-        if split_path[0] == "sites":
-            return "sites", split_path[1]
-
-        elif split_path[0] == "Shared%20Documents":
-            return "Shared Documents", "Shared Documents"
-
-        elif split_path[0] == "personal":
-            return "Personal", "Personal"
-
-        elif split_path[0] == "_layouts":
-            return "layouts", "layouts"
-
-        # if other weburl structures are found, additional logic might need to be implemented
-
-        logger.warning(
-            """Couldn't extract sitename, unknown site or parent type. Skipping permissions
-            ingestion for the document with the URL:""",
-            weburl,
-        )
-
-        return None, None
-
-    @requires_dependencies(["requests"], extras="sharepoint")
-    def get_permissions_for_drive_item(self, site, drive_id, item_id):
-        import requests
-
-        url = f"https://graph.microsoft.com/v1.0/sites/ \
-        {site}/drives/{drive_id}/items/{item_id}/permissions"
-
-        headers = {
-            "Authorization": f"Bearer {self.access_token}",
-        }
-
-        response = requests.get(url, headers=headers)
-
-        return self.validated_response(response)
-
-    def write_all_permissions(self, output_dir):
-        sites = [(site["id"], site["webUrl"]) for site in self.get_sites()["value"]]
-        drive_ids = []
-
-        logger.info("Obtaining drive data for sites for permissions (rbac)")
-        for site_id, site_url in sites:
-            drives = self.get_drives(site_id)
-            if drives:
-                drives_for_site = drives["value"]
-                drive_ids.extend([(site_id, drive["id"]) for drive in drives_for_site])
-
-        logger.info("Obtaining item data from drives for permissions (rbac)")
-        item_ids = []
-        for site, drive_id in drive_ids:
-            drive_items = self.get_drive_items(site, drive_id)
-            if drive_items:
-                item_ids.extend(
-                    [
-                        (site, drive_id, item["id"], item["name"], item["webUrl"])
-                        for item in drive_items["value"]
-                    ],
-                )
-
-        permissions_dir = Path(output_dir) / "permissions_data"
-
-        logger.info("Writing permissions data to disk")
-        for site, drive_id, item_id, item_name, item_web_url in item_ids:
-            res = self.get_permissions_for_drive_item(site, drive_id, item_id)
-            if res:
-                parent_type, parent_name = self.extract_site_name_from_weburl(item_web_url)
-
-                if parent_type == "sites":
-                    write_path = permissions_dir / "sites" / f"{parent_name}_SEP_{item_name}.json"
-
-                elif parent_type == "Personal" or parent_type == "Shared Documents":
-                    write_path = permissions_dir / "other" / f"{parent_name}_SEP_{item_name}.json"
-                else:
-                    write_path = permissions_dir / "other" / f"{parent_name}_SEP_{item_name}.json"
-
-                if not Path(os.path.dirname(write_path)).is_dir():
-                    os.makedirs(os.path.dirname(write_path))
-
-                with open(write_path, "w") as f:
-                    json.dump(res["value"], f)