castor-extractor 0.20.0__py3-none-any.whl → 0.20.5__py3-none-any.whl

castor_extractor/visualization/powerbi/client/rest.py

@@ -1,305 +0,0 @@
-import logging
-from datetime import date, datetime
-from time import sleep
-from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
-
-import msal  # type: ignore
-import requests
-
-from ....utils import at_midnight, format_date, yesterday
-from ..assets import PowerBiAsset
-from .constants import (
-    DEFAULT_TIMEOUT_IN_SECS,
-    GET,
-    POST,
-    SCAN_READY,
-    Batches,
-    Keys,
-    QueryParams,
-    Urls,
-)
-from .credentials import PowerbiCredentials
-from .utils import batch_size_is_valid_or_assert, datetime_is_recent_or_assert
-
-logger = logging.getLogger(__name__)
-
-
-def _time_filter(day: Optional[date]) -> Tuple[datetime, datetime]:
-    target_day = day or yesterday()
-    start = at_midnight(target_day)
-    end = datetime.combine(target_day, datetime.max.time())
-    return start, end
-
-
-def _url(
-    day: Optional[date],
-    continuation_uri: Optional[str],
-) -> str:
-    if continuation_uri:
-        return continuation_uri
-
-    url = Urls.ACTIVITY_EVENTS
-    start, end = _time_filter(day)
-    url += "?$filter=Activity eq 'viewreport'"
-    url += f"&startDateTime='{format_date(start)}'"
-    url += f"&endDateTime='{format_date(end)}'"
-    return url
-
-
-class Client:
-    """
-    PowerBI rest admin api
-    https://learn.microsoft.com/en-us/rest/api/power-bi/admin
-    """
-
-    def __init__(self, credentials: PowerbiCredentials):
-        self.creds = credentials
-        client_app = f"{Urls.CLIENT_APP_BASE}{self.creds.tenant_id}"
-        self.app = msal.ConfidentialClientApplication(
-            client_id=self.creds.client_id,
-            authority=client_app,
-            client_credential=self.creds.secret,
-        )
-
-    def _access_token(self) -> dict:
-        token = self.app.acquire_token_for_client(scopes=self.creds.scopes)
-
-        if Keys.ACCESS_TOKEN not in token:
-            raise ValueError(f"No access token in token response: {token}")
-
-        return token
-
-    def _header(self) -> Dict:
-        """Return header used in following rest api call"""
-        token = self._access_token()
-        return {"Authorization": f"Bearer {token[Keys.ACCESS_TOKEN]}"}
-
-    def _call(
-        self,
-        url: str,
-        method: str = GET,
-        *,
-        params: Optional[Dict] = None,
-        data: Optional[dict] = None,
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        """
-        Make either a get or a post http request.Request, by default
-        result.json is returned. Optionally you can provide a processor callback
-        to transform the result.
-        """
-        logger.debug(f"Calling {method} on {url}")
-        result = requests.request(
-            method,
-            url,
-            headers=self._header(),
-            params=params,
-            data=data,
-        )
-        result.raise_for_status()
-
-        if processor:
-            return processor(result)
-
-        return result.json()
-
-    def _get(
-        self,
-        url: str,
-        *,
-        params: Optional[Dict] = None,
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        return self._call(url, GET, params=params, processor=processor)
-
-    def _post(
-        self,
-        url: str,
-        *,
-        params: Optional[dict],
-        data: Optional[dict],
-        processor: Optional[Callable] = None,
-    ) -> Any:
-        return self._call(
-            url,
-            POST,
-            params=params,
-            data=data,
-            processor=processor,
-        )
-
-    def _workspace_ids(
-        self,
-        modified_since: Optional[datetime] = None,
-    ) -> List[str]:
-        """
-        Get workspaces ids from powerBI admin API.
-        If modified_since, take only workspaces that have been modified since
-
-        more: https://learn.microsoft.com/en-us/rest/api/power-bi/admin/workspace-info-get-modified-workspaces
-        """
-
-        def result_callback(call_result: requests.models.Response) -> List[str]:
-            return [x["id"] for x in call_result.json()]
-
-        params: Dict[str, Union[bool, str]] = {
-            Keys.INACTIVE_WORKSPACES: True,
-            Keys.PERSONAL_WORKSPACES: True,
-        }
-
-        if modified_since:
-            datetime_is_recent_or_assert(modified_since)
-            modified_since_iso = f"{modified_since.isoformat()}0Z"
-            params[Keys.MODIFIED_SINCE] = modified_since_iso
-
-        result = self._get(
-            Urls.WORKSPACE_IDS,
-            params=params,
-            processor=result_callback,
-        )
-
-        return result
-
-    def _create_scan(self, workspaces_ids: List[str]) -> int:
-        batch_size_is_valid_or_assert(workspaces_ids)
-        request_body = {"workspaces": workspaces_ids}
-        params = QueryParams.METADATA_SCAN
-        scan_id = self._post(
-            Urls.METADATA_POST,
-            params=params,
-            data=request_body,
-        )
-        return scan_id[Keys.ID]
-
-    def _wait_for_scan_result(self, scan_id: int) -> bool:
-        url = f"{Urls.METADATA_WAIT}/{scan_id}"
-        waiting_seconds = 0
-        sleep_seconds = 1
-        while True:
-            result = self._get(url, processor=lambda x: x)
-            if result.status_code != 200:
-                return False
-            if result.json()[Keys.STATUS] == SCAN_READY:
-                logger.info(f"scan {scan_id} ready")
-                return True
-            if waiting_seconds >= DEFAULT_TIMEOUT_IN_SECS:
-                break
-            waiting_seconds += sleep_seconds
-            logger.info(
-                f"Waiting {sleep_seconds} sec for scan {scan_id} to be ready…",
-            )
-            sleep(sleep_seconds)
-        return False
-
-    def _get_scan(self, scan_id: int) -> List[dict]:
-        url = f"{Urls.METADATA_GET}/{scan_id}"
-        return self._get(url)[Keys.WORKSPACES]
-
-    def _activity_events(
-        self,
-        *,
-        day: Optional[date] = None,
-        continuation_uri: Optional[str] = None,
-    ) -> List[Dict]:
-        """
-        Returns a list of activity events for the organization.
-        https://learn.microsoft.com/en-us/power-bi/admin/service-admin-auditing#activityevents-rest-api
-        - when no day is specified, fallback is yesterday
-        - continuation_uri allows to fetch paginated data (internal usage)
-        """
-        url = _url(day, continuation_uri)
-        answer = self._get(url)
-        activity_events = answer[Keys.ACTIVITY_EVENT_ENTITIES]
-        is_last = answer[Keys.LAST_RESULT_SET]
-        assert isinstance(is_last, bool)
-        if is_last:
-            return activity_events
-
-        # there are more data to fetch
-        # https://learn.microsoft.com/en-us/rest/api/power-bi/admin/get-activity-events#get-the-next-set-of-audit-activity-events-by-sending-the-continuation-token-to-the-api-example
-        continuation_uri = answer[Keys.CONTINUATION_URI]
-        rest = self._activity_events(continuation_uri=continuation_uri)
-        activity_events.extend(rest)
-        return activity_events
-
-    def _datasets(self) -> List[Dict]:
-        """
-        Returns a list of datasets for the organization.
-        https://learn.microsoft.com/en-us/rest/api/power-bi/admin/datasets-get-datasets-as-admin
-        """
-        return self._get(Urls.DATASETS)[Keys.VALUE]
-
-    def _reports(self) -> List[Dict]:
-        """
-        Returns a list of reports for the organization.
-        https://learn.microsoft.com/en-us/rest/api/power-bi/admin/reports-get-reports-as-admin
-        """
-        reports = self._get(Urls.REPORTS)[Keys.VALUE]
-        for report in reports:
-            report_id = report.get("id")
-            try:
-                url = Urls.REPORTS + f"/{report_id}/pages"
-                pages = self._get(url)[Keys.VALUE]
-                report["pages"] = pages
-            except (requests.HTTPError, requests.exceptions.Timeout) as e:
-                logger.debug(e)
-                continue
-        return reports
-
-    def _dashboards(self) -> List[Dict]:
-        """
-        Returns a list of dashboards for the organization.
-        https://learn.microsoft.com/en-us/rest/api/power-bi/admin/dashboards-get-dashboards-as-admin
-        """
-        return self._get(Urls.DASHBOARD)[Keys.VALUE]
-
-    def _metadata(
-        self,
-        modified_since: Optional[datetime] = None,
-    ) -> Iterator[List[Dict]]:
-        """
-        Fetch metadata by workspace.
-        https://learn.microsoft.com/en-us/power-bi/enterprise/service-admin-metadata-scanning
-        """
-        ids = self._workspace_ids(modified_since)
-
-        for ix in range(0, len(ids), Batches.METADATA):
-            batch_ids = [w_id for w_id in ids[ix : ix + Batches.METADATA]]
-            scan_id = self._create_scan(batch_ids)
-            self._wait_for_scan_result(scan_id)
-            yield self._get_scan(scan_id)
-
-    def test_connection(self) -> None:
-        """Use credentials & verify requesting the API doesn't raise an error"""
-        self._header()
-
-    def fetch(
-        self,
-        asset: PowerBiAsset,
-        *,
-        modified_since: Optional[datetime] = None,
-        day: Optional[date] = None,
-    ) -> List[Dict]:
-        """
-        Given a PowerBi asset, returns the corresponding data using the
-        appropriate client.
-        """
-        logger.info(f"Starting extraction of {asset}")
-        asset = PowerBiAsset(asset)
-
-        if asset == PowerBiAsset.ACTIVITY_EVENTS:
-            return self._activity_events(day=day)
-
-        if asset == PowerBiAsset.DATASETS:
-            return self._datasets()
-
-        if asset == PowerBiAsset.DASHBOARDS:
-            return self._dashboards()
-
-        if asset == PowerBiAsset.REPORTS:
-            return self._reports()
-
-        assert asset == PowerBiAsset.METADATA
-        return [
-            item for batch in self._metadata(modified_since) for item in batch
-        ]

castor_extractor/visualization/powerbi/client/rest_test.py

@@ -1,290 +0,0 @@
-from datetime import datetime, timedelta
-from unittest.mock import ANY, Mock, call, patch
-
-import pytest
-from requests import HTTPError
-
-from .constants import GET, POST, Assertions, Keys, QueryParams, Urls
-from .credentials import PowerbiCredentials
-from .rest import Client, msal
-
-FAKE_TENANT_ID = "IamFake"
-FAKE_CLIENT_ID = "MeTwo"
-FAKE_SECRET = "MeThree"
-
-
-def _client() -> Client:
-    creds = PowerbiCredentials(
-        tenant_id=FAKE_TENANT_ID,
-        client_id=FAKE_CLIENT_ID,
-        secret=FAKE_SECRET,
-    )
-    return Client(creds)
-
-
-def _raise_http_error() -> None:
-    raise HTTPError(request=Mock(), response=Mock())
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-def test__access_token(mock_app):
-    # init mocks
-    valid_response = {"access_token": "mock_token"}
-    returning_valid_token = Mock(return_value=valid_response)
-    mock_app.return_value.acquire_token_for_client = returning_valid_token
-
-    # init client
-    client = _client()
-
-    # generated token
-    assert client._access_token() == valid_response
-
-    # token missing in response
-    invalid_response = {"not_access_token": "666"}
-    returning_invalid_token = Mock(return_value=invalid_response)
-    mock_app.return_value.acquire_token_for_client = returning_invalid_token
-
-    with pytest.raises(ValueError):
-        client._access_token()
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch.object(Client, "_access_token")
-def test__headers(mock_access_token, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    mock_access_token.return_value = {Keys.ACCESS_TOKEN: "666"}
-    assert client._header() == {"Authorization": "Bearer 666"}
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch("requests.request")
-@patch.object(Client, "_access_token")
-def test__get(mocked_access_token, mocked_request, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    mocked_access_token.return_value = {Keys.ACCESS_TOKEN: "666"}
-    fact = {"fact": "Approximately 24 cat skins can make a coat.", "length": 43}
-    mocked_request.return_value = Mock(json=lambda: fact)
-
-    result = client._get("https://catfact.ninja/fact")
-    assert result == fact
-
-    result = client._get("https://catfact.ninja/fact")["length"]
-    assert result == 43
-
-    mocked_request.return_value = Mock(raise_for_status=_raise_http_error)
-
-    with pytest.raises(HTTPError):
-        result = client._get("https/whatev.er")
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch("requests.request")
-@patch.object(Client, "_access_token")
-def test__workspace_ids(_, mocked_request, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    mocked_request.return_value = Mock(
-        json=lambda: [{"id": 1000}, {"id": 1001}, {"id": 1003}],
-    )
-    ids = client._workspace_ids()
-    assert ids == [1000, 1001, 1003]
-
-    with pytest.raises(AssertionError, match=Assertions.DATETIME_TOO_OLD):
-        good_old_time = datetime(1998, 7, 12)
-        client._workspace_ids(modified_since=good_old_time)
-
-    yesterday = datetime.today() - timedelta(1)
-    ids = client._workspace_ids(modified_since=yesterday)
-    params = {
-        Keys.INACTIVE_WORKSPACES: True,
-        Keys.PERSONAL_WORKSPACES: True,
-        Keys.MODIFIED_SINCE: f"{yesterday.isoformat()}0Z",
-    }
-
-    mocked_request.assert_called_with(
-        GET,
-        Urls.WORKSPACE_IDS,
-        data=None,
-        headers=ANY,
-        params=params,
-    )
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch("requests.request")
-@patch.object(Client, "_access_token")
-def test__post_default(_, mocked_request, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    url = "https://estcequecestbientotleweekend.fr/"
-    params = QueryParams.METADATA_SCAN
-    data = {"bonjour": "hello"}
-    client._post(url, params=params, data=data)
-    mocked_request.assert_called_with(
-        POST,
-        url,
-        headers=ANY,
-        params=QueryParams.METADATA_SCAN,
-        data=data,
-    )
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch("requests.request")
-@patch.object(Client, "_access_token")
-def test__post_with_processor(_, mocked_request, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    url = "https://estcequecestbientotleweekend.fr/"
-    params = QueryParams.METADATA_SCAN
-    data = {"bonjour": "hello"}
-    mocked_request.return_value = Mock(json=lambda: {"id": 1000})
-    result = client._post(
-        url,
-        params=params,
-        data=data,
-        processor=lambda x: x.json()["id"],
-    )
-    assert result == 1000
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch("requests.request")
-@patch.object(Client, "_access_token")
-def test__datasets(_, mocked_request, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    mocked_request.return_value = Mock(
-        json=lambda: {"value": [{"id": 1, "type": "dataset"}]},
-    )
-    datasets = client._datasets()
-    mocked_request.assert_called_with(
-        GET,
-        Urls.DATASETS,
-        data=None,
-        headers=ANY,
-        params=None,
-    )
-    assert datasets == [{"id": 1, "type": "dataset"}]
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch("requests.request")
-@patch.object(Client, "_access_token")
-def test__reports(_, mocked_request, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    page_url = f"{Urls.REPORTS}/1/pages"
-    calls = [
-        call(GET, Urls.REPORTS, data=None, headers=ANY, params=None),
-        call(
-            GET,
-            page_url,
-            data=None,
-            headers=ANY,
-            params=None,
-        ),
-    ]
-    mocked_request.side_effect = [
-        Mock(json=lambda: {"value": [{"id": 1, "type": "report"}]}),
-        Mock(
-            json=lambda: {
-                "value": [
-                    {"name": "page_name", "displayName": "page", "order": 0}
-                ]
-            }
-        ),
-    ]
-    reports = client._reports()
-    mocked_request.assert_has_calls(calls)
-
-    assert reports == [
-        {
-            "id": 1,
-            "type": "report",
-            "pages": [{"name": "page_name", "displayName": "page", "order": 0}],
-        }
-    ]
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch("requests.request")
-@patch.object(Client, "_access_token")
-def test__dashboards(_, mocked_request, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    mocked_request.return_value = Mock(
-        json=lambda: {"value": [{"id": 1, "type": "dashboard"}]},
-    )
-    dashboards = client._dashboards()
-    mocked_request.assert_called_with(
-        GET,
-        Urls.DASHBOARD,
-        data=None,
-        headers=ANY,
-        params=None,
-    )
-    assert dashboards == [{"id": 1, "type": "dashboard"}]
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch.object(Client, "_workspace_ids")
-@patch.object(Client, "_create_scan")
-@patch.object(Client, "_wait_for_scan_result")
-@patch.object(Client, "_get_scan")
-def test__metadata(
-    mocked_get_scan,
-    mocked_wait,
-    mocked_create_scan,
-    mocked_workspace_ids,
-    mock_app,
-):
-    mock_app.return_value = None
-    mocked_workspace_ids.return_value = list(range(200))
-    mocked_create_scan.return_value = 314
-    mocked_wait.return_value = True
-    mocked_get_scan.return_value = [{"workspace_id": 1871}]
-
-    client = _client()
-    result = client._metadata()
-
-    assert list(result) == [[{"workspace_id": 1871}], [{"workspace_id": 1871}]]
-
-
-_CALLS = [
-    {
-        Keys.ACTIVITY_EVENT_ENTITIES: ["foo", "bar"],
-        Keys.LAST_RESULT_SET: False,
-        Keys.CONTINUATION_URI: "https://next-call-1",
-    },
-    {
-        Keys.ACTIVITY_EVENT_ENTITIES: ["baz"],
-        Keys.LAST_RESULT_SET: False,
-        Keys.CONTINUATION_URI: "https://next-call-2",
-    },
-    {
-        Keys.ACTIVITY_EVENT_ENTITIES: ["biz"],
-        Keys.LAST_RESULT_SET: True,
-        Keys.CONTINUATION_URI: None,
-    },
-]
-
-
-@patch.object(msal, "ConfidentialClientApplication")
-@patch.object(Client, "_call")
-def test__activity_events(mocked, mock_app):
-    mock_app.return_value = None
-    client = _client()
-    mocked.side_effect = _CALLS
-
-    result = client._activity_events()
-    assert result == ["foo", "bar", "baz", "biz"]
-
-    expected_calls = [
-        call(ANY, GET, params=None, processor=None),
-        call("https://next-call-1", GET, params=None, processor=None),
-        call("https://next-call-2", GET, params=None, processor=None),
-    ]
-    mocked.assert_has_calls(expected_calls)

castor_extractor/visualization/powerbi/client/utils.py

@@ -1,19 +0,0 @@
-from datetime import datetime, timedelta
-from typing import List
-
-from .constants import RECENT_DAYS, Assertions, Batches
-
-
-def batch_size_is_valid_or_assert(ids: List) -> None:
-    """
-    assert that current batch is smaller than expected size
-    """
-    assert len(ids) <= Batches.METADATA, Assertions.BATCH_TOO_BIG
-
-
-def datetime_is_recent_or_assert(dt: datetime) -> None:
-    """
-    assert that given datetime is recent
-    """
-    valid = dt > datetime.utcnow() - timedelta(RECENT_DAYS)
-    assert valid, Assertions.DATETIME_TOO_OLD

castor_extractor/visualization/powerbi/client/utils_test.py

@@ -1,24 +0,0 @@
-from datetime import datetime, timedelta
-
-import pytest
-
-from .constants import Assertions
-from .utils import batch_size_is_valid_or_assert, datetime_is_recent_or_assert
-
-
-def test_batch_size_is_valid_or_assert():
-    valid = [1, 3, 4]
-    batch_size_is_valid_or_assert(valid)
-
-    invalid = list(range(8000))
-    with pytest.raises(AssertionError, match=Assertions.BATCH_TOO_BIG):
-        batch_size_is_valid_or_assert(invalid)
-
-
-def test_datetime_is_recent_or_assert():
-    krach = datetime(1929, 10, 29)
-    with pytest.raises(AssertionError, match=Assertions.DATETIME_TOO_OLD):
-        datetime_is_recent_or_assert(krach)
-
-    yesterday = datetime.today() - timedelta(1)
-    datetime_is_recent_or_assert(yesterday)