castor-extractor 0.5.0__tar.gz → 0.5.2__tar.gz

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.5.2 - 2023-07-12
+* Fix Metabase DbClient url
+
+## 0.5.1 - 2023-07-03
+* Add support for Looker's `ContentViews`
+
 ## 0.5.0 - 2023-06-28
 
 * Stop supporting python3.7

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.5.0
+Version: 0.5.2
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/castor_extractor/utils/__init__.py

@@ -17,7 +17,7 @@ from .retry import RetryStrategy, retry
 from .safe import SafeMode, safe_mode
 from .store import AbstractStorage, LocalStorage
 from .string import string_to_tuple
-from .time import current_date, current_datetime, current_timestamp
+from .time import current_date, current_datetime, current_timestamp, past_date
 from .type import Callback, Getter, JsonType, SerializedAsset
 from .uri import uri_encode
 from .validation import validate_baseurl

castor_extractor-0.5.2/castor_extractor/utils/time.py

@@ -0,0 +1,38 @@
+from datetime import date, datetime, timedelta, timezone
+
+
+def current_datetime() -> datetime:
+    """Returns the current datetime"""
+    return datetime.utcnow()
+
+
+def current_date() -> date:
+    """Returns the current datetime"""
+    return current_datetime().date()
+
+
+def current_timestamp() -> int:
+    """
+    Returns the current timestamp from epoch (rounded to the nearest second)
+    """
+    return int(datetime.timestamp(current_datetime()))
+
+
+def _set_uct_timezone(ts: datetime) -> datetime:
+    if ts.tzinfo is None:
+        return ts.replace(tzinfo=timezone.utc)
+    return ts
+
+
+def now(tz: bool = False) -> datetime:
+    """
+    provide current time
+    optionally localize timezone
+    """
+    ts = datetime.utcnow()
+    return _set_uct_timezone(ts) if tz else ts
+
+
+def past_date(past_days: int) -> date:
+    """returns a date in the past"""
+    return now().date() - timedelta(past_days)

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/castor_extractor/visualization/looker/api/client.py

@@ -1,11 +1,18 @@
+import functools
 import logging
+from datetime import date, timedelta
 from typing import Callable, Iterator, List, Optional, Sequence, Tuple
 
-from ....utils import Pager, PagerLogger, SafeMode, safe_mode
+from dateutil.utils import today
+from looker_sdk.sdk.api40.models import ContentView
+
+from ....utils import Pager, PagerLogger, SafeMode, past_date, safe_mode
 from ..env import page_size
 from ..fields import format_fields
 from .constants import (
     CONNECTION_FIELDS,
+    CONTENT_VIEWS_FIELDS,
+    CONTENT_VIEWS_HISTORY_DAYS,
     DASHBOARD_FIELDS,
     FOLDER_FIELDS,
     GROUPS_HIERARCHY_FIELDS,
@@ -38,6 +45,20 @@ logger = logging.getLogger(__name__)
 OnApiCall = Callable[[], None]
 
 
+def _mondays(history_depth_in_days: int) -> Iterator[date]:
+    """
+    Fetch all Mondays of the elapsed period
+    """
+    _MSG = "History depth must be strictly positive"
+    assert history_depth_in_days > 0, _MSG
+    current_date = past_date(history_depth_in_days)
+    end_date = today().date()
+    while current_date < end_date:
+        if current_date.weekday() == 0:
+            yield current_date
+        current_date += timedelta(days=1)
+
+
 class ApiPagerLogger(PagerLogger):
     def __init__(self, on_api_call: Optional[OnApiCall]):
         self._on_api_call = on_api_call
@@ -217,3 +238,30 @@ class ApiClient:
         )
         logger.info("All looker groups_roles fetched")
         return list(groups_roles)
+
+    def content_views(self) -> List[ContentView]:
+        """
+        List the number of views per {user x week x dashboard|look}
+        https://cloud.google.com/looker/docs/reference/looker-api/latest/types/ContentView
+        """
+        content_views: List[ContentView] = []
+
+        for day in _mondays(history_depth_in_days=CONTENT_VIEWS_HISTORY_DAYS):
+            formatted_day = day.strftime("%Y-%m-%d")
+            logger.info(f"Fetching content views for week {formatted_day}")
+
+            _fetch = functools.partial(
+                self._sdk.search_content_views,
+                fields=format_fields(CONTENT_VIEWS_FIELDS),
+                start_of_week_date=formatted_day,
+                user_id="NOT NULL",
+            )
+            look_views = list(_fetch(look_id="NOT NULL"))
+            dashboard_views = list(_fetch(dashboard_id="NOT NULL"))
+
+            content_views.extend(look_views + dashboard_views)
+
+        logger.info(
+            f"All looker content views fetched - {len(content_views)} rows"
+        )
+        return content_views

castor_extractor-0.5.2/castor_extractor/visualization/looker/api/client_test.py

@@ -0,0 +1,65 @@
+import datetime
+from unittest.mock import patch
+
+import pytest
+from castor_extractor.visualization.looker import (  # type: ignore
+    ApiClient,
+    Credentials,
+)
+from dateutil.utils import today
+from freezegun import freeze_time
+from source.packages.extractor.castor_extractor.visualization.looker.api.client import (
+    _mondays,
+)
+
+
+def _credentials():
+    return Credentials(
+        base_url="base_url",
+        client_id="client_id",
+        client_secret="secret",
+    )
+
+
+@patch("castor_extractor.visualization.looker.api.client.init40")
+@patch("castor_extractor.visualization.looker.api.client.has_admin_permissions")
+def test_api_client_has_admin_permissions(
+    mock_has_admin_permission, mock_init40
+):
+    mock_has_admin_permission.return_value = False
+    with pytest.raises(PermissionError):
+        ApiClient(_credentials())
+
+    mock_has_admin_permission.return_value = True
+    mock_init40.return_value = "sdk"
+    client = ApiClient(_credentials())
+    assert client._sdk == "sdk"
+
+
+@freeze_time("2023-7-4")
+def test__mondays():
+    expected_30_days = {
+        datetime.date(2023, 7, 3),
+        datetime.date(2023, 6, 26),
+        datetime.date(2023, 6, 19),
+        datetime.date(2023, 6, 12),
+        datetime.date(2023, 6, 5),
+    }
+    assert set(_mondays(history_depth_in_days=30)) == expected_30_days
+
+    # only mondays
+    assert all([day.weekday() == 0 for day in _mondays(30)])
+    assert all([day.weekday() == 0 for day in _mondays(100)])
+
+    # all days must remain in the elapsed history
+    history_days = 1_000
+    end = today().date()
+    start = end - datetime.timedelta(days=history_days)
+    output = _mondays(history_depth_in_days=history_days)
+
+    assert all([day >= start for day in output])
+    assert all([day < end for day in output])
+
+    with pytest.raises(AssertionError):
+        list(_mondays(history_depth_in_days=0))
+        list(_mondays(history_depth_in_days=-5))

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/castor_extractor/visualization/looker/api/constants.py

@@ -208,6 +208,15 @@ GROUPS_ROLES_FIELDS = (
     },
 )
 
+CONTENT_VIEWS_FIELDS = (
+    "dashboard_id",
+    "look_id",
+    "start_of_week_date",
+    "user_id",
+    "view_count",
+)
+CONTENT_VIEWS_HISTORY_DAYS = 30
+
 
 # Model from looker
 LOOKML_PROJECT_NAME_BLOCKLIST = ("looker-data", "system__activity")

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/castor_extractor/visualization/looker/assets.py

@@ -4,13 +4,14 @@ from enum import Enum
 class LookerAsset(Enum):
     """Looker assets"""
 
+    CONNECTIONS = "connections"
+    CONTENT_VIEWS = "content_views"
     DASHBOARDS = "dashboards"
     EXPLORES = "explores"
     FOLDERS = "folders"
-    LOOKS = "looks"
-    LOOKML_MODELS = "lookml_models"
-    USERS = "users"
-    CONNECTIONS = "connections"
-    PROJECTS = "projects"
     GROUPS_HIERARCHY = "groups_hierarchy"
     GROUPS_ROLES = "groups_roles"
+    LOOKML_MODELS = "lookml_models"
+    LOOKS = "looks"
+    PROJECTS = "projects"
+    USERS = "users"

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/castor_extractor/visualization/looker/extract.py

@@ -80,6 +80,10 @@ def iterate_all_data(
     groups_roles = client.groups_roles()
     yield LookerAsset.GROUPS_ROLES, deep_serialize(groups_roles)
 
+    logger.info("Extracting content views from Looker API")
+    content_views = client.content_views()
+    yield LookerAsset.CONTENT_VIEWS, deep_serialize(content_views)
+
 
 def extract_all(**kwargs) -> None:
     """

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/castor_extractor/visualization/metabase/client/db/client.py

@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 
-PG_URL = "postgresql://{username}:{password}@{host}:{port}/{database}"
+PG_URL = "postgresql://{user}:{password}@{host}:{port}/{database}"
 SQL_FILE_PATH = "queries/{name}.sql"
 
 ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY"

{castor_extractor-0.5.0 → castor_extractor-0.5.2}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.5.0"
+version = "0.5.2"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"

castor_extractor-0.5.0/castor_extractor/utils/time.py

@@ -1,18 +0,0 @@
-from datetime import date, datetime
-
-
-def current_datetime() -> datetime:
-    """Returns the current datetime"""
-    return datetime.utcnow()
-
-
-def current_date() -> date:
-    """Returns the current datetime"""
-    return current_datetime().date()
-
-
-def current_timestamp() -> int:
-    """
-    Returns the current timestamp from epoch (rounded to the nearest second)
-    """
-    return int(datetime.timestamp(current_datetime()))

castor_extractor-0.5.0/castor_extractor/visualization/looker/api/client_test.py

@@ -1,30 +0,0 @@
-from unittest.mock import patch
-
-import pytest
-from castor_extractor.visualization.looker import (  # type: ignore
-    ApiClient,
-    Credentials,
-)
-
-
-def _credentials():
-    return Credentials(
-        base_url="base_url",
-        client_id="client_id",
-        client_secret="secret",
-    )
-
-
-@patch("castor_extractor.visualization.looker.api.client.init40")
-@patch("castor_extractor.visualization.looker.api.client.has_admin_permissions")
-def test_api_client_has_admin_permissions(
-    mock_has_admin_permission, mock_init40
-):
-    mock_has_admin_permission.return_value = False
-    with pytest.raises(PermissionError):
-        ApiClient(_credentials())
-
-    mock_has_admin_permission.return_value = True
-    mock_init40.return_value = "sdk"
-    client = ApiClient(_credentials())
-    assert client._sdk == "sdk"