castor-extractor 0.5.0__py3-none-any.whl → 0.5.2__py3-none-any.whl

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.5.2 - 2023-07-12
+* Fix Metabase DbClient url
+
+## 0.5.1 - 2023-07-03
+* Add support for Looker's `ContentViews`
+
 ## 0.5.0 - 2023-06-28
 
 * Stop supporting python3.7

castor_extractor/utils/__init__.py

@@ -17,7 +17,7 @@ from .retry import RetryStrategy, retry
 from .safe import SafeMode, safe_mode
 from .store import AbstractStorage, LocalStorage
 from .string import string_to_tuple
-from .time import current_date, current_datetime, current_timestamp
+from .time import current_date, current_datetime, current_timestamp, past_date
 from .type import Callback, Getter, JsonType, SerializedAsset
 from .uri import uri_encode
 from .validation import validate_baseurl

castor_extractor/utils/time.py

@@ -1,4 +1,4 @@
-from datetime import date, datetime
+from datetime import date, datetime, timedelta, timezone
 
 
 def current_datetime() -> datetime:
@@ -16,3 +16,23 @@ def current_timestamp() -> int:
     Returns the current timestamp from epoch (rounded to the nearest second)
     """
     return int(datetime.timestamp(current_datetime()))
+
+
+def _set_uct_timezone(ts: datetime) -> datetime:
+    if ts.tzinfo is None:
+        return ts.replace(tzinfo=timezone.utc)
+    return ts
+
+
+def now(tz: bool = False) -> datetime:
+    """
+    provide current time
+    optionally localize timezone
+    """
+    ts = datetime.utcnow()
+    return _set_uct_timezone(ts) if tz else ts
+
+
+def past_date(past_days: int) -> date:
+    """returns a date in the past"""
+    return now().date() - timedelta(past_days)

castor_extractor/visualization/looker/api/client.py

@@ -1,11 +1,18 @@
+import functools
 import logging
+from datetime import date, timedelta
 from typing import Callable, Iterator, List, Optional, Sequence, Tuple
 
-from ....utils import Pager, PagerLogger, SafeMode, safe_mode
+from dateutil.utils import today
+from looker_sdk.sdk.api40.models import ContentView
+
+from ....utils import Pager, PagerLogger, SafeMode, past_date, safe_mode
 from ..env import page_size
 from ..fields import format_fields
 from .constants import (
     CONNECTION_FIELDS,
+    CONTENT_VIEWS_FIELDS,
+    CONTENT_VIEWS_HISTORY_DAYS,
     DASHBOARD_FIELDS,
     FOLDER_FIELDS,
     GROUPS_HIERARCHY_FIELDS,
@@ -38,6 +45,20 @@ logger = logging.getLogger(__name__)
 OnApiCall = Callable[[], None]
 
 
+def _mondays(history_depth_in_days: int) -> Iterator[date]:
+    """
+    Fetch all Mondays of the elapsed period
+    """
+    _MSG = "History depth must be strictly positive"
+    assert history_depth_in_days > 0, _MSG
+    current_date = past_date(history_depth_in_days)
+    end_date = today().date()
+    while current_date < end_date:
+        if current_date.weekday() == 0:
+            yield current_date
+        current_date += timedelta(days=1)
+
+
 class ApiPagerLogger(PagerLogger):
     def __init__(self, on_api_call: Optional[OnApiCall]):
         self._on_api_call = on_api_call
@@ -217,3 +238,30 @@ class ApiClient:
         )
         logger.info("All looker groups_roles fetched")
         return list(groups_roles)
+
+    def content_views(self) -> List[ContentView]:
+        """
+        List the number of views per {user x week x dashboard|look}
+        https://cloud.google.com/looker/docs/reference/looker-api/latest/types/ContentView
+        """
+        content_views: List[ContentView] = []
+
+        for day in _mondays(history_depth_in_days=CONTENT_VIEWS_HISTORY_DAYS):
+            formatted_day = day.strftime("%Y-%m-%d")
+            logger.info(f"Fetching content views for week {formatted_day}")
+
+            _fetch = functools.partial(
+                self._sdk.search_content_views,
+                fields=format_fields(CONTENT_VIEWS_FIELDS),
+                start_of_week_date=formatted_day,
+                user_id="NOT NULL",
+            )
+            look_views = list(_fetch(look_id="NOT NULL"))
+            dashboard_views = list(_fetch(dashboard_id="NOT NULL"))
+
+            content_views.extend(look_views + dashboard_views)
+
+        logger.info(
+            f"All looker content views fetched - {len(content_views)} rows"
+        )
+        return content_views

castor_extractor/visualization/looker/api/client_test.py

@@ -1,3 +1,4 @@
+import datetime
 from unittest.mock import patch
 
 import pytest
@@ -5,6 +6,11 @@ from castor_extractor.visualization.looker import (  # type: ignore
     ApiClient,
     Credentials,
 )
+from dateutil.utils import today
+from freezegun import freeze_time
+from source.packages.extractor.castor_extractor.visualization.looker.api.client import (
+    _mondays,
+)
 
 
 def _credentials():
@@ -28,3 +34,32 @@ def test_api_client_has_admin_permissions(
     mock_init40.return_value = "sdk"
     client = ApiClient(_credentials())
     assert client._sdk == "sdk"
+
+
+@freeze_time("2023-7-4")
+def test__mondays():
+    expected_30_days = {
+        datetime.date(2023, 7, 3),
+        datetime.date(2023, 6, 26),
+        datetime.date(2023, 6, 19),
+        datetime.date(2023, 6, 12),
+        datetime.date(2023, 6, 5),
+    }
+    assert set(_mondays(history_depth_in_days=30)) == expected_30_days
+
+    # only mondays
+    assert all([day.weekday() == 0 for day in _mondays(30)])
+    assert all([day.weekday() == 0 for day in _mondays(100)])
+
+    # all days must remain in the elapsed history
+    history_days = 1_000
+    end = today().date()
+    start = end - datetime.timedelta(days=history_days)
+    output = _mondays(history_depth_in_days=history_days)
+
+    assert all([day >= start for day in output])
+    assert all([day < end for day in output])
+
+    with pytest.raises(AssertionError):
+        list(_mondays(history_depth_in_days=0))
+        list(_mondays(history_depth_in_days=-5))

castor_extractor/visualization/looker/api/constants.py

@@ -208,6 +208,15 @@ GROUPS_ROLES_FIELDS = (
     },
 )
 
+CONTENT_VIEWS_FIELDS = (
+    "dashboard_id",
+    "look_id",
+    "start_of_week_date",
+    "user_id",
+    "view_count",
+)
+CONTENT_VIEWS_HISTORY_DAYS = 30
+
 
 # Model from looker
 LOOKML_PROJECT_NAME_BLOCKLIST = ("looker-data", "system__activity")

castor_extractor/visualization/looker/assets.py

@@ -4,13 +4,14 @@ from enum import Enum
 class LookerAsset(Enum):
     """Looker assets"""
 
+    CONNECTIONS = "connections"
+    CONTENT_VIEWS = "content_views"
     DASHBOARDS = "dashboards"
     EXPLORES = "explores"
     FOLDERS = "folders"
-    LOOKS = "looks"
-    LOOKML_MODELS = "lookml_models"
-    USERS = "users"
-    CONNECTIONS = "connections"
-    PROJECTS = "projects"
     GROUPS_HIERARCHY = "groups_hierarchy"
     GROUPS_ROLES = "groups_roles"
+    LOOKML_MODELS = "lookml_models"
+    LOOKS = "looks"
+    PROJECTS = "projects"
+    USERS = "users"

castor_extractor/visualization/looker/extract.py

@@ -80,6 +80,10 @@ def iterate_all_data(
     groups_roles = client.groups_roles()
     yield LookerAsset.GROUPS_ROLES, deep_serialize(groups_roles)
 
+    logger.info("Extracting content views from Looker API")
+    content_views = client.content_views()
+    yield LookerAsset.CONTENT_VIEWS, deep_serialize(content_views)
+
 
 def extract_all(**kwargs) -> None:
     """

castor_extractor/visualization/metabase/client/db/client.py

@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 
-PG_URL = "postgresql://{username}:{password}@{host}:{port}/{database}"
+PG_URL = "postgresql://{user}:{password}@{host}:{port}/{database}"
 SQL_FILE_PATH = "queries/{name}.sql"
 
 ENCRYPTION_SECRET_KEY = "CASTOR_METABASE_ENCRYPTION_SECRET_KEY"

{castor_extractor-0.5.0.dist-info → castor_extractor-0.5.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.5.0
+Version: 0.5.2
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA

{castor_extractor-0.5.0.dist-info → castor_extractor-0.5.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-CHANGELOG.md,sha256=JyLIQXxoZhPRtgV6sNO8zNWIV5Y3DfagWytL-MpT1Dg,5240
+CHANGELOG.md,sha256=ud3JHRlNCrRF1T77IUh5tSEI1ezW8RqcWzYWAngh1m4,5356
 LICENCE,sha256=sL-IGa4hweyya1HgzMskrRdybbIa2cktzxb5qmUgDg8,8254
 README.md,sha256=EL6JpZxvaQFOYv5WFuSjZvSk9Hcpsf7alMlUC5IPFjA,3423
 castor_extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -43,7 +43,7 @@ castor_extractor/uploader/env_test.py,sha256=ClCWWtwd2N-5ClIDUxVMeKkWfhhOTxpppsX
 castor_extractor/uploader/upload.py,sha256=NTXF4BlVEsRCA6S-SunyNHkcXN9JaRpK1W65ZeMWAcw,3353
 castor_extractor/uploader/upload_test.py,sha256=WzGtYoe45-Q_AAR_gj7aOZU_C342GdyZxiUHwITi2oU,329
 castor_extractor/uploader/utils.py,sha256=NCe0tkB28BVhqzOaDhDjaSfODjjcPWB17X6chnvyCWs,478
-castor_extractor/utils/__init__.py,sha256=R8P4994ZN8E7O1uavThDAP1qQO73eii9RfnWOsq3UKE,918
+castor_extractor/utils/__init__.py,sha256=cENngVhgk311f4xvmaRlqO5cu5lERTrdZm0r_bJCrJk,929
 castor_extractor/utils/collection.py,sha256=uenJvfamphxV5ZFt12BgfsRs99pWffYJIMjAD_Laz2Q,417
 castor_extractor/utils/constants.py,sha256=qBQprS9U66mS-RIBXiLujdTSV3WvGv40Bc0khP4Abdk,39
 castor_extractor/utils/deprecate.py,sha256=J-UzSNPGyIBW0oAEnQae6zLL6kgqUNmjtrS0K21fb6s,818
@@ -66,7 +66,7 @@ castor_extractor/utils/safe_test.py,sha256=IHN1Z761tYMFslYC-2HAfkXmFPh4LYSqNLs4Q
 castor_extractor/utils/store.py,sha256=mlfg0sS4azJ1eg28D_cWUWkF-npy3R9d4VPC4EAVNnI,2092
 castor_extractor/utils/string.py,sha256=-fXrp4CLJoECwOaYTuv2d8vatA74EI22URschGD8pvo,1961
 castor_extractor/utils/string_test.py,sha256=5n4leOBxphtnyD0iHajzsTog4vLlYk8h8LDYV4ZMSeM,2206
-castor_extractor/utils/time.py,sha256=8eTRskem2HLbhTGoNPB9R_enKu9ZwvTHMSW8UkxZe2g,430
+castor_extractor/utils/time.py,sha256=yQC2dOTadqPnOgk-50OJ2Nt_0wD3ySU8XuwdXFWIiNU,903
 castor_extractor/utils/type.py,sha256=87t32cTctEjX-_BqZLtPLWu-M9OVvw_lFU4DbaQ6V0U,313
 castor_extractor/utils/uri.py,sha256=jmP9hY-6PRqdc3-vAOdtll_U6q9VCqSqmBAN6QRs3ZI,150
 castor_extractor/utils/uri_test.py,sha256=1XKF6qSseCeD4G4ckaNO07JXfGbt7XUVinOZdpEYrDQ,259
@@ -76,16 +76,16 @@ castor_extractor/utils/write.py,sha256=CbLMz-mkUFduwogERwe69GXXVO65HEDirVm-kDJkR
 castor_extractor/visualization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 castor_extractor/visualization/looker/__init__.py,sha256=F1DkZQStdmPRc2AwurWL0Hhaz2SRW9ERfZLvMcBJc0g,191
 castor_extractor/visualization/looker/api/__init__.py,sha256=eMtzoMAE7ZnvfwJJAOh9Cooztj4Juzc40wSEHhIYXw4,181
-castor_extractor/visualization/looker/api/client.py,sha256=iagryhCNvYAlPTpPmQQx4L6WrHVLSDjchb-rMpbb0cc,7137
-castor_extractor/visualization/looker/api/client_test.py,sha256=hww1Yj38lndkxUtF40oRfA3bTWkA47xMQhVS-22HZDc,816
-castor_extractor/visualization/looker/api/constants.py,sha256=urLKuLQ6HXX1xeD5EVZArd9ENYZJ0EMrSP92b-MJIso,3895
+castor_extractor/visualization/looker/api/client.py,sha256=V-TcUCU65BTMUbXInMP5SQKuQDbylpFc9af4O5KyjL8,8859
+castor_extractor/visualization/looker/api/client_test.py,sha256=LlQCI_Yuu4DWmT1_lcgZP6-BZMCzgz9--F9TV0lrPCk,1924
+castor_extractor/visualization/looker/api/constants.py,sha256=__ExzGkhnVadOeIplEgHixqzM4lmZWiOZnC0GRZXLTg,4049
 castor_extractor/visualization/looker/api/sdk.py,sha256=MMHFABzpWg84pSVDqYvaDxjKB50Pr7ttNUqrcdtR-po,3526
 castor_extractor/visualization/looker/api/sdk_test.py,sha256=NHtKZTflPhqzBFHs1TyAQaubgxfzLLwYKFT8rEqR55I,1742
 castor_extractor/visualization/looker/api/utils.py,sha256=vWSz6iFFF4uA3Y3eqqGEH1CkXcZbtHqVFMnPS2o8LCQ,1320
-castor_extractor/visualization/looker/assets.py,sha256=HF_gXewUsdztSr9v7-Z-FTWmVzf1Fwn0Tf-8BIvgIfE,364
+castor_extractor/visualization/looker/assets.py,sha256=50Tr3RrWzbGZCpUMkNK42wnBJMUkKdlCW1IT2KHyt3A,400
 castor_extractor/visualization/looker/constant.py,sha256=ZqgUJxsSkyoyA7fM5fHKNkPDA2ryr34gG97dmgSgB9A,467
 castor_extractor/visualization/looker/env.py,sha256=spfVc6E2CdYDdrPepX-YEUUfHbq-ZlueU9q4pxiaRPQ,864
-castor_extractor/visualization/looker/extract.py,sha256=XJq8fz5WWn-CAg0haoDJXfj6tlgrBcUwxjfBjbxT7oU,3514
+castor_extractor/visualization/looker/extract.py,sha256=C6lBh6b9KbYjkgUigxMWE4af7ameCBQ9nSvpG8OKcTE,3685
 castor_extractor/visualization/looker/fields.py,sha256=WmiSehmczWTufCLg4r2Ozq2grUpzxDNvIAHyGuOoGs4,636
 castor_extractor/visualization/looker/fields_test.py,sha256=7Cwq8Qky6aTZg8nCHp1gmPJtd9pGNB4QeMIRRWdHo5w,782
 castor_extractor/visualization/looker/parameters.py,sha256=JZ5o5BgRs4SxxX5artUfhPaQBX6YPfX8wRp1m_txXUs,1527
@@ -96,7 +96,7 @@ castor_extractor/visualization/metabase/client/api/__init__.py,sha256=bWs7daLw2N
 castor_extractor/visualization/metabase/client/api/client.py,sha256=n4jV7jjLY4meM2sKsqtHXbw76sNlbeUKd1mbLU_GbeY,5609
 castor_extractor/visualization/metabase/client/api/credentials.py,sha256=HsyJ3X_N-smw_6cwQdi9pQ2Pl3KWoBqZL_Xg8dy7hGo,1334
 castor_extractor/visualization/metabase/client/db/__init__.py,sha256=xmqEI59hvioLxE8LXcH4_njCrBy8_3QjIrZNb6bW6Rs,29
-castor_extractor/visualization/metabase/client/db/client.py,sha256=h8s4mYoN_wsPR7kA7SZBT-f5b_RgJcoJWy1lA-_VM0g,4095
+castor_extractor/visualization/metabase/client/db/client.py,sha256=yXjHVeuh-GRFA3rUHMVtOHFhcxxnp_0TDpBpI8UISLA,4091
 castor_extractor/visualization/metabase/client/db/credentials.py,sha256=xWTQ6xDBJak7YS13B5kmh26L36VY0XYpCX_DbEAB4QM,1863
 castor_extractor/visualization/metabase/client/db/queries/.sqlfluff,sha256=sOQQOpAa9QMj9cBlulfmt-DZ_kQzMpzSAEnh10QGSB0,76
 castor_extractor/visualization/metabase/client/db/queries/base_url.sql,sha256=p2EL9kdt-hw_yh3aeCE91AXEB4RrYAbG2QrBBNqQjDE,79
@@ -273,7 +273,7 @@ castor_extractor/warehouse/synapse/queries/schema.sql,sha256=aX9xNrBD_ydwl-znGSF
 castor_extractor/warehouse/synapse/queries/table.sql,sha256=mCE8bR1Vb7j7SwZW2gafcXidQ2fo1HwxcybA8wP2Kfs,1049
 castor_extractor/warehouse/synapse/queries/user.sql,sha256=sTb_SS7Zj3AXW1SggKPLNMCd0qoTpL7XI_BJRMaEpBg,67
 castor_extractor/warehouse/synapse/queries/view_ddl.sql,sha256=3EVbp5_yTgdByHFIPLHmnoOnqqLE77SrjAwFDvu4e54,249
-castor_extractor-0.5.0.dist-info/METADATA,sha256=ZE51CdA4LsLr6kVHK5kgs3v5lLAtPgJoDrg-VYMUXTM,6308
-castor_extractor-0.5.0.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-castor_extractor-0.5.0.dist-info/entry_points.txt,sha256=q_a-iEB9OZF05IUsF43OTLYvgEbHFGp52u9lWtWf9rU,981
-castor_extractor-0.5.0.dist-info/RECORD,,
+castor_extractor-0.5.2.dist-info/METADATA,sha256=WOnF0a8bvVOGTD1yRdLEHhMr2aZV1QZFbMCSRLrG6VU,6308
+castor_extractor-0.5.2.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+castor_extractor-0.5.2.dist-info/entry_points.txt,sha256=q_a-iEB9OZF05IUsF43OTLYvgEbHFGp52u9lWtWf9rU,981
+castor_extractor-0.5.2.dist-info/RECORD,,