castor-extractor 0.24.25__tar.gz → 0.24.27__tar.gz

{castor_extractor-0.24.25 → castor_extractor-0.24.27}/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.24.27 - 2025-06-20
+
+* Strategy: extract logical tables
+
+## 0.24.26 - 2025-06-16
+
+* Coalesce: increase _MAX_ERRORS client parameter
+
 ## 0.24.25 - 2025-06-12
 
 * DBT: Fix API base url

{castor_extractor-0.24.25 → castor_extractor-0.24.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.24.25
+Version: 0.24.27
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -215,6 +215,14 @@ For any questions or bug report, contact us at [support@coalesce.io](mailto:supp
 
 # Changelog
 
+## 0.24.27 - 2025-06-20
+
+* Strategy: extract logical tables
+
+## 0.24.26 - 2025-06-16
+
+* Coalesce: increase _MAX_ERRORS client parameter
+
 ## 0.24.25 - 2025-06-12
 
 * DBT: Fix API base url

{castor_extractor-0.24.25 → castor_extractor-0.24.27}/castor_extractor/transformation/coalesce/client/client.py

@@ -19,7 +19,7 @@ from .type import NodeIDToNamesMapping
 from .utils import column_names_per_node, is_test, test_names_per_node
 
 _LIMIT_MAX = 1_000
-_MAX_ERRORS = 50
+_MAX_ERRORS = 200
 
 logger = logging.getLogger(__name__)
 

{castor_extractor-0.24.25 → castor_extractor-0.24.27}/castor_extractor/visualization/strategy/assets.py

@@ -1,14 +1,20 @@
-from ...types import ExternalAsset
+from ...types import ExternalAsset, classproperty
 
 
 class StrategyAsset(ExternalAsset):
     """Strategy assets that can be extracted"""
 
     ATTRIBUTE = "attribute"
+    COLUMN = "column"
     CUBE = "cube"
     DASHBOARD = "dashboard"
     DOCUMENT = "document"
     FACT = "fact"
+    LOGICAL_TABLE = "logical_table"
     METRIC = "metric"
     REPORT = "report"
     USER = "user"
+
+    @classproperty
+    def optional(cls) -> set["StrategyAsset"]:
+        return {StrategyAsset.COLUMN}

{castor_extractor-0.24.25 → castor_extractor-0.24.27}/castor_extractor/visualization/strategy/client/client.py

@@ -1,14 +1,18 @@
 import logging
 from collections.abc import Iterator
-from enum import Enum
 from typing import Any, Callable, Optional
 from urllib.parse import urlparse
 
 from mstrio.connection import Connection  # type: ignore
 from mstrio.helpers import IServerError  # type: ignore
 from mstrio.modeling import (  # type: ignore
+    Attribute,
+    LogicalTable,
+    PhysicalTable,
+    PhysicalTableType,
     list_attributes,
     list_facts,
+    list_logical_tables,
     list_metrics,
 )
 from mstrio.project_objects import (  # type: ignore
@@ -19,93 +23,21 @@ from mstrio.project_objects import (  # type: ignore
     list_reports,
 )
 from mstrio.server import Environment  # type: ignore
-from mstrio.types import ObjectSubTypes, ObjectTypes  # type: ignore
 from mstrio.users_and_groups import User, list_users  # type: ignore
 from mstrio.utils.entity import Entity  # type: ignore
-from mstrio.utils.helper import is_dashboard  # type: ignore
-from pydantic import BaseModel, ConfigDict
 
 from ..assets import StrategyAsset
 from .credentials import StrategyCredentials
+from .properties import (
+    column_properties,
+    format_url,
+    list_dependencies,
+    lookup_table_id,
+    safe_get_property,
+)
 
 logger = logging.getLogger(__name__)
 
-_BATCH_SIZE: int = 100
-
-
-class URLTemplates(Enum):
-    DASHBOARD = "https://{hostname}/MicroStrategyLibrary/app/{project_id}/{id_}"
-    DOCUMENT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?documentID={id_}&projectID={project_id}"
-    REPORT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?reportID={id_}&projectID={project_id}"
-    FOLDER = "https://{hostname}/MicroStrategy/servlet/mstrWeb?folderID={id_}&projectID={project_id}"
-
-
-def _is_dashboard(entity: Entity) -> bool:
-    """
-    Returns True if the entity is a Dashboard. They can only be distinguished
-    from Documents by checking the `view_media` property.
-    """
-    is_type_document = entity.type == ObjectTypes.DOCUMENT_DEFINITION
-    return is_type_document and is_dashboard(entity.view_media)
-
-
-def _is_report(entity: Entity) -> bool:
-    """
-    Returns True if the entity is a Report. Cubes share the same type as Reports,
-    so the subtype must be checked.
-    """
-    is_type_report = entity.type == ObjectTypes.REPORT_DEFINITION
-    is_subtype_cube = entity.subtype == ObjectSubTypes.OLAP_CUBE.value
-    return is_type_report and not is_subtype_cube
-
-
-def _safe_get_attribute(entity: Entity, attribute: str) -> Optional[str]:
-    """
-    Some properties may raise an error. Example: retrieving a Report's `sql` fails if the Report has not been published.
-    This safely returns the attribute value, or None if the retrieval fails.
-    """
-    try:
-        value = getattr(entity, attribute)
-    except IServerError as e:
-        logger.error(f"Could not get {attribute} for entity {entity.id}: {e}")
-        value = None
-    return value
-
-
-class Dependency(BaseModel):
-    id: str
-    name: str
-    subtype: int
-    type: int
-
-    model_config = ConfigDict(extra="ignore")
-
-
-def _list_dependencies(entity: Entity) -> list[dict]:
-    """Lists the entity's dependencies, keeping only relevant fields."""
-    dependencies: list[dict] = []
-
-    offset = 0
-    while True:
-        batch = entity.list_dependencies(offset=offset, limit=_BATCH_SIZE)
-        dependencies.extend(batch)
-        if len(batch) < _BATCH_SIZE:
-            break
-        offset += _BATCH_SIZE
-
-    return [
-        Dependency(**dependency).model_dump() for dependency in dependencies
-    ]
-
-
-def _level_1_folder_id(folders: list[dict]) -> str:
-    """Searches for the first enclosing folder and returns its ID."""
-    for folder in folders:
-        if folder["level"] == 1:
-            return folder["id"]
-
-    raise ValueError("No level 1 folder found")
-
 
 class StrategyClient:
     """Connect to Strategy through mstrio-py and fetch main assets."""
@@ -116,6 +48,7 @@ class StrategyClient:
             base_url=self.base_url,
             username=credentials.username,
             password=credentials.password,
+            verbose=False,
         )
 
         self.hostname = urlparse(self.base_url).hostname
@@ -129,36 +62,6 @@ class StrategyClient:
     def close(self):
         self.connection.close()
 
-    def _url(self, entity: Entity) -> str:
-        """
-        Formats the right URL.
-        * Dashboards : viewed in MicroStrategy
-        * Reports and Documents : viewed in MicroStrategy Web
-        * other (i.e. Cubes): the URL leads to the folder in MicroStrategy Web
-        """
-        if _is_dashboard(entity):
-            id_ = entity.id
-            template = URLTemplates.DASHBOARD
-
-        elif entity.type == ObjectTypes.DOCUMENT_DEFINITION:
-            id_ = entity.id
-            template = URLTemplates.DOCUMENT
-
-        elif _is_report(entity):
-            id_ = entity.id
-            template = URLTemplates.REPORT
-
-        else:
-            # default to folder URL
-            id_ = _level_1_folder_id(entity.ancestors)
-            template = URLTemplates.FOLDER
-
-        return template.value.format(
-            hostname=self.hostname,
-            id_=id_,
-            project_id=entity.project_id,
-        )
-
     def _common_entity_properties(
         self,
         entity: Entity,
@@ -169,7 +72,7 @@ class StrategyClient:
         Returns the entity's properties, including its dependencies
         and optional URL and/or description.
         """
-        dependencies = _list_dependencies(entity)
+        dependencies = list_dependencies(entity)
         owner_id = entity.owner.id if isinstance(entity.owner, User) else None
         properties = {
             "dependencies": dependencies,
@@ -182,23 +85,80 @@ class StrategyClient:
         }
 
         if with_url:
-            properties["url"] = self._url(entity)
+            assert self.hostname
+            properties["url"] = format_url(
+                entity=entity, hostname=self.hostname
+            )
 
         if with_description:
-            properties["description"] = _safe_get_attribute(
-                entity, "description"
+            properties["description"] = safe_get_property(entity, "description")
+
+        return properties
+
+    def _attributes_properties(self, attribute: Attribute) -> dict[str, Any]:
+        """
+        Attributes have a lookup table, which we need to compute the table lineage.
+        """
+        return {
+            **self._common_entity_properties(attribute, with_url=False),
+            "lookup_table_id": lookup_table_id(attribute),
+        }
+
+    def _physical_table_properties(
+        self, table: Optional[PhysicalTable]
+    ) -> Optional[dict[str, Any]]:
+        """
+        Returns the properties of the physical table, including its columns.
+        A physical table can have 1 of these types:
+          * "normal": meaning it matches 1 warehouse table
+          * "sql": it is based on an SQL statement
+        Other types are not supported (and they technically shouldn't be possible.)
+        """
+        if not table:
+            return None
+
+        properties = {
+            "id": table.id,
+            "name": table.name,
+            "type": table.table_type.value,
+            "columns": column_properties(table.columns),
+        }
+
+        if table.table_type == PhysicalTableType.SQL:
+            physical_table = PhysicalTable(
+                connection=self.connection,
+                id=table.id,
             )
+            properties["sql_statement"] = physical_table.sql_statement
+
+        elif table.table_type == PhysicalTableType.NORMAL:
+            properties["table_prefix"] = table.table_prefix
+            properties["table_name"] = table.table_name
 
         return properties
 
+    def _logical_table_properties(self, table: LogicalTable) -> dict[str, Any]:
+        """
+        Returns properties for:
+            * the logical table itself
+            * its physical table (though it may not be accessible)
+            * the columns of the physical table
+        """
+        physical_table = safe_get_property(table, "physical_table")
+        return {
+            "id": table.id,
+            "name": table.name,
+            "physical_table": self._physical_table_properties(physical_table),
+        }
+
     def _report_properties(self, report: Report) -> dict[str, Any]:
         """
         Report properties contain an optional SQL source query. Due to a typing
         bug in the mstrio package, the typing must be ignored.
         """
         properties = self._common_entity_properties(report)  # type: ignore
-        properties["url"] = self._url(report)  # type: ignore
-        properties["sql"] = _safe_get_attribute(report, "sql")  # type: ignore
+        properties["url"] = format_url(entity=report, hostname=self.hostname)  # type: ignore
+        properties["sql"] = safe_get_property(report, "sql")  # type: ignore
         return properties
 
     @staticmethod
@@ -243,7 +203,7 @@ class StrategyClient:
     def _fetch_attributes(self) -> Iterator[dict[str, Any]]:
         return self._fetch_entities(
             list_attributes,
-            with_url=False,
+            custom_property_extractor=self._attributes_properties,
         )
 
     def _fetch_cubes(self) -> Iterator[dict[str, Any]]:
@@ -263,6 +223,15 @@ class StrategyClient:
             with_description=False,
         )
 
+    def _fetch_logical_tables(self) -> Iterator[dict[str, Any]]:
+        """
+        Yields all logical tables, including their physical tables and their columns.
+        """
+        return self._fetch_entities(
+            list_logical_tables,
+            custom_property_extractor=self._logical_table_properties,
+        )
+
     def _fetch_metrics(self) -> Iterator[dict[str, Any]]:
         return self._fetch_entities(
             list_metrics,
@@ -298,6 +267,9 @@ class StrategyClient:
         elif asset == StrategyAsset.FACT:
             yield from self._fetch_facts()
 
+        elif asset == StrategyAsset.LOGICAL_TABLE:
+            yield from self._fetch_logical_tables()
+
         elif asset == StrategyAsset.METRIC:
             yield from self._fetch_metrics()
 

castor_extractor-0.24.27/castor_extractor/visualization/strategy/client/properties.py

@@ -0,0 +1,145 @@
+import logging
+from enum import Enum
+from typing import Any, Optional
+
+from mstrio.helpers import IServerError  # type: ignore
+from mstrio.modeling import (  # type: ignore
+    Attribute,
+    TableColumn,
+)
+from mstrio.types import ObjectSubTypes, ObjectTypes  # type: ignore
+from mstrio.utils.entity import Entity  # type: ignore
+from mstrio.utils.helper import is_dashboard  # type: ignore
+from pydantic import BaseModel, ConfigDict
+
+logger = logging.getLogger(__name__)
+
+_BATCH_SIZE: int = 100
+
+
+class URLTemplates(Enum):
+    DASHBOARD = "https://{hostname}/MicroStrategyLibrary/app/{project_id}/{id_}"
+    DOCUMENT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?documentID={id_}&projectID={project_id}"
+    REPORT = "https://{hostname}/MicroStrategy/servlet/mstrWeb?reportID={id_}&projectID={project_id}"
+    FOLDER = "https://{hostname}/MicroStrategy/servlet/mstrWeb?folderID={id_}&projectID={project_id}"
+
+
+class Dependency(BaseModel):
+    id: str
+    name: str
+    subtype: int
+    type: int
+
+    model_config = ConfigDict(extra="ignore")
+
+
+def list_dependencies(entity: Entity) -> list[dict]:
+    """Lists the entity's dependencies, keeping only relevant fields."""
+    dependencies: list[dict] = []
+
+    offset = 0
+    while True:
+        batch = entity.list_dependencies(offset=offset, limit=_BATCH_SIZE)
+        dependencies.extend(batch)
+        if len(batch) < _BATCH_SIZE:
+            break
+        offset += _BATCH_SIZE
+
+    return [
+        Dependency(**dependency).model_dump() for dependency in dependencies
+    ]
+
+
+def _is_dashboard(entity: Entity) -> bool:
+    """
+    Returns True if the entity is a Dashboard. They can only be distinguished
+    from Documents by checking the `view_media` property.
+    """
+    is_type_document = entity.type == ObjectTypes.DOCUMENT_DEFINITION
+    return is_type_document and is_dashboard(entity.view_media)
+
+
+def _is_report(entity: Entity) -> bool:
+    """
+    Returns True if the entity is a Report. Cubes share the same type as Reports,
+    so the subtype must be checked.
+    """
+    is_type_report = entity.type == ObjectTypes.REPORT_DEFINITION
+    is_subtype_cube = entity.subtype == ObjectSubTypes.OLAP_CUBE.value
+    return is_type_report and not is_subtype_cube
+
+
+def format_url(entity: Entity, hostname: str) -> str:
+    """
+    Formats the right URL.
+    * Dashboards : viewed in MicroStrategy
+    * Reports and Documents : viewed in MicroStrategy Web
+    * other (i.e. Cubes): the URL leads to the folder in MicroStrategy Web
+    """
+    if _is_dashboard(entity):
+        id_ = entity.id
+        template = URLTemplates.DASHBOARD
+
+    elif entity.type == ObjectTypes.DOCUMENT_DEFINITION:
+        id_ = entity.id
+        template = URLTemplates.DOCUMENT
+
+    elif _is_report(entity):
+        id_ = entity.id
+        template = URLTemplates.REPORT
+
+    else:
+        # default to folder URL
+        id_ = level_1_folder_id(entity.ancestors)
+        template = URLTemplates.FOLDER
+
+    return template.value.format(
+        hostname=hostname,
+        id_=id_,
+        project_id=entity.project_id,
+    )
+
+
+def safe_get_property(entity: Entity, attribute: str) -> Optional[str]:
+    """
+    Some properties may raise an error. Example: retrieving a Report's `sql` fails if the Report has not been published.
+    This safely returns the attribute value, or None if the retrieval fails.
+    """
+    try:
+        value = getattr(entity, attribute)
+    except IServerError as e:
+        logger.error(f"Could not get {attribute} for entity {entity.id}: {e}")
+        value = None
+    return value
+
+
+def column_properties(columns: list[TableColumn]) -> list[dict[str, Any]]:
+    """Returns the properties of a physical table's columns."""
+    properties: list[dict[str, Any]] = []
+
+    for column in columns:
+        column_properties = {
+            "id": column.id,
+            "name": column.name,
+            "column_name": column.column_name,
+        }
+        properties.append(column_properties)
+
+    return properties
+
+
+def level_1_folder_id(folders: list[dict]) -> str:
+    """Searches for the first enclosing folder and returns its ID."""
+    for folder in folders:
+        if folder["level"] == 1:
+            return folder["id"]
+
+    raise ValueError("No level 1 folder found")
+
+
+def lookup_table_id(attribute: Attribute):
+    """Returns the lookup table's ID, if there is one."""
+    lookup_table = attribute.attribute_lookup_table
+    if not lookup_table:
+        return None
+    return lookup_table.object_id

{castor_extractor-0.24.25 → castor_extractor-0.24.27}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.2"]
 
 [tool.poetry]
 name = "castor-extractor"
-version = "0.24.25"
+version = "0.24.27"
 description = "Extract your metadata assets."
 authors = ["Castor <support@castordoc.com>"]
 license = "EULA"