acryl-datahub 1.2.0.10rc8__py3-none-any.whl → 1.2.0.11rc2__py3-none-any.whl

{acryl_datahub-1.2.0.10rc8.dist-info → acryl_datahub-1.2.0.11rc2.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.10rc8.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.2.0.11rc2.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=xDxMP7YG6O7bgAgZ4LVjgQh_jfbuV9rXJU1zFdpjQEA,324
+datahub/_version.py,sha256=wA3SrSgI7aUwEw5r0jMdQQHnFvzLaVy7PuHAjLJoDJI,324
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -115,7 +115,7 @@ datahub/configuration/source_common.py,sha256=G4fkFw-dG0zVMSSsXOZn1ua_w4QJBpVMMq
 datahub/configuration/time_window_config.py,sha256=c4mbrgmTobt4t_j6unDeYvmGSlbRB2hAgAst6yq4nHA,5412
 datahub/configuration/toml.py,sha256=Ohc5sAWLPoAinPYL8njyheZ3ak81fC2Sp8IbBbESPGg,380
 datahub/configuration/validate_field_deprecation.py,sha256=szzs0130AXcK6aoYMmvEK5oWXPKcnf-EqRqvE6SuW-U,1377
-datahub/configuration/validate_field_removal.py,sha256=6f3nzdjsukVqPQ8weaLCXXKmN7mBF-LoWsVe7cxu-DA,1128
+datahub/configuration/validate_field_removal.py,sha256=LWVmUD8aDS0WkuNCqBCuIG_wOSZ3G-ANv8WyvKOEdDo,1285
 datahub/configuration/validate_field_rename.py,sha256=o_MPYuVLiLRalhDa-p2pUxzx4_rBU-tQ-3wZFHXAOTg,2287
 datahub/configuration/validate_host_port.py,sha256=dgR9XPreNV_fABOmv2UHYF-OSN6AHD92Zi2nKsfdTiE,867
 datahub/configuration/validate_multiline_string.py,sha256=j5EABUvFOpV7c21eBWZSwy2ryIA2PI67lN5ebyrHv00,1370
@@ -383,26 +383,26 @@ datahub/ingestion/source/kafka_connect/sink_connectors.py,sha256=QTMY0FmOHkTxfIC
 datahub/ingestion/source/kafka_connect/source_connectors.py,sha256=A9q-u5IoV35swvoyMrzT75FVV9-SBeYGhLKDYRge-IQ,23845
 datahub/ingestion/source/looker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/looker/lkml_patched.py,sha256=XShEU7Wbz0DubDhYMjKf9wjKZrBJa2XPg9MIjp8rPhk,733
-datahub/ingestion/source/looker/looker_common.py,sha256=0V0nGejI3Do4KvE_nG0KpBD6yTKbM21vECel24T1xX0,67739
+datahub/ingestion/source/looker/looker_common.py,sha256=vXD6w1VAhxKlFp283WZkRcccGQt1uKij175o5oqO1HM,67912
 datahub/ingestion/source/looker/looker_config.py,sha256=4aYK9FQvH2BOH3IdtcL93QreFKqNC8_gm5OPRjcT2XA,13905
 datahub/ingestion/source/looker/looker_connection.py,sha256=yDmC6lDsHmL2e_Pw8ULylwOIHPWPp_6gT1iyLvD0fTw,2075
-datahub/ingestion/source/looker/looker_constant.py,sha256=GMKYtNXlpojPxa9azridKfcGLSJwKdUCTesp7U8dIrQ,402
+datahub/ingestion/source/looker/looker_constant.py,sha256=6v8DeeLOgnWXaz5t6Ghl514PYoqe0DTXykX_MJ2kYf0,530
 datahub/ingestion/source/looker/looker_dataclasses.py,sha256=MrDeZ4Nd0wQnJbCoI1qePYlYeObnUw5dvpWcmhKuNgc,12346
 datahub/ingestion/source/looker/looker_file_loader.py,sha256=gb2Z97_w28MsybYe01JFMMqlvBbn2occyUEknf_mYMA,4882
-datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=6smUt_Ya7ZJMHWdGZl3TnhM7XHZVpYQ6gz2i5hHejZ4,11547
+datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=M6dVvTLWi58f7WRY1kFkKwA3vRQO_QgtKYvjIjXlRBQ,12708
 datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=27WnOuTghayaH-HL4lLoq0IcHvNm1UybMqMnoaxN8Cs,5383
 datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
 datahub/ingestion/source/looker/looker_source.py,sha256=7mRlIJq2DoM1h2y-heNdNoqok8sNl7Qmpwsx0dQsYP8,67273
 datahub/ingestion/source/looker/looker_template_language.py,sha256=5fZFPKFP3IYbJg3jLifjaji4wWg8wRy-1XDvc8Qucus,17949
 datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
-datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
-datahub/ingestion/source/looker/lookml_concept_context.py,sha256=eDaze9S7cgO5eFP7-0azUMEJyR3EfMjmfj5pMPjpm8c,18066
-datahub/ingestion/source/looker/lookml_config.py,sha256=_iMHpRDrlg2FvOkgqjfJIaW5NQM-C2hAgCeC2yZRNsc,11402
-datahub/ingestion/source/looker/lookml_refinement.py,sha256=MkVreI0BylaCFyDHihDHaCcXyDSP84eF9p1h5d-ZHnM,9504
-datahub/ingestion/source/looker/lookml_source.py,sha256=aNvEoW0njzqQSoj1KeHiDOBbEGYWWDEcjX-a2RYgqOY,42735
+datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=smgH0Z3OlugaDKU0xSlSA4cggZVhuX8p5TNbeqX24II,4447
+datahub/ingestion/source/looker/lookml_concept_context.py,sha256=s3eSNKchikE9gg30rdW-kOmV2uki_wYUldhW4SgrISU,18066
+datahub/ingestion/source/looker/lookml_config.py,sha256=Z6sqKMwuJWJzE4pIX9Pm1ERtUgx5v0g-PM-SME4Q06M,13138
+datahub/ingestion/source/looker/lookml_refinement.py,sha256=_FV7-8zdZJhntkAaS3FuWXjibq7LQa91fQuolAdav4c,9559
+datahub/ingestion/source/looker/lookml_source.py,sha256=aaFntJjISMV1QeQqzEpMxjvFod0-CX9ONEaAfFvAxOA,43475
 datahub/ingestion/source/looker/str_functions.py,sha256=zceEX2ka_4WaWwWgEdyknUSz7X3GrO951BkwSbF2afo,766
 datahub/ingestion/source/looker/urn_functions.py,sha256=4VvqEfGvIMq3rNHHps0-HlPurMPnpqdxNtDAOOHIZww,528
-datahub/ingestion/source/looker/view_upstream.py,sha256=4FCjZaU6p2G7npB2RJpP4Gv2yLjbvbsYWEbAg55IvjY,26110
+datahub/ingestion/source/looker/view_upstream.py,sha256=Ajc9G1NKWTS7neuN5OOgg9maePrla8aKc1tokiNs7Ic,47489
 datahub/ingestion/source/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/metadata/business_glossary.py,sha256=sGOw_5IQxlLxqw7NvvQX4QeCaursuDg4aOyoAN973g4,19709
 datahub/ingestion/source/metadata/lineage.py,sha256=8jtlZqlgrHOfk1SpJ0lXWcyNfNxr0uYB_kvsqTyqFHo,9618
@@ -980,10 +980,12 @@ datahub/sdk/mlmodel.py,sha256=cO5R8BYVljmQ0w33RIOuZmj4nq8OJCDVAZGTQI6YFS8,12628
 datahub/sdk/mlmodelgroup.py,sha256=wlZZHny0UORpF0fRYuVkWLSQwIHX_fWl5lPb1NKR6dM,8194
 datahub/sdk/resolver_client.py,sha256=nKMAZJt2tRSGfKSzoREIh43PXqjM3umLiYkYHJjo1io,3243
 datahub/sdk/search_client.py,sha256=f2L_aOy-pPB9Mk7WdLSJ6Htp3OT4cEgDIrqnCweNtM8,3592
-datahub/sdk/search_filters.py,sha256=99D-dve11eh0Km60tWjkAZRFjWLQHREjjUpKnuA1pMs,18063
+datahub/sdk/search_filters.py,sha256=EAIsO12rwTDJexQcxiiCsuk_TjKRk8Cx8G1ENqYYnUc,22056
 datahub/secret/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/secret/datahub_secret_store.py,sha256=xyNAZY62d6KSz_kYF9wN7RDMLvNhu2ayOzcYvubOX1E,2519
+datahub/secret/datahub_secret_store.py,sha256=qW7JI_k2xJdTZyluiJDaG5B3C68VnDpQk_M_L-Uz2vY,2583
 datahub/secret/datahub_secrets_client.py,sha256=nDmhziKdvseJHlaDVUcAwK8Fv8maeAaG-ktZtWG2b70,1316
+datahub/secret/environment_secret_store.py,sha256=JbTAswfmYycjZc7xQyVOELb-3nc0Kswme_Ml7pjMpic,824
+datahub/secret/file_secret_store.py,sha256=At_EdN_KXNpPh3_zJVa9FPBBxUbA2D43Sua36MXvpL8,1614
 datahub/secret/secret_common.py,sha256=g4anQtYPm7cI6kEJUZHjpBqeCyiUKIim2rJQByaeOoY,2864
 datahub/secret/secret_store.py,sha256=2VP_Vd336Cy7C-2kwp4rx8MAqtYgtwv8XyzzNTXE5x8,1124
 datahub/specific/__init__.py,sha256=r5RYM5mDnskLzin3vc87HV-9GSz3P6uQw8AlsN14LaI,88
@@ -1044,7 +1046,7 @@ datahub/utilities/dedup_list.py,sha256=dUSpe1AajfuwlHVJKNv-CzDXSCkaw0HgSMOsxqUkQ
 datahub/utilities/delayed_iter.py,sha256=XlsI0DCXkVVejFKOW_uMT0E8DTqqOHQN3Ooak4EcULE,645
 datahub/utilities/delta.py,sha256=hkpF8W7Lvg2gUJBQR3mmIzOxsRQ6i5cchRPFlAVoV10,1128
 datahub/utilities/docs_build.py,sha256=uFMK3z1d4BExpsrvguHunidbEDAzQ8hoOP7iQ0A_IVw,211
-datahub/utilities/file_backed_collections.py,sha256=4_11YQUaIdjr7SUV4AHczArBi8OrrlzuX15ldR1GhKA,21673
+datahub/utilities/file_backed_collections.py,sha256=eOW7_8CzopvzFk1IATVuGGzQvS4yLQzHR_HU3h6T4kY,21675
 datahub/utilities/global_warning_util.py,sha256=adrEl3WhetQ-bymrPINjd976ZFndhbvk3QosUYGsos8,261
 datahub/utilities/groupby.py,sha256=pe6rP4ZCttYB98yjbs0Aey8C32aLb7rq-NJ_BFky0H4,524
 datahub/utilities/hive_schema_to_avro.py,sha256=YCdq3jNUTij8ehWgX9v6CiOrf5aTCXr4DERcp_-wBbo,11608
@@ -1121,8 +1123,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.10rc8.dist-info/METADATA,sha256=FnAZxap4iq7bDMvUTCEiquA1YbX_NclQz6-LeoenjG0,184162
-acryl_datahub-1.2.0.10rc8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.10rc8.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
-acryl_datahub-1.2.0.10rc8.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.10rc8.dist-info/RECORD,,
+acryl_datahub-1.2.0.11rc2.dist-info/METADATA,sha256=7a2zx4CgdIqVcUawhQNKvcgQmDg6sxLzleaTZ1mnWkA,184162
+acryl_datahub-1.2.0.11rc2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.2.0.11rc2.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
+acryl_datahub-1.2.0.11rc2.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.2.0.11rc2.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0.10rc8"
+__version__ = "1.2.0.11rc2"
 
 
 def is_dev_mode() -> bool:

datahub/configuration/validate_field_removal.py

@@ -24,6 +24,9 @@ def pydantic_removed_field(
             values.pop(field)
         return values
 
+    # Mark the function as handling a removed field for doc generation
+    _validate_field_removal._doc_removed_field = field  # type: ignore[attr-defined]
+
     # Hack: Pydantic maintains unique list of validators by referring its __name__.
     # https://github.com/pydantic/pydantic/blob/v1.10.9/pydantic/main.py#L264
     # This hack ensures that multiple field removals do not overwrite each other.

datahub/ingestion/source/looker/looker_common.py

@@ -307,6 +307,12 @@ class ViewFieldType(Enum):
     UNKNOWN = "Unknown"
 
 
+class ViewFieldDimensionGroupType(Enum):
+    # Ref: https://cloud.google.com/looker/docs/reference/param-field-dimension-group
+    TIME = "time"
+    DURATION = "duration"
+
+
 class ViewFieldValue(Enum):
     NOT_AVAILABLE = "NotAvailable"
 

datahub/ingestion/source/looker/looker_constant.py

@@ -11,3 +11,7 @@ prod = "prod"
 dev = "dev"
 NAME = "name"
 DERIVED_DOT_SQL = "derived.sql"
+
+VIEW_FIELD_TYPE_ATTRIBUTE = "type"
+VIEW_FIELD_INTERVALS_ATTRIBUTE = "intervals"
+VIEW_FIELD_TIMEFRAMES_ATTRIBUTE = "timeframes"

datahub/ingestion/source/looker/looker_lib_wrapper.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+from enum import Enum
 from functools import lru_cache
 from typing import Dict, List, MutableMapping, Optional, Sequence, Set, Union, cast
 
@@ -31,6 +32,14 @@ from datahub.configuration.common import ConfigurationError
 logger = logging.getLogger(__name__)
 
 
+class LookerQueryResponseFormat(Enum):
+    # result_format - Ref: https://cloud.google.com/looker/docs/reference/looker-api/latest/methods/Query/run_inline_query
+    JSON = "json"
+    SQL = (
+        "sql"  # Note: This does not execute the query, it only generates the SQL query.
+    )
+
+
 class TransportOptionsConfig(ConfigModel):
     timeout: int
     headers: MutableMapping[str, str]
@@ -69,6 +78,7 @@ class LookerAPIStats(BaseModel):
     search_looks_calls: int = 0
     search_dashboards_calls: int = 0
     all_user_calls: int = 0
+    generate_sql_query_calls: int = 0
 
 
 class LookerAPI:
@@ -170,17 +180,40 @@ class LookerAPI:
         logger.debug(f"Executing query {write_query}")
         self.client_stats.query_calls += 1
 
-        response_json = self.client.run_inline_query(
-            result_format="json",
+        response = self.client.run_inline_query(
+            result_format=LookerQueryResponseFormat.JSON.value,
             body=write_query,
             transport_options=self.transport_options,
         )
 
+        data = json.loads(response)
+
         logger.debug("=================Response=================")
-        data = json.loads(response_json)
         logger.debug("Length of response: %d", len(data))
         return data
 
+    def generate_sql_query(
+        self, write_query: WriteQuery, use_cache: bool = False
+    ) -> str:
+        """
+        Generates a SQL query string for a given WriteQuery.
+
+        Note: This does not execute the query, it only generates the SQL query.
+        """
+        logger.debug(f"Generating SQL query for {write_query}")
+        self.client_stats.generate_sql_query_calls += 1
+
+        response = self.client.run_inline_query(
+            result_format=LookerQueryResponseFormat.SQL.value,
+            body=write_query,
+            transport_options=self.transport_options,
+            cache=use_cache,
+        )
+
+        logger.debug("=================Response=================")
+        logger.debug("Length of SQL response: %d", len(response))
+        return str(response)
+
     def dashboard(self, dashboard_id: str, fields: Union[str, List[str]]) -> Dashboard:
         self.client_stats.dashboard_calls += 1
         return self.client.dashboard(

datahub/ingestion/source/looker/looker_view_id_cache.py

@@ -3,11 +3,11 @@ from typing import Dict, List, Optional
 
 from datahub.ingestion.source.looker.looker_common import LookerViewId, ViewFieldValue
 from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition
+from datahub.ingestion.source.looker.looker_constant import NAME
 from datahub.ingestion.source.looker.looker_dataclasses import LookerModel
 from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader
 from datahub.ingestion.source.looker.lookml_config import (
     BASE_PROJECT_NAME,
-    NAME,
     LookMLSourceReport,
 )
 

datahub/ingestion/source/looker/lookml_concept_context.py

@@ -12,12 +12,12 @@ from datahub.ingestion.source.looker.looker_constant import (
     DIMENSION_GROUPS,
     DIMENSIONS,
     MEASURES,
+    NAME,
 )
 from datahub.ingestion.source.looker.looker_dataclasses import LookerViewFile
 from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader
 from datahub.ingestion.source.looker.lookml_config import (
     DERIVED_VIEW_SUFFIX,
-    NAME,
     LookMLSourceReport,
 )
 from datahub.ingestion.source.looker.lookml_refinement import LookerRefinementResolver

datahub/ingestion/source/looker/lookml_config.py

@@ -28,11 +28,10 @@ from datahub.ingestion.source.state.stateful_ingestion_base import (
     StatefulIngestionConfigBase,
 )
 from datahub.utilities.lossy_collections import LossyList
+from datahub.utilities.stats_collections import TopKDict, float_top_k_dict
 
 logger = logging.getLogger(__name__)
 
-NAME: str = "name"
-
 BASE_PROJECT_NAME = "__BASE"
 
 EXPLORE_FILE_EXTENSION = ".explore.lkml"
@@ -47,6 +46,9 @@ DERIVED_VIEW_PATTERN: str = r"\$\{([^}]*)\}"
 @dataclass
 class LookMLSourceReport(StaleEntityRemovalSourceReport):
     git_clone_latency: Optional[timedelta] = None
+    looker_query_api_latency_seconds: TopKDict[str, float] = dataclass_field(
+        default_factory=float_top_k_dict
+    )
     models_discovered: int = 0
     models_dropped: LossyList[str] = dataclass_field(default_factory=LossyList)
     views_discovered: int = 0
@@ -81,6 +83,11 @@ class LookMLSourceReport(StaleEntityRemovalSourceReport):
             self.api_stats = self._looker_api.compute_stats()
         return super().compute_stats()
 
+    def report_looker_query_api_latency(
+        self, view_urn: str, latency: timedelta
+    ) -> None:
+        self.looker_query_api_latency_seconds[view_urn] = latency.total_seconds()
+
 
 class LookMLSourceConfig(
     LookerCommonConfig, StatefulIngestionConfigBase, EnvConfigMixin
@@ -122,6 +129,16 @@ class LookMLSourceConfig(
         description="List of regex patterns for LookML views to include in the extraction.",
     )
     parse_table_names_from_sql: bool = Field(True, description="See note below.")
+    use_api_for_view_lineage: bool = Field(
+        False,
+        description="When enabled, uses Looker API to get SQL representation of views for lineage parsing instead of parsing LookML files directly. Requires 'api' configuration to be provided."
+        "Coverage of regex based lineage extraction has limitations, it only supportes ${TABLE}.column_name syntax, See (https://cloud.google.com/looker/docs/reference/param-field-sql#sql_for_dimensions) to"
+        "understand the other substitutions and cross-references allowed in LookML.",
+    )
+    use_api_cache_for_view_lineage: bool = Field(
+        False,
+        description="When enabled, uses Looker API server-side caching for query execution. Requires 'api' configuration to be provided.",
+    )
     api: Optional[LookerAPIConfig] = None
     project_name: Optional[str] = Field(
         None,
@@ -239,6 +256,17 @@ class LookMLSourceConfig(
             )
         return values
 
+    @root_validator(skip_on_failure=True)
+    def check_api_provided_for_view_lineage(cls, values):
+        """Validate that we must have an api credential to use Looker API for view's column lineage"""
+        if not values.get("api") and values.get("use_api_for_view_lineage"):
+            raise ValueError(
+                "API credential was not found. LookML source requires api credentials "
+                "for Looker to use Looker APIs for view's column lineage extraction."
+                "Set `use_api_for_view_lineage` to False to skip using Looker APIs."
+            )
+        return values
+
     @validator("base_folder", always=True)
     def check_base_folder_if_not_provided(
         cls, v: Optional[pydantic.DirectoryPath], values: Dict[str, Any]

datahub/ingestion/source/looker/lookml_refinement.py

@@ -4,10 +4,10 @@ import logging
 from typing import ClassVar, Dict, List, Set
 
 from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition
+from datahub.ingestion.source.looker.looker_constant import NAME
 from datahub.ingestion.source.looker.looker_dataclasses import LookerModel
 from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader
 from datahub.ingestion.source.looker.lookml_config import (
-    NAME,
     LookMLSourceConfig,
     LookMLSourceReport,
 )

datahub/ingestion/source/looker/lookml_source.py

@@ -142,6 +142,8 @@ class LookerView:
         ctx: PipelineContext,
         extract_col_level_lineage: bool = False,
         populate_sql_logic_in_descriptions: bool = False,
+        looker_client: Optional[LookerAPI] = None,
+        view_to_explore_map: Optional[Dict[str, str]] = None,
     ) -> Optional["LookerView"]:
         view_name = view_context.name()
 
@@ -160,6 +162,8 @@ class LookerView:
             config=config,
             ctx=ctx,
             reporter=reporter,
+            looker_client=looker_client,
+            view_to_explore_map=view_to_explore_map,
         )
 
         field_type_vs_raw_fields = OrderedDict(
@@ -705,6 +709,11 @@ class LookMLSource(StatefulIngestionSourceBase):
         # Value: Tuple(model file name, connection name)
         view_connection_map: Dict[str, Tuple[str, str]] = {}
 
+        # Map of view name to explore name for API-based view lineage
+        # A view can be referenced by multiple explores, we only need one of the explores to use Looker Query API
+        # Key: view_name, Value: explore_name
+        view_to_explore_map: Dict[str, str] = {}
+
         # The ** means "this directory and all subdirectories", and hence should
         # include all the files we want.
         model_files = sorted(
@@ -759,37 +768,37 @@ class LookMLSource(StatefulIngestionSourceBase):
                 )
             )
 
-            if self.source_config.emit_reachable_views_only:
-                model_explores_map = {d["name"]: d for d in model.explores}
-                for explore_dict in model.explores:
-                    try:
-                        if LookerRefinementResolver.is_refinement(explore_dict["name"]):
-                            continue
+            model_explores_map = {d["name"]: d for d in model.explores}
+            for explore_dict in model.explores:
+                try:
+                    if LookerRefinementResolver.is_refinement(explore_dict["name"]):
+                        continue
 
-                        explore_dict = (
-                            looker_refinement_resolver.apply_explore_refinement(
-                                explore_dict
-                            )
-                        )
-                        explore: LookerExplore = LookerExplore.from_dict(
-                            model_name,
-                            explore_dict,
-                            model.resolved_includes,
-                            viewfile_loader,
-                            self.reporter,
-                            model_explores_map,
-                        )
-                        if explore.upstream_views:
-                            for view_name in explore.upstream_views:
+                    explore_dict = looker_refinement_resolver.apply_explore_refinement(
+                        explore_dict
+                    )
+                    explore: LookerExplore = LookerExplore.from_dict(
+                        model_name,
+                        explore_dict,
+                        model.resolved_includes,
+                        viewfile_loader,
+                        self.reporter,
+                        model_explores_map,
+                    )
+                    if explore.upstream_views:
+                        for view_name in explore.upstream_views:
+                            if self.source_config.emit_reachable_views_only:
                                 explore_reachable_views.add(view_name.include)
-                    except Exception as e:
-                        self.reporter.report_warning(
-                            title="Failed to process explores",
-                            message="Failed to process explore dictionary.",
-                            context=f"Explore Details: {explore_dict}",
-                            exc=e,
-                        )
-                        logger.debug("Failed to process explore", exc_info=e)
+                            # Build view to explore mapping for API-based view lineage
+                            view_to_explore_map[view_name.include] = explore.name
+                except Exception as e:
+                    self.reporter.report_warning(
+                        title="Failed to process explores",
+                        message="Failed to process explore dictionary.",
+                        context=f"Explore Details: {explore_dict}",
+                        exc=e,
+                    )
+                    logger.debug("Failed to process explore", exc_info=e)
 
             processed_view_files = processed_view_map.setdefault(
                 model.connection, set()
@@ -878,6 +887,10 @@ class LookMLSource(StatefulIngestionSourceBase):
                                 populate_sql_logic_in_descriptions=self.source_config.populate_sql_logic_for_missing_descriptions,
                                 config=self.source_config,
                                 ctx=self.ctx,
+                                looker_client=self.looker_client,
+                                view_to_explore_map=view_to_explore_map
+                                if view_to_explore_map
+                                else None,
                             )
                         except Exception as e:
                             self.reporter.report_warning(