acryl-datahub 1.2.0.10rc4__py3-none-any.whl → 1.2.0.10rc5__py3-none-any.whl

{acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.2.0.10rc4.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.2.0.10rc5.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=SkqxUSRQtqthxeXaTNtU22wjLSszUJ_BVZbM342CgWI,324
+datahub/_version.py,sha256=3JayCFLMEiiPnmNQMtSHZ8p5gHdHdXKS7AvrQiPCs3g,324
 datahub/entrypoints.py,sha256=9Qf-37rNnTzbGlx8S75OCDazIclFp6zWNcCEL1zCZto,9015
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -159,7 +159,7 @@ datahub/ingestion/api/auto_work_units/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCe
 datahub/ingestion/api/auto_work_units/auto_dataset_properties_aspect.py,sha256=ID_6N3nWl2qohsSGizUCqo3d2MNyDeVbyWroQpSOSsc,5059
 datahub/ingestion/api/auto_work_units/auto_ensure_aspect_size.py,sha256=0BwkpLhORbsiTHq0g_N_1cVVoZYdLR3qz02mNmsV9-M,4444
 datahub/ingestion/autogenerated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/autogenerated/capability_summary.json,sha256=6n3XZj4xPZ3WVfQ29QQvqX5ancSXAqvjaMfpDqFQ8LI,110531
+datahub/ingestion/autogenerated/capability_summary.json,sha256=cSA7jRLLphwkM9C9LK6HdbWRNM0s2febr-4Rh69vQss,111506
 datahub/ingestion/autogenerated/lineage.json,sha256=8BdZF-5V5kJbX4mfFav8Zg-jHjzfkAEGk-pu1atLN4I,10029
 datahub/ingestion/autogenerated/lineage_helper.py,sha256=I_k1pZSCCCjDbUVifPTfy6fkmV8jqdVhbirE8EkpmxI,4748
 datahub/ingestion/extractor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -392,7 +392,7 @@ datahub/ingestion/source/looker/looker_file_loader.py,sha256=gb2Z97_w28MsybYe01J
 datahub/ingestion/source/looker/looker_lib_wrapper.py,sha256=6smUt_Ya7ZJMHWdGZl3TnhM7XHZVpYQ6gz2i5hHejZ4,11547
 datahub/ingestion/source/looker/looker_liquid_tag.py,sha256=27WnOuTghayaH-HL4lLoq0IcHvNm1UybMqMnoaxN8Cs,5383
 datahub/ingestion/source/looker/looker_query_model.py,sha256=N0jBbFruiCIIGT6sJn6tNeppeQ78KGTkOwTLirhxFNc,2144
-datahub/ingestion/source/looker/looker_source.py,sha256=a-G_73NWHD0YPDetT-Eyvq5KenJjqbmb-bV5JMOByNU,65048
+datahub/ingestion/source/looker/looker_source.py,sha256=7mRlIJq2DoM1h2y-heNdNoqok8sNl7Qmpwsx0dQsYP8,67273
 datahub/ingestion/source/looker/looker_template_language.py,sha256=5fZFPKFP3IYbJg3jLifjaji4wWg8wRy-1XDvc8Qucus,17949
 datahub/ingestion/source/looker/looker_usage.py,sha256=qFBX7OHtIcarYIqFe0jQMrDV8MMPV_nN4PZrZRUznTw,23029
 datahub/ingestion/source/looker/looker_view_id_cache.py,sha256=92gDy6NONhJYBp92z_IBzDVZvezmUIkaBCZY1bdk6mE,4392
@@ -482,6 +482,12 @@ datahub/ingestion/source/sigma/sigma.py,sha256=7dbkwk8_wp94XH9mhmtI_8ihR35cqYywt
 datahub/ingestion/source/sigma/sigma_api.py,sha256=7PK5AQa838hYeaQ5L0dioi4n4bLrpN-r7COKTTNUYw8,19837
 datahub/ingestion/source/slack/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/slack/slack.py,sha256=JWanUfzFGynV_PWcH0YzJIbRcmL880DA6dEI9QW-QiQ,25800
+datahub/ingestion/source/snaplogic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+datahub/ingestion/source/snaplogic/snaplogic.py,sha256=VoCRSBS0kE4RAsn4hOJgQy3uadFvelLIOv3I0_gne-w,13540
+datahub/ingestion/source/snaplogic/snaplogic_config.py,sha256=oxuNBfAHWMHoOvh52gifOFcBOSN8aaPpFC8QgmgXwWI,1445
+datahub/ingestion/source/snaplogic/snaplogic_lineage_extractor.py,sha256=IzCYwXLN6IfRFqns7XXtZxzQjjHC-XNTGXgEMeSfW8U,3776
+datahub/ingestion/source/snaplogic/snaplogic_parser.py,sha256=q5dRfWtOpSELPZrpyLbszOD49MJBXNbKgnITLMPiyGI,5783
+datahub/ingestion/source/snaplogic/snaplogic_utils.py,sha256=SVrV9ZXVE2cKKPfoVsxjBN2fIcpYbs2PBLiyQIcJMVQ,1068
 datahub/ingestion/source/snowflake/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/snowflake/constants.py,sha256=iDTamMozHwLYyglpRfqwTbxPxYPhb-uJGRHIgDRHUkA,2767
 datahub/ingestion/source/snowflake/oauth_config.py,sha256=ol9D3RmruGStJAeL8PYSQguSqcD2HfkjPkMF2AB_eZs,1277
@@ -554,7 +560,7 @@ datahub/ingestion/source/state_provider/datahub_ingestion_checkpointing_provider
 datahub/ingestion/source/state_provider/file_ingestion_checkpointing_provider.py,sha256=DziD57PbHn2Tcy51tYXCG-GQgyTGMUxnkuzVS_xihFY,4079
 datahub/ingestion/source/state_provider/state_provider_registry.py,sha256=SVq4mIyGNmLXE9OZx1taOiNPqDoQp03-Ot9rYnB5F3k,401
 datahub/ingestion/source/tableau/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/source/tableau/tableau.py,sha256=Ml-xSQei-o1LFsQKNP9cUY-VQypa68blFy6amZKuxEU,158343
+datahub/ingestion/source/tableau/tableau.py,sha256=BgR_-IuzUYuaSV3EOvMcefwBZ_h0dYAvNEn5Dv6meA4,158448
 datahub/ingestion/source/tableau/tableau_common.py,sha256=2vE7DIigPvMNcTCWSou0tliaVy9MgFR1qwqnE4pilw8,27086
 datahub/ingestion/source/tableau/tableau_constant.py,sha256=2WPAHN-GAR83_c3eTTNd8cy0-zC8GIXeUdSxX_mNdas,2608
 datahub/ingestion/source/tableau/tableau_server_wrapper.py,sha256=wsVD0SkGUwb-H9_g0aDclKwYkcoxugaWyAcyAMgBCAU,1136
@@ -612,6 +618,7 @@ datahub/ingestion/transformer/pattern_cleanup_dataset_usage_user.py,sha256=jTURu
 datahub/ingestion/transformer/pattern_cleanup_ownership.py,sha256=YJH4lv1ztKAYwsdRz5RiUu6SX08h2GBr1S-9QlcmB18,3755
 datahub/ingestion/transformer/remove_dataset_ownership.py,sha256=kHiIcT19BDKNIuCQqAj827E1ZEvME0eGGrILEACALRc,1195
 datahub/ingestion/transformer/replace_external_url.py,sha256=Nw2V1m86fD1vMGLYGPlaoIJEORV0O9qAqWydL-1n2Ng,4058
+datahub/ingestion/transformer/set_browse_path.py,sha256=K8Y4O9vjeJQCdYGsFuNE0aClT73HKBpmj51Yy6Jm_uQ,4065
 datahub/ingestion/transformer/tags_to_terms.py,sha256=VDcd7cM5hGCxo6QP1x4RNEw5Q9v4WDxjRhQMpAl95-A,5558
 datahub/ingestion/transformer/transform_registry.py,sha256=bartmA1zEaULNy5W1Q7gRF8h5Y57BFC6XNOGfCzh1Zw,251
 datahub/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -957,11 +964,11 @@ datahub/pydantic/compat.py,sha256=TUEo4kSEeOWVAhV6LQtst1phrpVgGtK4uif4OI5vQ2M,19
 datahub/sdk/__init__.py,sha256=66OOcFi7qlnL6q72c_yUX2mWU2HudbOdRsC5CIoDxow,1922
 datahub/sdk/_all_entities.py,sha256=eQAmD_fcEHlTShe1_nHpdvHxLDN9njk9bdLnuTrYg8M,905
 datahub/sdk/_attribution.py,sha256=0Trh8steVd27GOr9MKCZeawbuDD2_q3GIsZlCtHqEUg,1321
-datahub/sdk/_shared.py,sha256=uSLPjXfUl_0SPt-kWirkhE6u1CKOC67q5c4hJzxS2uo,28812
+datahub/sdk/_shared.py,sha256=tlyxyxgo7x-8sJhUG9VvUiLpxcOP-GIg4ToqPixagbw,34221
 datahub/sdk/_utils.py,sha256=oXE2BzsXE5zmSkCP3R1tObD4RHnPeH_ps83D_Dw9JaQ,1169
-datahub/sdk/chart.py,sha256=_gixCcKp6kCMizWMXwNH1Ip1ZqJ05_Iu2t94dmONQFM,11774
+datahub/sdk/chart.py,sha256=gsyq_saUInGJKm0s4wwZ9pLMyWLmodrDhrwdZ_Y2h9E,13902
 datahub/sdk/container.py,sha256=IjnFVGDpSFDvgHuuMb7C3VdBxhJuIMq0q6crOs5PupE,7899
-datahub/sdk/dashboard.py,sha256=ekdchqZ57enVFIfeEOOl0Dk4ec-MFQSncArciTgU1sk,15109
+datahub/sdk/dashboard.py,sha256=Vsy_geQzwk3xzqpSuywbgQJ6T__nEvfK5rmrtvY2DPo,16812
 datahub/sdk/dataflow.py,sha256=gdAPVVkyKvsKtsa1AwhN_LpzidG_XzV3nhtd1cjnzDA,11128
 datahub/sdk/datajob.py,sha256=5kU0txTDcn2ce3AhNry83TazPVhoYZ2rAPPNWM1_FP8,13677
 datahub/sdk/dataset.py,sha256=-C4TCJAs1PFkLAgkUZEU1JOg3orm7AAIkqjw7oo_4PQ,31400
@@ -1114,8 +1121,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.2.0.10rc4.dist-info/METADATA,sha256=KAedvBAi1vl6TlLfQHVllq9dHQqHoTCd3D0CK5Z-MJ0,182842
-acryl_datahub-1.2.0.10rc4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub-1.2.0.10rc4.dist-info/entry_points.txt,sha256=qopCAD6qrsijaZ9mTw3UlPCKsE00C3t9MbkkWow7pi4,9943
-acryl_datahub-1.2.0.10rc4.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.2.0.10rc4.dist-info/RECORD,,
+acryl_datahub-1.2.0.10rc5.dist-info/METADATA,sha256=Nm6aD5BhD2bytkuh33mzJCtDW23cDHNlIw5fboab3dU,184162
+acryl_datahub-1.2.0.10rc5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub-1.2.0.10rc5.dist-info/entry_points.txt,sha256=pzsBoTx-D-iTcmpX8oCGCyzlHP2112EygUMzZWz56M8,10105
+acryl_datahub-1.2.0.10rc5.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.2.0.10rc5.dist-info/RECORD,,

{acryl_datahub-1.2.0.10rc4.dist-info → acryl_datahub-1.2.0.10rc5.dist-info}/entry_points.txt

@@ -94,6 +94,7 @@ sagemaker = datahub.ingestion.source.aws.sagemaker:SagemakerSource
 salesforce = datahub.ingestion.source.salesforce:SalesforceSource
 sigma = datahub.ingestion.source.sigma.sigma:SigmaSource
 slack = datahub.ingestion.source.slack.slack:SlackSource
+snaplogic = datahub.ingestion.source.snaplogic.snaplogic:SnaplogicSource
 snowflake = datahub.ingestion.source.snowflake.snowflake_v2:SnowflakeV2Source
 snowflake-queries = datahub.ingestion.source.snowflake.snowflake_queries:SnowflakeQueriesSource
 snowflake-summary = datahub.ingestion.source.snowflake.snowflake_summary:SnowflakeSummarySource
@@ -130,6 +131,7 @@ pattern_cleanup_dataset_usage_user = datahub.ingestion.transformer.pattern_clean
 pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership
 replace_external_url = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlDataset
 replace_external_url_container = datahub.ingestion.transformer.replace_external_url:ReplaceExternalUrlContainer
+set_browse_path = datahub.ingestion.transformer.set_browse_path:SetBrowsePathTransformer
 set_dataset_browse_path = datahub.ingestion.transformer.add_dataset_browse_path:AddDatasetBrowsePathTransformer
 simple_add_dataset_dataproduct = datahub.ingestion.transformer.add_dataset_dataproduct:SimpleAddDatasetDataProduct
 simple_add_dataset_domain = datahub.ingestion.transformer.dataset_domain:SimpleAddDatasetDomain

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.2.0.10rc4"
+__version__ = "1.2.0.10rc5"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/autogenerated/capability_summary.json

@@ -2968,6 +2968,38 @@
       "platform_name": "Slack",
       "support_status": "TESTING"
     },
+    "snaplogic": {
+      "capabilities": [
+        {
+          "capability": "LINEAGE_FINE",
+          "description": "Enabled by default",
+          "subtype_modifier": null,
+          "supported": true
+        },
+        {
+          "capability": "DELETION_DETECTION",
+          "description": "Not supported yet",
+          "subtype_modifier": null,
+          "supported": false
+        },
+        {
+          "capability": "PLATFORM_INSTANCE",
+          "description": "Snaplogic does not support platform instances",
+          "subtype_modifier": null,
+          "supported": false
+        },
+        {
+          "capability": "LINEAGE_COARSE",
+          "description": "Enabled by default",
+          "subtype_modifier": null,
+          "supported": true
+        }
+      ],
+      "classname": "datahub.ingestion.source.snaplogic.snaplogic.SnaplogicSource",
+      "platform_id": "snaplogic",
+      "platform_name": "Snaplogic",
+      "support_status": "TESTING"
+    },
     "snowflake": {
       "capabilities": [
         {
@@ -3617,4 +3649,4 @@
       "support_status": "CERTIFIED"
     }
   }
-}
+}

datahub/ingestion/source/looker/looker_source.py

@@ -736,7 +736,16 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
                 display_name=dashboard_element.title,  # title is (deprecated) using display_name
                 extra_aspects=chart_extra_aspects,
                 input_datasets=dashboard_element.get_view_urns(self.source_config),
-                last_modified=self._get_last_modified_time(dashboard),
+                last_modified=self._get_last_modified_time(
+                    dashboard
+                ),  # Inherited from Dashboard
+                last_modified_by=self._get_last_modified_by(
+                    dashboard
+                ),  # Inherited from Dashboard
+                created_at=self._get_created_at(dashboard),  # Inherited from Dashboard
+                created_by=self._get_created_by(dashboard),  # Inherited from Dashboard
+                deleted_on=self._get_deleted_on(dashboard),  # Inherited from Dashboard
+                deleted_by=self._get_deleted_by(dashboard),  # Inherited from Dashboard
                 name=dashboard_element.get_urn_element_id(),
                 owners=chart_ownership,
                 parent_container=chart_parent_container,
@@ -803,6 +812,11 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
                 display_name=looker_dashboard.title,  # title is (deprecated) using display_name
                 extra_aspects=dashboard_extra_aspects,
                 last_modified=self._get_last_modified_time(looker_dashboard),
+                last_modified_by=self._get_last_modified_by(looker_dashboard),
+                created_at=self._get_created_at(looker_dashboard),
+                created_by=self._get_created_by(looker_dashboard),
+                deleted_on=self._get_deleted_on(looker_dashboard),
+                deleted_by=self._get_deleted_by(looker_dashboard),
                 name=looker_dashboard.get_urn_dashboard_id(),
                 owners=dashboard_ownership,
                 parent_container=dashboard_parent_container,
@@ -988,9 +1002,44 @@ class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
     def _get_last_modified_time(
         self, looker_dashboard: Optional[LookerDashboard]
     ) -> Optional[datetime.datetime]:
-        if looker_dashboard is None:
+        return looker_dashboard.last_updated_at if looker_dashboard else None
+
+    def _get_last_modified_by(
+        self, looker_dashboard: Optional[LookerDashboard]
+    ) -> Optional[str]:
+        if not looker_dashboard or not looker_dashboard.last_updated_by:
+            return None
+        return looker_dashboard.last_updated_by.get_urn(
+            self.source_config.strip_user_ids_from_email
+        )
+
+    def _get_created_at(
+        self, looker_dashboard: Optional[LookerDashboard]
+    ) -> Optional[datetime.datetime]:
+        return looker_dashboard.created_at if looker_dashboard else None
+
+    def _get_created_by(
+        self, looker_dashboard: Optional[LookerDashboard]
+    ) -> Optional[str]:
+        if not looker_dashboard or not looker_dashboard.owner:
+            return None
+        return looker_dashboard.owner.get_urn(
+            self.source_config.strip_user_ids_from_email
+        )
+
+    def _get_deleted_on(
+        self, looker_dashboard: Optional[LookerDashboard]
+    ) -> Optional[datetime.datetime]:
+        return looker_dashboard.deleted_at if looker_dashboard else None
+
+    def _get_deleted_by(
+        self, looker_dashboard: Optional[LookerDashboard]
+    ) -> Optional[str]:
+        if not looker_dashboard or not looker_dashboard.deleted_by:
             return None
-        return looker_dashboard.last_updated_at
+        return looker_dashboard.deleted_by.get_urn(
+            self.source_config.strip_user_ids_from_email
+        )
 
     def _get_looker_folder(self, folder: Union[Folder, FolderBase]) -> LookerFolder:
         assert folder.id

datahub/ingestion/source/snaplogic/snaplogic.py

@@ -0,0 +1,355 @@
+from typing import Iterable, List, Optional
+
+from datahub.emitter.mce_builder import (
+    make_data_flow_urn,
+    make_data_job_urn,
+    make_data_platform_urn,
+    make_dataset_urn_with_platform_instance,
+    make_schema_field_urn,
+)
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    capability,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.source import (
+    MetadataWorkUnitProcessor,
+    SourceCapability,
+    SourceReport,
+)
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.graph.client import DataHubGraph
+from datahub.ingestion.source.snaplogic.snaplogic_config import SnaplogicConfig
+from datahub.ingestion.source.snaplogic.snaplogic_lineage_extractor import (
+    SnaplogicLineageExtractor,
+)
+from datahub.ingestion.source.snaplogic.snaplogic_parser import (
+    ColumnMapping,
+    Dataset,
+    SnapLogicParser,
+)
+from datahub.ingestion.source.snaplogic.snaplogic_utils import SnaplogicUtils
+from datahub.ingestion.source.state.redundant_run_skip_handler import (
+    RedundantLineageRunSkipHandler,
+)
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StaleEntityRemovalHandler,
+    StaleEntityRemovalSourceReport,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulIngestionSourceBase,
+)
+from datahub.metadata.schema_classes import (
+    DataFlowInfoClass,
+    DataJobInfoClass,
+    DataJobInputOutputClass,
+    DatasetPropertiesClass,
+    FineGrainedLineageClass,
+    FineGrainedLineageDownstreamTypeClass,
+    OtherSchemaClass,
+    SchemaFieldClass,
+    SchemaMetadataClass,
+)
+
+
+@platform_name("Snaplogic")
+@config_class(SnaplogicConfig)
+@support_status(SupportStatus.TESTING)
+@capability(
+    SourceCapability.PLATFORM_INSTANCE,
+    "Snaplogic does not support platform instances",
+    supported=False,
+)
+@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default")
+@capability(SourceCapability.LINEAGE_FINE, "Enabled by default")
+@capability(SourceCapability.DELETION_DETECTION, "Not supported yet", supported=False)
+class SnaplogicSource(StatefulIngestionSourceBase):
+    """
+    A source plugin for ingesting lineage and metadata from Snaplogic.
+    """
+
+    def __init__(self, config: SnaplogicConfig, ctx: PipelineContext):
+        super().__init__(config, ctx)
+        self.config = config
+        self.report = StaleEntityRemovalSourceReport()
+        self.graph: Optional[DataHubGraph] = ctx.graph
+        self.snaplogic_parser = SnapLogicParser(
+            config.case_insensitive_namespaces, self.config.namespace_mapping
+        )
+        self.redundant_lineage_run_skip_handler: Optional[
+            RedundantLineageRunSkipHandler
+        ] = None
+        if self.config.enable_stateful_lineage_ingestion:
+            self.redundant_lineage_run_skip_handler = RedundantLineageRunSkipHandler(
+                source=self,
+                config=self.config,
+                pipeline_name=ctx.pipeline_name,
+                run_id=ctx.run_id,
+            )
+        self.snaplogic_lineage_extractor = SnaplogicLineageExtractor(
+            config=config,
+            redundant_run_skip_handler=self.redundant_lineage_run_skip_handler,
+            report=self.report,
+        )
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        try:
+            self.report.info(
+                message="Starting lineage ingestion from Snaplogic",
+                title="Lineage Ingestion",
+            )
+
+            records_processed = 0
+            for lineage in self.snaplogic_lineage_extractor.get_lineages():
+                try:
+                    for workunit in self._process_lineage_record(lineage):
+                        yield workunit
+                    records_processed += 1
+
+                    if records_processed % 20 == 0:
+                        self.report.info(
+                            message=f"Processed {records_processed} lineage records",
+                            title="Lineage Ingestion Progress",
+                        )
+                except Exception as e:
+                    self.report.report_failure(
+                        message="Failed to process lineage record",
+                        context=str(lineage),
+                        exc=e,
+                    )
+            self.report.info(
+                message=f"Completed processing {records_processed} lineage records",
+                title="Lineage Ingestion Complete",
+            )
+            self.snaplogic_lineage_extractor.report_status("lineage_ingestion", True)
+            self.snaplogic_lineage_extractor.update_stats()
+        except Exception as e:
+            self.report.report_failure(message="Failed to fetch lineages", exc=e)
+            self.snaplogic_lineage_extractor.report_status("lineage_ingestion", False)
+
+    def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]:
+        return [
+            *super().get_workunit_processors(),
+            StaleEntityRemovalHandler.create(
+                self, self.config, self.ctx
+            ).workunit_processor,
+        ]
+
+    def _process_lineage_record(self, lineage: dict) -> Iterable[MetadataWorkUnit]:
+        """Process a lineage record to create pipeline and task workunits with relationships."""
+        producer = lineage.get("producer")
+        if not producer:
+            return
+        pipeline_snode_id = producer.split("#pipe_snode=")[1]
+        if not pipeline_snode_id:
+            return
+        datasets = self.snaplogic_parser.extract_datasets_from_lineage(lineage)
+        pipeline = self.snaplogic_parser.extract_pipeline_from_lineage(lineage)
+        task = self.snaplogic_parser.extract_task_from_lineage(lineage)
+        columns_mapping = self.snaplogic_parser.extract_columns_mapping_from_lineage(
+            lineage
+        )
+
+        # Create pipeline MCP
+        for pipeline_workunit in self.create_pipeline_mcp(
+            name=pipeline.name,
+            pipeline_snode_id=pipeline.id,
+            namespace=pipeline.namespace,
+        ):
+            self.report.report_workunit(pipeline_workunit)
+            yield pipeline_workunit
+
+        # Create dataset MCP
+        for dataset in datasets:
+            for dataset_workunit in self.create_dataset_mcp(
+                dataset_name=dataset.name,
+                dataset_display_name=dataset.display_name,
+                fields=dataset.fields,
+                platform=dataset.platform,
+                platform_instance=dataset.platform_instance,
+            ):
+                self.report.report_workunit(dataset_workunit)
+                yield dataset_workunit
+
+        # Create task MCP
+        for task_workunit in self.create_task_mcp(
+            name=task.name,
+            task_id=task.id,
+            namespace=task.namespace,
+            pipeline_snode_id=pipeline_snode_id,
+            input_datasets=[dataset for dataset in datasets if dataset.type == "INPUT"],
+            output_datasets=[
+                dataset for dataset in datasets if dataset.type == "OUTPUT"
+            ],
+            columns_mapping=columns_mapping,
+        ):
+            self.report.report_workunit(task_workunit)
+            yield task_workunit
+
+    def create_task_mcp(
+        self,
+        task_id: str,
+        name: str,
+        namespace: str,
+        pipeline_snode_id: str,
+        input_datasets: list[Dataset],
+        output_datasets: list[Dataset],
+        columns_mapping: list[ColumnMapping],
+    ) -> Iterable[MetadataWorkUnit]:
+        """Create MCPs for a task (snap) including metadata and lineage."""
+        job_urn = make_data_job_urn(
+            orchestrator=namespace,
+            flow_id=pipeline_snode_id,
+            job_id=task_id,
+            cluster="PROD",
+        )
+        yield MetadataChangeProposalWrapper(
+            entityUrn=job_urn,
+            aspect=DataJobInfoClass(
+                name=name,
+                description="",
+                externalUrl=f"{self.config.base_url}/sl/designer.html?v=21818#pipe_snode={pipeline_snode_id}",
+                type="SNAPLOGIC_SNAP",
+            ),
+        ).as_workunit()
+
+        # Helper functions
+        def dataset_urn(d: Dataset) -> str:
+            return make_dataset_urn_with_platform_instance(
+                d.platform, d.name, d.platform_instance
+            )
+
+        def field_urn(d, f):
+            return make_schema_field_urn(dataset_urn(d), f["name"])
+
+        # Emit lineage
+        yield MetadataChangeProposalWrapper(
+            entityUrn=job_urn,
+            aspect=DataJobInputOutputClass(
+                inputDatasets=[dataset_urn(d) for d in input_datasets],
+                outputDatasets=[dataset_urn(d) for d in output_datasets],
+                inputDatasetFields=[
+                    field_urn(d, f) for d in input_datasets for f in d.fields
+                ],
+                outputDatasetFields=[
+                    field_urn(d, f) for d in output_datasets for f in d.fields
+                ],
+                fineGrainedLineages=[
+                    FineGrainedLineageClass(
+                        upstreamType=FineGrainedLineageDownstreamTypeClass.FIELD_SET,
+                        upstreams=[
+                            make_schema_field_urn(
+                                make_dataset_urn_with_platform_instance(
+                                    cl.input_dataset.platform,
+                                    cl.input_dataset.name,
+                                    cl.input_dataset.platform_instance,
+                                    cl.input_dataset.env,
+                                ),
+                                cl.input_field,
+                            )
+                        ],
+                        downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD_SET,
+                        downstreams=[
+                            make_schema_field_urn(
+                                make_dataset_urn_with_platform_instance(
+                                    cl.output_dataset.platform,
+                                    cl.output_dataset.name,
+                                    cl.output_dataset.platform_instance,
+                                    cl.output_dataset.env,
+                                ),
+                                cl.output_field,
+                            )
+                        ],
+                    )
+                    for cl in columns_mapping
+                ],
+            ),
+        ).as_workunit()
+
+    def create_dataset_mcp(
+        self,
+        dataset_name: str,
+        dataset_display_name: str,
+        fields: list[dict],
+        platform: str = "snaplogic",
+        env: str = "PROD",
+        platform_instance: Optional[str] = None,
+    ) -> Iterable[MetadataWorkUnit]:
+        dataset_urn = make_dataset_urn_with_platform_instance(
+            platform=platform,
+            name=dataset_name,
+            env=env,
+            platform_instance=platform_instance,
+        )
+
+        # Skip dataset creation if:
+        # 1. The platform is not "snaplogic" AND
+        # 2. Either:
+        #    a) The config `create_non_snaplogic_datasets` is disabled (False), meaning
+        #       we do not create datasets for non-snaplogic platforms, OR
+        #    b) The dataset already exists in DataHub (`self.graph.exists(dataset_urn)`).
+        if platform != "snaplogic" and (
+            not self.config.create_non_snaplogic_datasets
+            or (self.graph and self.graph.exists(dataset_urn))
+        ):
+            return
+
+        dataset_properties = DatasetPropertiesClass(
+            name=dataset_display_name,
+            qualifiedName=dataset_name,
+        )
+        schema_fields = [
+            SchemaFieldClass(
+                fieldPath=field["name"],
+                type=SnaplogicUtils.get_datahub_type(field.get("type", "Varchar")),
+                nativeDataType=field.get("type", "Varchar"),
+            )
+            for field in fields
+        ]
+        schema_metadata = SchemaMetadataClass(
+            schemaName=dataset_name,
+            platform=make_data_platform_urn(platform),
+            version=0,
+            hash="",
+            platformSchema=OtherSchemaClass(rawSchema=""),
+            fields=schema_fields,
+        )
+
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn, aspect=dataset_properties
+        ).as_workunit()
+
+        yield MetadataChangeProposalWrapper(
+            entityUrn=dataset_urn, aspect=schema_metadata
+        ).as_workunit()
+
+    def create_pipeline_mcp(
+        self, name: str, namespace: str, pipeline_snode_id: str
+    ) -> Iterable[MetadataWorkUnit]:
+        flow_urn = make_data_flow_urn(
+            orchestrator=namespace, flow_id=pipeline_snode_id, cluster="PROD"
+        )
+
+        yield MetadataChangeProposalWrapper(
+            entityUrn=flow_urn,
+            aspect=DataFlowInfoClass(
+                name=name,
+                description="",
+                externalUrl=f"{self.config.base_url}/sl/designer.html?v=21818#pipe_snode={pipeline_snode_id}",
+            ),
+        ).as_workunit()
+
+    def get_report(self) -> SourceReport:
+        return self.report
+
+    def close(self) -> None:
+        super().close()
+
+    @classmethod
+    def create(cls, config_dict: dict, ctx: PipelineContext) -> "SnaplogicSource":
+        config = SnaplogicConfig.parse_obj(config_dict)
+        return cls(config, ctx)

datahub/ingestion/source/snaplogic/snaplogic_config.py

@@ -0,0 +1,37 @@
+from typing import Optional
+
+from pydantic import Field, SecretStr
+
+from datahub.ingestion.source.state.stale_entity_removal_handler import (
+    StatefulIngestionConfigBase,
+    StatefulStaleMetadataRemovalConfig,
+)
+from datahub.ingestion.source.state.stateful_ingestion_base import (
+    StatefulLineageConfigMixin,
+    StatefulUsageConfigMixin,
+)
+
+
+class SnaplogicConfig(
+    StatefulIngestionConfigBase, StatefulLineageConfigMixin, StatefulUsageConfigMixin
+):
+    platform: str = "Snaplogic"
+    username: str = Field(description="Username")
+    password: SecretStr = Field(description="Password")
+    base_url: str = Field(
+        default="https://elastic.snaplogic.com",
+        description="Url to your Snaplogic instance: `https://elastic.snaplogic.com`, or similar. Used for making API calls to Snaplogic.",
+    )
+    org_name: str = Field(description="Organization name from Snaplogic instance")
+    namespace_mapping: dict = Field(
+        default={}, description="Mapping of namespaces to platform instances"
+    )
+    case_insensitive_namespaces: list = Field(
+        default=[],
+        description="List of namespaces that should be treated as case insensitive",
+    )
+    create_non_snaplogic_datasets: bool = Field(
+        default=False,
+        description="Whether to create datasets for non-Snaplogic datasets (e.g., databases, S3, etc.)",
+    )
+    stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None