openmetadata-ingestion 1.8.3.0__py3-none-any.whl → 1.8.4.0__py3-none-any.whl

metadata/generated/schema/type/entityUsage.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/entityUsage.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/filterPattern.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/filterPattern.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/function.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/function.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/include.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/include.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-07-10T07:55:35+00:00
+#   timestamp: 2025-07-16T10:25:30+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/database/common_db_source.py

@@ -217,16 +217,17 @@ class CommonDbSourceService(
             else None
         )
 
-        yield Either(
-            right=CreateDatabaseRequest(
-                name=EntityName(database_name),
-                service=FullyQualifiedEntityName(self.context.get().database_service),
-                description=description,
-                sourceUrl=source_url,
-                tags=self.get_database_tag_labels(database_name=database_name),
-            )
+        database_request = CreateDatabaseRequest(
+            name=EntityName(database_name),
+            service=FullyQualifiedEntityName(self.context.get().database_service),
+            description=description,
+            sourceUrl=source_url,
+            tags=self.get_database_tag_labels(database_name=database_name),
         )
 
+        yield Either(right=database_request)
+        self.register_record_database_request(database_request=database_request)
+
     def get_raw_database_schema_names(self) -> Iterable[str]:
         if self.service_connection.__dict__.get("databaseSchema"):
             yield self.service_connection.databaseSchema
@@ -264,23 +265,24 @@ class CommonDbSourceService(
             else None
         )
 
-        yield Either(
-            right=CreateDatabaseSchemaRequest(
-                name=EntityName(schema_name),
-                database=FullyQualifiedEntityName(
-                    fqn.build(
-                        metadata=self.metadata,
-                        entity_type=Database,
-                        service_name=self.context.get().database_service,
-                        database_name=self.context.get().database,
-                    )
-                ),
-                description=description,
-                sourceUrl=source_url,
-                tags=self.get_schema_tag_labels(schema_name=schema_name),
-            )
+        schema_request = CreateDatabaseSchemaRequest(
+            name=EntityName(schema_name),
+            database=FullyQualifiedEntityName(
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                )
+            ),
+            description=description,
+            sourceUrl=source_url,
+            tags=self.get_schema_tag_labels(schema_name=schema_name),
         )
 
+        yield Either(right=schema_request)
+        self.register_record_schema_request(schema_request=schema_request)
+
     @staticmethod
     @calculate_execution_time()
     def get_table_description(

metadata/ingestion/source/database/common_nosql_source.py

@@ -122,13 +122,16 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
         Prepare a database request and pass it to the sink
         """
 
+        database_request = CreateDatabaseRequest(
+            name=EntityName(database_name),
+            service=self.context.get().database_service,
+            sourceUrl=self.get_source_url(database_name=database_name),
+        )
+
         yield Either(
-            right=CreateDatabaseRequest(
-                name=EntityName(database_name),
-                service=self.context.get().database_service,
-                sourceUrl=self.get_source_url(database_name=database_name),
-            )
+            right=database_request,
         )
+        self.register_record_database_request(database_request=database_request)
 
     @abstractmethod
     def get_schema_name_list(self) -> List[str]:
@@ -164,24 +167,25 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
         Prepare a database schema request and pass it to the sink
         """
 
-        yield Either(
-            right=CreateDatabaseSchemaRequest(
-                name=EntityName(schema_name),
-                database=FullyQualifiedEntityName(
-                    fqn.build(
-                        metadata=self.metadata,
-                        entity_type=Database,
-                        service_name=self.context.get().database_service,
-                        database_name=self.context.get().database,
-                    )
-                ),
-                sourceUrl=self.get_source_url(
+        schema_request = CreateDatabaseSchemaRequest(
+            name=EntityName(schema_name),
+            database=FullyQualifiedEntityName(
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.get().database_service,
                     database_name=self.context.get().database,
-                    schema_name=schema_name,
-                ),
-            )
+                )
+            ),
+            sourceUrl=self.get_source_url(
+                database_name=self.context.get().database,
+                schema_name=schema_name,
+            ),
         )
 
+        yield Either(right=schema_request)
+        self.register_record_schema_request(schema_request=schema_request)
+
     @abstractmethod
     def query_table_names_and_types(
         self, schema_name: str

metadata/ingestion/source/database/database_service.py

@@ -135,6 +135,7 @@ class DatabaseServiceTopology(ServiceTopology):
             ),
         ],
         children=["databaseSchema"],
+        post_process=["mark_databases_as_deleted"],
     )
     databaseSchema: Annotated[
         TopologyNode, Field(description="Database Schema Node")
@@ -158,7 +159,11 @@ class DatabaseServiceTopology(ServiceTopology):
             ),
         ],
         children=["table", "stored_procedure"],
-        post_process=["mark_tables_as_deleted", "mark_stored_procedures_as_deleted"],
+        post_process=[
+            "mark_schemas_as_deleted",
+            "mark_tables_as_deleted",
+            "mark_stored_procedures_as_deleted",
+        ],
         threads=True,
     )
     table: Annotated[
@@ -217,6 +222,8 @@ class DatabaseServiceSource(
     config: WorkflowSource
     database_source_state: Set = set()
     stored_procedure_source_state: Set = set()
+    database_entity_source_state: Set = set()
+    schema_entity_source_state: Set = set()
     # Big union of types we want to fetch dynamically
     service_connection: DatabaseConnection.model_fields["config"].annotation
 
@@ -494,6 +501,64 @@ class DatabaseServiceSource(
 
         self.stored_procedure_source_state.add(table_fqn)
 
+    def register_record_database_request(
+        self, database_request: CreateDatabaseRequest
+    ) -> None:
+        """
+        Mark the database record as scanned and update the database_entity_source_state
+        """
+        database_fqn = fqn.build(
+            self.metadata,
+            entity_type=Database,
+            service_name=self.context.get().database_service,
+            database_name=database_request.name.root,
+        )
+
+        self.database_entity_source_state.add(database_fqn)
+
+    def register_record_schema_request(
+        self, schema_request: CreateDatabaseSchemaRequest
+    ) -> None:
+        """
+        Mark the schema record as scanned and update the schema_entity_source_state
+        """
+        schema_fqn = fqn.build(
+            self.metadata,
+            entity_type=DatabaseSchema,
+            service_name=self.context.get().database_service,
+            database_name=self.context.get().database,
+            schema_name=schema_request.name.root,
+        )
+
+        self.schema_entity_source_state.add(schema_fqn)
+
+    def _get_filtered_database_names(
+        self, return_fqn: bool = False, add_to_status: bool = True
+    ) -> Iterable[str]:
+        """
+        Get filtered database names based on the database filter pattern
+        """
+        database_names_iterable = getattr(
+            self, "get_database_names_raw", self.get_database_names
+        )()
+        for database_name in database_names_iterable:
+            database_fqn = fqn.build(
+                self.metadata,
+                entity_type=Database,
+                service_name=self.context.get().database_service,
+                database_name=database_name,
+            )
+            if filter_by_schema(
+                self.source_config.databaseFilterPattern,
+                database_fqn
+                if self.source_config.useFqnForFiltering
+                else database_name,
+            ):
+                if add_to_status:
+                    self.status.filter(database_fqn, "Database Filtered Out")
+                continue
+            yield database_fqn if return_fqn else database_name
+
     def _get_filtered_schema_names(
         self, return_fqn: bool = False, add_to_status: bool = True
     ) -> Iterable[str]:
@@ -585,6 +650,89 @@ class DatabaseServiceSource(
                     params={"databaseSchema": schema_fqn},
                 )
 
+    def mark_databases_as_deleted(self):
+        """
+        Use the current inspector to mark databases as deleted
+        """
+        if self.source_config.markDeletedDatabases:
+            logger.info(
+                f"Mark Deleted Databases set to True. Processing service [{self.context.get().database_service}]"
+            )
+
+            # We need to include ALL databases from the source in the source state
+            # This includes both processed databases and all databases (filtered-in and filtered-out)
+            # to ensure we mark as deleted any databases that were previously ingested but are now
+            # filtered out, as well as any databases that were processed in this run
+            all_database_fqns = set()
+
+            # Get all databases from the source (both filtered-in and filtered-out)
+            for database_name in self._get_filtered_database_names():
+                database_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.get().database_service,
+                    database_name=database_name,
+                )
+                all_database_fqns.add(database_fqn)
+
+            # Combine the processed databases with all databases from source
+            complete_db_source_state = self.database_entity_source_state.union(
+                all_database_fqns
+            )
+
+            yield from delete_entity_from_source(
+                metadata=self.metadata,
+                entity_type=Database,
+                entity_source_state=complete_db_source_state,
+                mark_deleted_entity=self.source_config.markDeletedDatabases,
+                params={"service": self.context.get().database_service},
+            )
+
+    def mark_schemas_as_deleted(self):
+        """
+        Use the current inspector to mark schemas as deleted
+        """
+        if not self.context.get().__dict__.get("database"):
+            raise ValueError(
+                "No Database found in the context. We cannot run the schema deletion."
+            )
+
+        if self.source_config.markDeletedSchemas:
+            logger.info(
+                f"Mark Deleted Schemas set to True. Processing database [{self.context.get().database}]"
+            )
+
+            # Build the database FQN to use as parameter
+            database_fqn = fqn.build(
+                self.metadata,
+                entity_type=Database,
+                service_name=self.context.get().database_service,
+                database_name=self.context.get().database,
+            )
+
+            # Get all filtered-in schema FQNs to create a complete source state
+            # We need to include both processed schemas and filtered schemas in the source state
+            # to ensure we mark as deleted any schemas that were previously ingested but are now
+            # filtered out, as well as any schemas that were processed in this run
+            filtered_schema_fqns = set()
+            for schema_name in self._get_filtered_schema_names(
+                return_fqn=True, add_to_status=False
+            ):
+                filtered_schema_fqns.add(schema_name)
+
+            # Combine the processed schemas with filtered schemas
+            complete_source_state = self.schema_entity_source_state.union(
+                filtered_schema_fqns
+            )
+
+            yield from delete_entity_from_source(
+                metadata=self.metadata,
+                entity_type=DatabaseSchema,
+                entity_source_state=complete_source_state,
+                mark_deleted_entity=self.source_config.markDeletedSchemas,
+                params={"database": database_fqn},
+            )
+
     def yield_life_cycle_data(self, _) -> Iterable[Either[OMetaLifeCycleData]]:
         """
         Get the life cycle data of the table

metadata/ingestion/source/database/datalake/metadata.py

@@ -157,12 +157,13 @@ class DatalakeSource(DatabaseServiceSource):
         """
         if isinstance(self.config_source, GCSConfig):
             database_name = self.client.project
-        yield Either(
-            right=CreateDatabaseRequest(
-                name=EntityName(database_name),
-                service=FullyQualifiedEntityName(self.context.get().database_service),
-            )
+
+        database_request = CreateDatabaseRequest(
+            name=EntityName(database_name),
+            service=FullyQualifiedEntityName(self.context.get().database_service),
         )
+        yield Either(right=database_request)
+        self.register_record_database_request(database_request=database_request)
 
     def get_database_schema_names(self) -> Iterable[str]:
         """
@@ -208,20 +209,21 @@ class DatalakeSource(DatabaseServiceSource):
         From topology.
         Prepare a database schema request and pass it to the sink
         """
-        yield Either(
-            right=CreateDatabaseSchemaRequest(
-                name=EntityName(schema_name),
-                database=FullyQualifiedEntityName(
-                    fqn.build(
-                        metadata=self.metadata,
-                        entity_type=Database,
-                        service_name=self.context.get().database_service,
-                        database_name=self.context.get().database,
-                    )
-                ),
-            )
+        schema_request = CreateDatabaseSchemaRequest(
+            name=EntityName(schema_name),
+            database=FullyQualifiedEntityName(
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                )
+            ),
         )
 
+        yield Either(right=schema_request)
+        self.register_record_schema_request(schema_request=schema_request)
+
     def get_tables_name_and_type(  # pylint: disable=too-many-branches
         self,
     ) -> Iterable[Tuple[str, TableType, SupportedTypes]]:

metadata/ingestion/source/database/deltalake/metadata.py

@@ -115,12 +115,12 @@ class DeltalakeSource(DatabaseServiceSource):
         From topology.
         Prepare a database request and pass it to the sink
         """
-        yield Either(
-            right=CreateDatabaseRequest(
-                name=database_name,
-                service=self.context.get().database_service,
-            )
+        database_request = CreateDatabaseRequest(
+            name=database_name,
+            service=self.context.get().database_service,
         )
+        yield Either(right=database_request)
+        self.register_record_database_request(database_request=database_request)
 
     def get_database_schema_names(self) -> Iterable[str]:
         """
@@ -151,20 +151,21 @@ class DeltalakeSource(DatabaseServiceSource):
         From topology.
         Prepare a database schema request and pass it to the sink
         """
-        yield Either(
-            right=CreateDatabaseSchemaRequest(
-                name=EntityName(schema_name),
-                database=FullyQualifiedEntityName(
-                    fqn.build(
-                        metadata=self.metadata,
-                        entity_type=Database,
-                        service_name=self.context.get().database_service,
-                        database_name=self.context.get().database,
-                    )
-                ),
-            )
+        schema_request = CreateDatabaseSchemaRequest(
+            name=EntityName(schema_name),
+            database=FullyQualifiedEntityName(
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                )
+            ),
         )
 
+        yield Either(right=schema_request)
+        self.register_record_schema_request(schema_request=schema_request)
+
     def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
         """
         Handle table and views.

metadata/ingestion/source/database/domodatabase/metadata.py

@@ -109,12 +109,13 @@ class DomodatabaseSource(DatabaseServiceSource):
     def yield_database(
         self, database_name: str
     ) -> Iterable[Either[CreateDatabaseRequest]]:
-        yield Either(
-            right=CreateDatabaseRequest(
-                name=EntityName(database_name),
-                service=self.context.get().database_service,
-            )
+
+        database_request = CreateDatabaseRequest(
+            name=EntityName(database_name),
+            service=self.context.get().database_service,
         )
+        yield Either(right=database_request)
+        self.register_record_database_request(database_request=database_request)
 
     def get_database_schema_names(self) -> Iterable[str]:
         scheme_name = "default"
@@ -123,19 +124,19 @@ class DomodatabaseSource(DatabaseServiceSource):
     def yield_database_schema(
         self, schema_name: str
     ) -> Iterable[Either[CreateDatabaseSchemaRequest]]:
-        yield Either(
-            right=CreateDatabaseSchemaRequest(
-                name=EntityName(schema_name),
-                database=FullyQualifiedEntityName(
-                    fqn.build(
-                        metadata=self.metadata,
-                        entity_type=Database,
-                        service_name=self.context.get().database_service,
-                        database_name=self.context.get().database,
-                    )
-                ),
-            )
+        schema_request = CreateDatabaseSchemaRequest(
+            name=EntityName(schema_name),
+            database=FullyQualifiedEntityName(
+                fqn.build(
+                    metadata=self.metadata,
+                    entity_type=Database,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                )
+            ),
         )
+        yield Either(right=schema_request)
+        self.register_record_schema_request(schema_request=schema_request)
 
     def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
         schema_name = self.context.get().database_schema