openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/recognizer.py

@@ -0,0 +1,159 @@
+# generated by datamodel-codegen:
+#   filename:  type/recognizer.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Optional, Union
+
+from pydantic import ConfigDict, Field, RootModel
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from . import (
+    basic,
+    contextRecognizer,
+    customRecognizer,
+    denyListRecognizer,
+    entityHistory,
+    entityReference,
+    patternRecognizer,
+    predefinedRecognizer,
+)
+
+
+class Target(Enum):
+    content = 'content'
+    column_name = 'column_name'
+
+
+class RecognizerException(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    entityLink: Annotated[
+        basic.EntityLink, Field(description='Entity link to exclude from recognition')
+    ]
+    reason: Annotated[Optional[str], Field(None, description='Reason for exclusion')]
+    addedBy: Annotated[
+        Optional[entityReference.EntityReference],
+        Field(None, description='User who added this exception'),
+    ]
+    addedAt: Optional[basic.Timestamp] = None
+    feedbackId: Annotated[
+        Optional[basic.Uuid],
+        Field(None, description='ID of the feedback that triggered this exception'),
+    ]
+
+
+class RecognizerType(Enum):
+    pattern = 'pattern'
+    deny_list = 'deny_list'
+    context = 'context'
+    predefined = 'predefined'
+    custom = 'custom'
+
+
+class PredefinedRecognizerType(RootModel[predefinedRecognizer.PredefinedRecognizer]):
+    root: predefinedRecognizer.PredefinedRecognizer
+
+
+class RecognizerConfig(
+    RootModel[
+        Union[
+            patternRecognizer.PatternRecognizer,
+            denyListRecognizer.DenyListRecognizer,
+            contextRecognizer.ContextRecognizer,
+            customRecognizer.CustomRecognizer,
+            predefinedRecognizer.PredefinedRecognizer,
+        ]
+    ]
+):
+    root: Annotated[
+        Union[
+            patternRecognizer.PatternRecognizer,
+            denyListRecognizer.DenyListRecognizer,
+            contextRecognizer.ContextRecognizer,
+            customRecognizer.CustomRecognizer,
+            predefinedRecognizer.PredefinedRecognizer,
+        ],
+        Field(description='Complete recognizer configuration'),
+    ]
+
+
+class PatternRecognizerType(RootModel[patternRecognizer.PatternRecognizer]):
+    root: patternRecognizer.PatternRecognizer
+
+
+class DenyListRecognizerType(RootModel[denyListRecognizer.DenyListRecognizer]):
+    root: denyListRecognizer.DenyListRecognizer
+
+
+class ContextRecognizerType(RootModel[contextRecognizer.ContextRecognizer]):
+    root: contextRecognizer.ContextRecognizer
+
+
+class CustomRecognizerType(RootModel[customRecognizer.CustomRecognizer]):
+    root: customRecognizer.CustomRecognizer
+
+
+class Recognizer(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    id: Annotated[
+        Optional[basic.Uuid],
+        Field(None, description='Unique identifier of the recognizer'),
+    ]
+    name: Annotated[basic.EntityName, Field(description='Name of the recognizer')]
+    displayName: Annotated[
+        Optional[str], Field(None, description='Display name for the recognizer')
+    ]
+    description: Annotated[
+        Optional[basic.Markdown],
+        Field(None, description='Description of what this recognizer detects'),
+    ]
+    enabled: Annotated[
+        Optional[bool], Field(True, description='Whether this recognizer is enabled')
+    ]
+    isSystemDefault: Annotated[
+        Optional[bool],
+        Field(False, description='Whether this is a system default recognizer'),
+    ]
+    recognizerConfig: RecognizerConfig
+    confidenceThreshold: Annotated[
+        Optional[float],
+        Field(
+            0.6,
+            description='Minimum confidence score required for detection',
+            ge=0.0,
+            le=1.0,
+        ),
+    ]
+    exceptionList: Annotated[
+        Optional[List[RecognizerException]],
+        Field(
+            [],
+            description='Entity links where this recognizer should NOT run (based on user feedback)',
+        ),
+    ]
+    version: Annotated[
+        Optional[entityHistory.EntityVersion],
+        Field(None, description='Version of the recognizer configuration'),
+    ]
+    updatedAt: Annotated[
+        Optional[basic.Timestamp],
+        Field(None, description='Last update time in Unix epoch time milliseconds'),
+    ]
+    updatedBy: Annotated[
+        Optional[str], Field(None, description='User who made the update')
+    ]
+    target: Annotated[
+        Optional[Target],
+        Field(
+            Target.content,
+            description="What the recognizer will analyze for classification. `column_name` means it classifies based on the column's name, `content` uses sample data from the column instead.",
+        ),
+    ]

metadata/generated/schema/type/recognizerFeedback.py

@@ -0,0 +1,120 @@
+# generated by datamodel-codegen:
+#   filename:  type/recognizerFeedback.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import List, Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+from . import basic, entityReference
+
+
+class FeedbackType(Enum):
+    FALSE_POSITIVE = 'FALSE_POSITIVE'
+    INCORRECT_CLASSIFICATION = 'INCORRECT_CLASSIFICATION'
+    OVERLY_BROAD = 'OVERLY_BROAD'
+    CONTEXT_SPECIFIC = 'CONTEXT_SPECIFIC'
+
+
+class UserReason(Enum):
+    NOT_SENSITIVE_DATA = 'NOT_SENSITIVE_DATA'
+    WRONG_DATA_TYPE = 'WRONG_DATA_TYPE'
+    INTERNAL_IDENTIFIER = 'INTERNAL_IDENTIFIER'
+    PUBLIC_INFORMATION = 'PUBLIC_INFORMATION'
+    TEST_DATA = 'TEST_DATA'
+    ENCRYPTED_DATA = 'ENCRYPTED_DATA'
+    OTHER = 'OTHER'
+
+
+class RecognizerInfo(BaseModel):
+    recognizerId: Optional[basic.Uuid] = None
+    recognizerName: Optional[str] = None
+    matchPattern: Annotated[
+        Optional[str],
+        Field(None, description='The pattern that matched (for debugging)'),
+    ]
+    confidenceScore: Optional[float] = None
+
+
+class Status(Enum):
+    PENDING = 'PENDING'
+    REVIEWED = 'REVIEWED'
+    APPLIED = 'APPLIED'
+    REJECTED = 'REJECTED'
+
+
+class Action(Enum):
+    ADDED_TO_EXCEPTION_LIST = 'ADDED_TO_EXCEPTION_LIST'
+    PATTERN_ADJUSTED = 'PATTERN_ADJUSTED'
+    THRESHOLD_INCREASED = 'THRESHOLD_INCREASED'
+    RECOGNIZER_DISABLED_FOR_ENTITY = 'RECOGNIZER_DISABLED_FOR_ENTITY'
+    NO_ACTION_NEEDED = 'NO_ACTION_NEEDED'
+
+
+class Resolution(BaseModel):
+    action: Optional[Action] = None
+    resolvedBy: Optional[entityReference.EntityReference] = None
+    resolvedAt: Optional[basic.Timestamp] = None
+    resolutionNotes: Optional[str] = None
+
+
+class RecognizerFeedback(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    id: Annotated[
+        Optional[basic.Uuid],
+        Field(None, description='Unique identifier of the feedback'),
+    ]
+    entityLink: Annotated[
+        basic.EntityLink,
+        Field(
+            description='Link to the specific field where the tag was incorrectly applied (e.g., <#E::table::customers::columns::company_name>)'
+        ),
+    ]
+    tagFQN: Annotated[
+        basic.FullyQualifiedEntityName,
+        Field(description='Fully qualified name of the incorrectly applied tag'),
+    ]
+    feedbackType: Annotated[FeedbackType, Field(description='Type of feedback')]
+    userReason: Annotated[
+        Optional[UserReason],
+        Field(None, description='User-selected reason for reporting'),
+    ]
+    userComments: Annotated[
+        Optional[str], Field(None, description='Additional context from the user')
+    ]
+    suggestedTag: Annotated[
+        Optional[basic.FullyQualifiedEntityName],
+        Field(
+            None, description='Tag the user thinks should be applied instead (optional)'
+        ),
+    ]
+    sampleValues: Annotated[
+        Optional[List[str]],
+        Field(
+            None,
+            description='Example values from this field that triggered the false positive (anonymized)',
+        ),
+    ]
+    recognizerInfo: Annotated[
+        Optional[RecognizerInfo],
+        Field(None, description='Information about which recognizer triggered this'),
+    ]
+    createdBy: Annotated[
+        Optional[entityReference.EntityReference],
+        Field(None, description='User who provided the feedback'),
+    ]
+    createdAt: Optional[basic.Timestamp] = None
+    status: Annotated[
+        Optional[Status], Field(Status.PENDING, description='Processing status')
+    ]
+    resolution: Annotated[
+        Optional[Resolution], Field(None, description='How this feedback was resolved')
+    ]

metadata/generated/schema/type/recognizers/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   timestamp: 2025-12-14T12:54:10+00:00

metadata/generated/schema/type/recognizers/patterns.py

@@ -0,0 +1,29 @@
+# generated by datamodel-codegen:
+#   filename:  type/recognizers/patterns.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+
+class Pattern(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    name: Annotated[str, Field(description='Name of the pattern')]
+    regex: Annotated[str, Field(description='Regular expression pattern')]
+    score: Annotated[
+        Optional[float],
+        Field(
+            0.8,
+            description='Confidence score for this pattern (0.0 to 1.0)',
+            ge=0.0,
+            le=1.0,
+        ),
+    ]

metadata/generated/schema/type/recognizers/regexFlags.py

@@ -0,0 +1,29 @@
+# generated by datamodel-codegen:
+#   filename:  type/recognizers/regexFlags.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import ConfigDict, Field
+from typing_extensions import Annotated
+
+from metadata.ingestion.models.custom_pydantic import BaseModel
+
+
+class RegexFlags(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    dotAll: Annotated[
+        Optional[bool], Field(True, description='DOTALL flag - `.` matches newlines')
+    ]
+    multiline: Annotated[
+        Optional[bool],
+        Field(True, description='MULTILINE flag - `^` and `$` match line boundaries'),
+    ]
+    ignoreCase: Annotated[
+        Optional[bool],
+        Field(True, description='IGNORECASE flag - case insensitive matching'),
+    ]

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/status.py

@@ -0,0 +1,16 @@
+# generated by datamodel-codegen:
+#   filename:  type/status.json
+#   timestamp: 2025-12-14T12:54:10+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class EntityStatus(Enum):
+    Draft = 'Draft'
+    In_Review = 'In Review'
+    Approved = 'Approved'
+    Deprecated = 'Deprecated'
+    Rejected = 'Rejected'
+    Unprocessed = 'Unprocessed'

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 
@@ -69,3 +69,10 @@ class TagLabel(BaseModel):
     href: Annotated[
         Optional[basic.Href], Field(None, description='Link to the tag resource.')
     ]
+    reason: Annotated[
+        Optional[str],
+        Field(
+            None,
+            description='An explanation of why this tag was proposed, specially for autoclassification tags',
+        ),
+    ]

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2025-08-08T13:50:05+00:00
+#   timestamp: 2025-12-14T12:54:10+00:00
 
 from __future__ import annotations
 

metadata/great_expectations/action1xx.py

@@ -86,7 +86,8 @@ class OpenMetadataValidationAction1xx(ValidationAction):
     # This will be initialized in the run method
     ometa_conn: Optional[OpenMetadata] = None
 
-    def run(  # pylint: disable=unused-argument, arguments-differ
+    # pylint: disable=unused-argument,arguments-differ
+    def run(
         self,
         checkpoint_result: CheckpointResult,
         action_context: Union[ActionContext, None],

metadata/ingestion/api/status.py

@@ -53,6 +53,7 @@ class Status(BaseModel):
     )
 
     records: Annotated[List[Any], Field(default_factory=list)]
+    record_count: int = Field(default=0)
     updated_records: Annotated[List[Any], Field(default_factory=list)]
     warnings: Annotated[List[Any], Field(default_factory=list)]
     filtered: Annotated[List[Dict[str, str]], Field(default_factory=list)]
@@ -75,6 +76,9 @@ class Status(BaseModel):
         if log_name := get_log_name(record):
             self.updated_records.append(log_name)
 
+    def increment_record_count(self, increment: int = 1) -> None:
+        self.record_count += increment
+
     def warning(self, key: str, reason: str) -> None:
         self.warnings.append({key: reason})
 
@@ -108,8 +112,9 @@ class Status(BaseModel):
         self.failures.extend(failures)
 
     def calculate_success(self) -> float:
+        record_count = self.record_count if self.record_count > 0 else len(self.records)
         source_success = max(
-            len(self.records) + len(self.updated_records), 1
+            record_count + len(self.updated_records), 1
         )  # To avoid ZeroDivisionError using minimum value as 1
         source_failed = len(self.failures)
         return round(source_success * 100 / (source_success + source_failed), 2)

metadata/ingestion/api/step.py

@@ -77,7 +77,9 @@ class Summary(StepSummary):
         """Compute summary from Step"""
         return Summary(
             name=step.name,
-            records=len(step.status.records),
+            records=step.status.record_count
+            if step.status.record_count > 0
+            else len(step.status.records),
             updated_records=len(step.status.updated_records),
             warnings=len(step.status.warnings),
             errors=len(step.status.failures),

metadata/ingestion/bulksink/metadata_usage.py

@@ -122,10 +122,18 @@ class MetadataUsageBulkSink(BulkSink):
                 "database": table_usage.databaseName,
                 "database_schema": table_usage.databaseSchema,
             }
+            logger.debug(
+                f"[UsageSink] Added new table usage entry for {table_entity.id.root} "
+                f"(count={table_usage.count}, date={table_usage.date})"
+            )
         else:
             self.table_usage_map[table_entity.id.root][
                 "usage_count"
             ] += table_usage.count
+            logger.debug(
+                f"[UsageSink] Updated usage count for {table_entity.id.root} "
+                f"(+={table_usage.count}, total={self.table_usage_map[table_entity.id.root]['usage_count']})"
+            )
 
     def __publish_usage_records(self) -> None:
         """
@@ -196,9 +204,17 @@ class MetadataUsageBulkSink(BulkSink):
                 self.service_name = table_usage.serviceName
                 table_entities = None
                 try:
+                    logger.debug(
+                        f"[UsageSink] Fetching table entities for "
+                        f"service={self.service_name}, "
+                        f"database={table_usage.databaseName}, "
+                        f"schema={table_usage.databaseSchema}, "
+                        f"table={table_usage.table}"
+                    )
+
                     table_entities = get_table_entities_from_query(
                         metadata=self.metadata,
-                        service_name=self.service_name,
+                        service_names=self.service_name,
                         database_name=table_usage.databaseName,
                         database_schema=table_usage.databaseSchema,
                         table_name=table_usage.table,
@@ -239,6 +255,7 @@ class MetadataUsageBulkSink(BulkSink):
         tables and publish the join information.
         """
         for table_entity in table_entities:
+            logger.debug(f"Processing table entity {table_entity.name.root}")
             if table_entity is not None:
                 table_join_request = None
                 try:
@@ -359,7 +376,7 @@ class MetadataUsageBulkSink(BulkSink):
         """
         table_entities = get_table_entities_from_query(
             metadata=self.metadata,
-            service_name=self.service_name,
+            service_names=self.service_name,
             database_name=database,
             database_schema=database_schema,
             table_name=table_column.table,

metadata/ingestion/connections/builders.py

@@ -56,7 +56,10 @@ def get_connection_args_common(connection) -> Dict[str, Any]:
 
 
 def create_generic_db_connection(
-    connection, get_connection_url_fn: Callable, get_connection_args_fn: Callable
+    connection,
+    get_connection_url_fn: Callable,
+    get_connection_args_fn: Callable,
+    **kwargs,
 ) -> Engine:
     """
     Generic Engine creation from connection object
@@ -75,6 +78,7 @@ def create_generic_db_connection(
         pool_reset_on_return=None,  # https://docs.sqlalchemy.org/en/14/core/pooling.html#reset-on-return
         echo=False,
         max_overflow=-1,
+        **kwargs,
     )
 
     attach_query_tracker(engine)

metadata/ingestion/lineage/masker.py

@@ -21,6 +21,7 @@ from sqlparse.sql import Comparison
 from sqlparse.tokens import Literal, Number, String
 
 from metadata.ingestion.lineage.models import Dialect
+from metadata.utils.execution_time_tracker import calculate_execution_time
 
 MASK_TOKEN = "?"
 
@@ -113,10 +114,12 @@ def mask_literals_with_sqlfluff(query: str, parser: LineageRunner) -> str:
     return query
 
 
+@calculate_execution_time(context="MaskQuery")
 def mask_query(
     query: str,
     dialect: str = Dialect.ANSI.value,
     parser: Optional[LineageRunner] = None,
+    parser_required: bool = False,
 ) -> str:
     """
     Mask a query using sqlparse or sqlfluff.
@@ -125,6 +128,8 @@ def mask_query(
     try:
         if masked_query_cache.get((query, dialect)):
             return masked_query_cache.get((query, dialect))
+        if parser_required and not parser:
+            return None
         if not parser:
             try:
                 parser = LineageRunner(query, dialect=dialect)

metadata/ingestion/lineage/models.py

@@ -40,6 +40,9 @@ from metadata.generated.schema.entity.services.connections.database.deltaLakeCon
 from metadata.generated.schema.entity.services.connections.database.exasolConnection import (
     ExasolType,
 )
+from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
+    GreenplumType,
+)
 from metadata.generated.schema.entity.services.connections.database.hiveConnection import (
     HiveType,
 )
@@ -143,6 +146,7 @@ MAP_CONNECTION_TYPE_DIALECT: Dict[str, Dialect] = {
     str(ExasolType.Exasol.value): Dialect.EXASOL,
     str(TrinoType.Trino.value): Dialect.TRINO,
     str(VerticaType.Vertica.value): Dialect.VERTICA,
+    str(GreenplumType.Greenplum.value): Dialect.POSTGRES,
 }