openmetadata-ingestion 1.4.4.0rc1__py3-none-any.whl → 1.4.4.2__py3-none-any.whl

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-06-26T06:28:13+00:00
+#   timestamp: 2024-07-09T05:45:21+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-06-26T06:28:13+00:00
+#   timestamp: 2024-07-09T05:45:21+00:00
 
 from __future__ import annotations
 

metadata/ingestion/lineage/models.py

@@ -64,6 +64,9 @@ from metadata.generated.schema.entity.services.connections.database.snowflakeCon
 from metadata.generated.schema.entity.services.connections.database.sqliteConnection import (
     SQLiteType,
 )
+from metadata.generated.schema.entity.services.connections.database.teradataConnection import (
+    TeradataType,
+)
 from metadata.utils.singleton import Singleton
 
 
@@ -112,6 +115,7 @@ MAP_CONNECTION_TYPE_DIALECT: Dict[str, Dialect] = {
     str(SQLiteType.SQLite.value): Dialect.SQLITE,
     str(MssqlType.Mssql.value): Dialect.TSQL,
     str(AzureSQLType.AzureSQL.value): Dialect.TSQL,
+    str(TeradataType.Teradata.value): Dialect.TERADATA,
 }
 
 

metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -13,7 +13,7 @@ REST Auth & Client for QlikCloud
 """
 import json
 import traceback
-from typing import Dict, List, Optional
+from typing import Dict, Iterable, List, Optional
 
 from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import (
     QlikCloudConnection,
@@ -26,7 +26,10 @@ from metadata.ingestion.source.dashboard.qlikcloud.constants import (
     GET_SHEET_LAYOUT,
     OPEN_DOC_REQ,
 )
-from metadata.ingestion.source.dashboard.qlikcloud.models import QlikApp, QlikAppList
+from metadata.ingestion.source.dashboard.qlikcloud.models import (
+    QlikApp,
+    QlikAppResponse,
+)
 from metadata.ingestion.source.dashboard.qliksense.models import (
     QlikDataModelResult,
     QlikSheet,
@@ -40,6 +43,7 @@ from metadata.utils.logger import ingestion_logger
 logger = ingestion_logger()
 
 API_VERSION = "api"
+API_LIMIT = 100
 
 
 class QlikCloudClient:
@@ -53,13 +57,12 @@ class QlikCloudClient:
     ):
         self.config = config
         self.socket_connection = None
-        self.config.token = self.config.token.get_secret_value()
 
         client_config: ClientConfig = ClientConfig(
             base_url=self.config.hostPort,
             api_version=API_VERSION,
             auth_header=AUTHORIZATION_HEADER,
-            auth_token=lambda: (self.config.token, 0),
+            auth_token=lambda: (self.config.token.get_secret_value(), 0),
         )
         self.client = REST(client_config)
 
@@ -77,7 +80,7 @@ class QlikCloudClient:
         self.socket_connection = create_connection(
             f"wss://{clean_uri(self.config.hostPort.host)}/app/{dashboard_id or ''}",
             sslopt={"cert_reqs": ssl.CERT_NONE},
-            header={"Authorization": f"Bearer {self.config.token}"},
+            header={"Authorization": f"Bearer {self.config.token.get_secret_value()}"},
         )
         self.socket_connection.recv()
 
@@ -117,19 +120,32 @@ class QlikCloudClient:
             logger.warning("Failed to fetch the dashboard charts")
         return []
 
-    def get_dashboards_list(self) -> List[QlikAppList]:
+    def get_dashboards_list(self) -> Iterable[QlikApp]:
         """
         Get List of all apps
         """
         try:
-            resp_apps = self.client.get("/v1/items?resourceType=app")
-            if resp_apps:
-                app_list = QlikAppList(apps=resp_apps.get("data", []))
-                return app_list.apps
+            link = f"/v1/items?resourceType=app&limit={API_LIMIT}"
+            while True:
+                resp_apps = self.client.get(link)
+                if resp_apps:
+                    resp = QlikAppResponse(**resp_apps)
+                    yield from resp.apps
+                    if resp.links and resp.links.next and resp.links.next.href:
+                        link = resp.links.next.href.replace(
+                            f"{self.config.hostPort}{API_VERSION}", ""
+                        )
+                    else:
+                        break
         except Exception:
             logger.debug(traceback.format_exc())
             logger.warning("Failed to fetch the app list")
-        return []
+
+    def get_dashboards_list_test_conn(self) -> Iterable[QlikApp]:
+        resp_apps = self.client.get("/v1/items?resourceType=app")
+        if resp_apps:
+            resp = QlikAppResponse(**resp_apps)
+            yield from resp.apps
 
     def get_dashboard_details(self, dashboard_id: str) -> Optional[QlikApp]:
         """

metadata/ingestion/source/dashboard/qlikcloud/connection.py

@@ -43,10 +43,7 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    def custom_executor():
-        return client.get_dashboards_list()
-
-    test_fn = {"GetDashboards": custom_executor}
+    test_fn = {"GetDashboards": client.get_dashboards_list_test_conn}
 
     test_connection_steps(
         metadata=metadata,

metadata/ingestion/source/dashboard/qlikcloud/metadata.py

@@ -36,7 +36,7 @@ from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.qlikcloud.client import QlikCloudClient
-from metadata.ingestion.source.dashboard.qlikcloud.models import QlikApp, QlikAppList
+from metadata.ingestion.source.dashboard.qlikcloud.models import QlikApp
 from metadata.ingestion.source.dashboard.qliksense.metadata import QliksenseSource
 from metadata.ingestion.source.dashboard.qliksense.models import QlikTable
 from metadata.utils import fqn
@@ -74,7 +74,7 @@ class QlikcloudSource(QliksenseSource):
         metadata: OpenMetadata,
     ):
         super().__init__(config, metadata)
-        self.collections: List[QlikAppList] = []
+        self.collections: List[QlikApp] = []
         self.data_models: List[QlikTable] = []
 
     def filter_draft_dashboard(self, dashboard: QlikApp) -> bool:

metadata/ingestion/source/dashboard/qlikcloud/models.py

@@ -27,7 +27,16 @@ class QlikApp(BaseModel):
     published: Optional[bool]
 
 
-class QlikAppList(BaseModel):
+class QlikLink(BaseModel):
+    href: Optional[str] = None
+
+
+class QlikLinks(BaseModel):
+    next: Optional[QlikLink] = None
+
+
+class QlikAppResponse(BaseModel):
     """QlikCloud Apps List"""
 
-    apps: Optional[List[QlikApp]]
+    apps: Optional[List[QlikApp]] = Field(None, alias="data")
+    links: Optional[QlikLinks] = None

metadata/ingestion/source/database/azuresql/connection.py

@@ -22,6 +22,7 @@ from metadata.generated.schema.entity.automations.workflow import (
 )
 from metadata.generated.schema.entity.services.connections.database.azureSQLConnection import (
     Authentication,
+    AuthenticationMode,
     AzureSQLConnection,
 )
 from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
@@ -42,7 +43,8 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
     """
 
     if (
-        hasattr(connection.authenticationMode, "authentication")
+        isinstance(connection, AzureSQLConnection)
+        and isinstance(connection.authenticationMode, AuthenticationMode)
         and connection.authenticationMode.authentication is not None
     ):
         connection_string = f"Driver={connection.driver};Server={connection.hostPort};Database={connection.database};"
@@ -82,7 +84,7 @@ def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -
         params = "&".join(
             f"{key}={quote_plus(value)}" for key, value in options.items() if value
         )
-        url = f"{url}?{params}"
+        url = f"{url}&{params}"
 
     return url
 

metadata/ingestion/source/database/column_type_parser.py

@@ -291,6 +291,17 @@ class ColumnTypeParser:
     except ImportError:
         pass
 
+    try:
+        # pylint: disable=import-outside-toplevel
+        from teradatasqlalchemy import BYTE, VARBYTE
+
+        _COLUMN_TYPE_MAPPING[BYTE] = "BINARY"
+        _SOURCE_TYPE_TO_OM_TYPE["BYTE"] = "BINARY"
+        _COLUMN_TYPE_MAPPING[VARBYTE] = "VARBINARY"
+        _SOURCE_TYPE_TO_OM_TYPE["VARBYTE"] = "VARBINARY"
+    except ImportError:
+        pass
+
     @staticmethod
     def get_column_type(column_type: Any) -> str:
         for func in [

metadata/ingestion/source/database/teradata/connection.py

@@ -0,0 +1,98 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Source connection handler
+"""
+import enum
+from typing import Optional
+from urllib.parse import quote_plus
+
+from sqlalchemy.engine import Engine
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.database.teradataConnection import (
+    TeradataConnection,
+)
+from metadata.ingestion.connections.builders import (
+    create_generic_db_connection,
+    get_connection_args_common,
+    get_connection_options_dict,
+)
+from metadata.ingestion.connections.test_connections import test_connection_db_common
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.teradata.queries import TERADATA_GET_DATABASE
+
+
+def get_connection_url(connection: TeradataConnection) -> str:
+    """
+    Create Teradtaa connection url
+    """
+    url = f"{connection.scheme.value}://{connection.hostPort}/"
+    url += f"?user={quote_plus(connection.username)}"
+    if connection.password:
+        url += f"&password={quote_plus(connection.password.get_secret_value())}"
+
+    # add standard options
+    params = "&".join(
+        [
+            f"{key}={quote_plus(str(getattr(connection, key) if not isinstance(getattr(connection, key), enum.Enum) else getattr(connection, key).value))}"
+            for key in ["account", "logdata", "logmech", "tmode"]
+            if getattr(connection, key, None)
+        ]
+    )
+    url = f"{url}&{params}"
+
+    # add additional options if specified
+    options = get_connection_options_dict(connection)
+    if options:
+        params = "&".join(
+            f"{key}={quote_plus(str(value if not isinstance(value, enum.Enum) else value.value))}"
+            for (key, value) in options.items()
+            if value
+        )
+        url += f"{url}&{params}"
+
+    return url
+
+
+def get_connection(connection: TeradataConnection) -> Engine:
+    """
+    Create connection
+    """
+    return create_generic_db_connection(
+        connection=connection,
+        get_connection_url_fn=get_connection_url,
+        get_connection_args_fn=get_connection_args_common,
+    )
+
+
+def test_connection(
+    metadata: OpenMetadata,
+    engine: Engine,
+    service_connection: TeradataConnection,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+) -> None:
+    """
+    Test connection. This can be executed either as part
+    of a metadata workflow or during an Automation Workflow
+    """
+    queries = {"GetDatabases": TERADATA_GET_DATABASE}
+
+    test_connection_db_common(
+        metadata=metadata,
+        engine=engine,
+        service_connection=service_connection,
+        automation_workflow=automation_workflow,
+        queries=queries,
+    )

metadata/ingestion/source/database/teradata/metadata.py

@@ -0,0 +1,148 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Teradata source implementation.
+"""
+import traceback
+from typing import Iterable, Optional
+
+from teradatasqlalchemy.dialect import TeradataDialect
+
+from metadata.generated.schema.api.data.createStoredProcedure import (
+    CreateStoredProcedureRequest,
+)
+from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
+from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode
+from metadata.generated.schema.entity.services.connections.database.teradataConnection import (
+    TeradataConnection,
+)
+from metadata.generated.schema.entity.services.ingestionPipelines.status import (
+    StackTraceError,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.generated.schema.type.basic import EntityName
+from metadata.ingestion.api.models import Either
+from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
+from metadata.ingestion.source.database.teradata.models import (
+    STORED_PROC_LANGUAGE_MAP,
+    TeradataStoredProcedure,
+)
+from metadata.ingestion.source.database.teradata.queries import (
+    TERADATA_GET_STORED_PROCEDURES,
+    TERADATA_SHOW_STORED_PROCEDURE,
+)
+from metadata.ingestion.source.database.teradata.utils import get_table_comment
+from metadata.utils import fqn
+from metadata.utils.logger import ingestion_logger
+from metadata.utils.sqlalchemy_utils import get_all_table_comments
+
+logger = ingestion_logger()
+
+TeradataDialect.get_table_comment = get_table_comment
+TeradataDialect.get_all_table_comments = get_all_table_comments
+
+
+class TeradataSource(CommonDbSourceService):
+    """
+    Implements the necessary methods to extract
+    Database metadata from Teradata Source
+    """
+
+    @classmethod
+    def create(
+        cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
+    ):
+        config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
+        connection = config.serviceConnection.__root__.config
+        if not isinstance(connection, TeradataConnection):
+            raise InvalidSourceException(
+                f"Expected TeradataConnection, but got {connection}"
+            )
+        return cls(config, metadata)
+
+    def get_stored_procedures(self) -> Iterable[TeradataStoredProcedure]:
+        """List Teradata stored procedures"""
+        if self.source_config.includeStoredProcedures:
+            results = self.engine.execute(
+                TERADATA_GET_STORED_PROCEDURES.format(
+                    schema_name=self.context.get().database_schema,
+                )
+            ).all()
+            for row in results:
+                try:
+                    stored_procedure = TeradataStoredProcedure.parse_obj(dict(row))
+                    stored_procedure.definition = self.describe_procedure_definition(
+                        stored_procedure
+                    )
+                    yield stored_procedure
+                except Exception as exc:
+                    logger.error()
+                    self.status.failed(
+                        error=StackTraceError(
+                            name=dict(row).get("name", "UNKNOWN"),
+                            error=f"Error parsing Stored Procedure payload: {exc}",
+                            stackTrace=traceback.format_exc(),
+                        )
+                    )
+
+    def describe_procedure_definition(
+        self, stored_procedure: TeradataStoredProcedure
+    ) -> str:
+        """
+        We can only get the SP definition via SHOW PROCEDURE
+        """
+        res = self.engine.execute(
+            TERADATA_SHOW_STORED_PROCEDURE.format(
+                schema_name=stored_procedure.database_schema,
+                procedure_name=stored_procedure.procedure_name,
+            )
+        )
+        return str(res.first()[0])
+
+    def yield_stored_procedure(
+        self, stored_procedure: TeradataStoredProcedure
+    ) -> Iterable[Either[CreateStoredProcedureRequest]]:
+        """Prepare the stored procedure payload"""
+
+        try:
+            stored_procedure_request = CreateStoredProcedureRequest(
+                name=EntityName(__root__=stored_procedure.procedure_name),
+                description=None,
+                storedProcedureCode=StoredProcedureCode(
+                    language=STORED_PROC_LANGUAGE_MAP.get(
+                        stored_procedure.procedure_type
+                    ),
+                    code=stored_procedure.definition,
+                ),
+                databaseSchema=fqn.build(
+                    metadata=self.metadata,
+                    entity_type=DatabaseSchema,
+                    service_name=self.context.get().database_service,
+                    database_name=self.context.get().database,
+                    schema_name=stored_procedure.database_schema,
+                ),
+            )
+            yield Either(right=stored_procedure_request)
+            self.register_record_stored_proc_request(stored_procedure_request)
+
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=stored_procedure.procedure_name,
+                    error=f"Error yielding Stored Procedure [{stored_procedure.procedure_name}] due to [{exc}]",
+                    stackTrace=traceback.format_exc(),
+                )
+            )

metadata/ingestion/source/database/teradata/models.py

@@ -0,0 +1,22 @@
+"""
+Teradata models
+"""
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from metadata.generated.schema.entity.data.storedProcedure import Language
+
+STORED_PROC_LANGUAGE_MAP = {
+    "SQL": Language.SQL,
+    "EXTERNAL": Language.External,
+}
+
+
+class TeradataStoredProcedure(BaseModel):
+    """Teradata stored procedure list query results"""
+
+    procedure_name: str = Field(...)
+    database_schema: Optional[str] = Field(None)
+    procedure_type: str = Field(Language.SQL)
+    definition: str = Field(None)

metadata/ingestion/source/database/teradata/queries.py

@@ -0,0 +1,77 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+SQL Queries used during ingestion
+"""
+
+TERADATA_GET_TABLE_NAMES = """
+SELECT tablename, databasename from dbc.tablesvx
+WHERE DataBaseName = :schema AND TableKind in ('T','V','O')
+"""
+
+TERADATA_TABLE_COMMENTS = """
+    SELECT DataBaseName as schema,
+            TableName as table_name,
+            CommentString as table_comment
+    FROM dbc.tablesvx
+    WHERE TableKind in ('T','V','O')
+    ORDER BY "schema", "table_name"
+"""
+
+TERADATA_GET_STORED_PROCEDURES = """
+SELECT  T.DatabaseName AS database_schema,
+        T.TableName AS procedure_name,
+        case T.TableKind 
+           when 'P' then 'SQL'
+           when 'E' then 'EXTERNAL'
+        END as procedure_type
+FROM    DBC.TablesVX T
+WHERE T.TableKind in ('P', 'E')
+  and T.DatabaseName = '{schema_name}'
+"""
+
+TERADATA_SHOW_STORED_PROCEDURE = """
+SHOW PROCEDURE {schema_name}.{procedure_name};
+"""
+
+TERADATA_VIEW_DEFINITIONS = """
+select dbase.DatabaseNameI,
+tvm.TVMNameI,
+tvm.CreateText
+from dbc.tvm  tvm join DBC.Dbase dbase
+on  tvm.DatabaseId = dbase.DatabaseId
+where TableKind in ('V')
+   AND tvm.tvmid NOT IN ( '00C001000000'xb, '00C002000000'xb,
+                          '00C009000000'xb, '00C010000000'xb,
+                          '00C017000000'xb, '000000000000'xb)
+AND   (tvm.tvmid IN
+         /* IDs of Tables accessible to the USER or *PUBLIC* */
+         (SELECT TVMId FROM DBC.View_UserTablesExtVX)
+       OR
+       dbase.DatabaseId IN
+         /* IDs of databases accessible to the USER or *PUBLIC* */
+        (SELECT DatabaseID FROM DBC.View_UserDBsExtVX)
+        )
+"""
+
+TERADATA_GET_DATABASE = """
+select databasename from dbc.databasesvx
+"""
+
+
+TERADATA_GET_DB_NAMES = """
+select databasename from dbc.databasesvx
+"""
+
+TERADATA_GET_SERVER_VERSION = """
+SELECT InfoData FROM dbc.dbcinfo
+where InfoKey = 'VERSION'
+"""

metadata/ingestion/source/database/teradata/utils.py

@@ -0,0 +1,34 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Teradata SQLAlchemy Helper Methods
+"""
+
+from sqlalchemy.engine import reflection
+
+from metadata.ingestion.source.database.teradata.queries import TERADATA_TABLE_COMMENTS
+from metadata.utils.logger import ingestion_logger
+from metadata.utils.sqlalchemy_utils import get_table_comment_wrapper
+
+logger = ingestion_logger()
+
+
+@reflection.cache
+def get_table_comment(
+    self, connection, table_name, schema=None, **kw
+):  # pylint: disable=unused-argument
+    return get_table_comment_wrapper(
+        self,
+        connection,
+        table_name=table_name,
+        schema=schema,
+        query=TERADATA_TABLE_COMMENTS,
+    )

metadata/ingestion/source/storage/s3/metadata.py

@@ -175,6 +175,18 @@ class S3Source(StorageServiceSource):
         yield Either(right=container_request)
         self.register_record(container_request=container_request)
 
+    def get_size(self, bucket_name: str, file_path: str) -> Optional[float]:
+        """
+        Method to get the size of the file
+        """
+        try:
+            file_obj = self.s3_client.head_object(Bucket=bucket_name, Key=file_path)
+            return file_obj["ContentLength"]
+        except Exception as exc:
+            logger.debug(f"Failed to get size of file due to {exc}")
+            logger.debug(traceback.format_exc())
+        return None
+
     def _generate_container_details(
         self,
         bucket_response: S3BucketResponse,
@@ -182,6 +194,31 @@ class S3Source(StorageServiceSource):
         parent: Optional[EntityReference] = None,
     ) -> Optional[S3ContainerDetails]:
         bucket_name = bucket_response.name
+        object_size = self.get_size(
+            bucket_name=bucket_name,
+            file_path=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+        )
+        if not metadata_entry.structureFormat and object_size:
+            prefix = f"{KEY_SEPARATOR}{metadata_entry.dataPath.strip(KEY_SEPARATOR)}"
+            return S3ContainerDetails(
+                name=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                prefix=prefix,
+                creation_date=bucket_response.creation_date.isoformat()
+                if bucket_response.creation_date
+                else None,
+                file_formats=[],
+                data_model=None,
+                parent=parent,
+                size=self.get_size(
+                    bucket_name=bucket_name,
+                    file_path=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                ),
+                fullPath=self._get_full_path(bucket_name, prefix),
+                sourceUrl=self._get_object_source_url(
+                    bucket_name=bucket_name,
+                    prefix=metadata_entry.dataPath.strip(KEY_SEPARATOR),
+                ),
+            )
         sample_key = self._get_sample_file_path(
             bucket_name=bucket_name, metadata_entry=metadata_entry
         )