openmetadata-ingestion 1.5.7.1__py3-none-any.whl → 1.5.7.1.dev0__py3-none-any.whl

metadata/generated/schema/type/jdbcConnection.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/jdbcConnection.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/lifeCycle.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/lifeCycle.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/paging.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/paging.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/profile.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/profile.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/queryParserData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/queryParserData.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/reaction.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/reaction.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schedule.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schedule.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/schema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/schema.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableQuery.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableQuery.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tableUsageCount.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tableUsageCount.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/tagLabel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/tagLabel.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageDetails.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageDetails.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/usageRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/usageRequest.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/type/votes.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  type/votes.json
-#   timestamp: 2024-10-24T15:21:08+00:00
+#   timestamp: 2024-10-25T13:20:28+00:00
 
 from __future__ import annotations
 

metadata/ingestion/source/database/couchbase/metadata.py

@@ -15,6 +15,9 @@ import re
 import traceback
 from typing import Dict, Iterable, List, Optional
 
+import requests
+from requests.auth import HTTPBasicAuth
+
 from metadata.generated.schema.entity.services.connections.database.couchbaseConnection import (
     CouchbaseConnection,
 )
@@ -23,13 +26,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 )
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.ingestion.source.database.common_nosql_source import (
-    SAMPLE_SIZE,
-    CommonNoSQLSource,
-)
+from metadata.ingestion.source.database.common_nosql_source import CommonNoSQLSource
 from metadata.ingestion.source.database.couchbase.models import IndexObject as Index
 from metadata.ingestion.source.database.couchbase.queries import (
-    COUCHBASE_GET_DATA,
     COUCHBASE_GET_INDEX_KEYS,
 )
 from metadata.utils.logger import ingestion_logger
@@ -45,6 +44,8 @@ class CouchbaseSource(CommonNoSQLSource):
     Database metadata from Dynamo Source
     """
 
+    service_connection: CouchbaseConnection
+
     def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
         super().__init__(config, metadata)
         self.couchbase = self.connection_obj
@@ -151,6 +152,59 @@ class CouchbaseSource(CommonNoSQLSource):
         self.index_condition_map[(bucket_name, schema_name)] = ""
         return ""
 
+    def _fetch_document_ids(self, schema_name: str, table_name: str) -> List[str]:
+        # Parameters for pagination
+        page_size = 100  # Number of documents to fetch per page
+        document_ids = []
+
+        try:
+            page = 0
+            while len(document_ids) < 1000:
+                # REST API endpoint with pagination
+                couchbase_port = 8091
+                database_name = self.context.get().database
+                connection_scheme = "http"
+                url = (
+                    f"{connection_scheme}://{self.service_connection.hostport}:{couchbase_port}"
+                    f"/pools/default/buckets/{database_name}/scopes/{schema_name}/collections/{table_name}/docs"
+                )
+                params = {"skip": page * page_size, "limit": page_size}
+
+                # Fetch the next page
+                response = requests.get(
+                    url,
+                    auth=HTTPBasicAuth(
+                        self.service_connection.username,
+                        self.service_connection.password.get_secret_value(),
+                    ),
+                    params=params,
+                )
+                response.raise_for_status()
+                data = response.json()
+
+                # Add document IDs from the current page
+                ids = [doc["id"] for doc in data["rows"]]
+                document_ids.extend(ids)
+
+                # If fewer documents than page_size were returned, we’re at the end
+                if len(ids) < page_size:
+                    break
+
+                # Increment the page number
+                page += 1
+
+            # Limit the list to the first 1,000 IDs
+            document_ids = document_ids[:1000]
+
+            return document_ids
+
+        except Exception as exc:
+            logger.debug(f"Error fetching document IDs: [{table_name}]: {exc}")
+            logger.debug(traceback.format_exc())
+
+        logger.info("Document IDs not found")
+        return []
+
     def get_table_columns_dict(self, schema_name: str, table_name: str) -> List[Dict]:
         """
         Method to get actual data available within table
@@ -159,17 +213,14 @@ class CouchbaseSource(CommonNoSQLSource):
         from couchbase.exceptions import QueryIndexNotFoundException
 
         try:
-            condition = self.get_index_condition(schema_name)
             database_name = self.context.get().database
-            query_coln = COUCHBASE_GET_DATA.format(
-                database_name=database_name,
-                schema_name=schema_name,
-                table_name=table_name,
-                sample_size=SAMPLE_SIZE,
-                condition=condition,
+            bucket = self.couchbase.bucket(database_name)
+            scope = bucket.scope(schema_name)
+            collection = scope.collection(table_name)
+            documents = collection.get_multi(
+                self._fetch_document_ids(schema_name, table_name)
             )
-            query_iter = self.couchbase.query(query_coln)
-            return list(query_iter.rows())
+            return list(map(lambda x: x.value, documents.results.values()))
         except QueryIndexNotFoundException as exp:
             logger.warning(
                 f"Fetching columns failed for [`{database_name}`.`{schema_name}`.`{table_name}`],"