graphxr-database-proxy 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- graphxr_database_proxy/__init__.py +16 -0
- graphxr_database_proxy/api/__init__.py +1 -0
- graphxr_database_proxy/api/database.py +266 -0
- graphxr_database_proxy/api/google.py +437 -0
- graphxr_database_proxy/api/projects.py +99 -0
- graphxr_database_proxy/common/util.py +38 -0
- graphxr_database_proxy/drivers/__init__.py +1 -0
- graphxr_database_proxy/drivers/base.py +56 -0
- graphxr_database_proxy/drivers/factory.py +36 -0
- graphxr_database_proxy/drivers/spanner.py +814 -0
- graphxr_database_proxy/main.py +110 -0
- graphxr_database_proxy/models/__init__.py +1 -0
- graphxr_database_proxy/models/google.py +50 -0
- graphxr_database_proxy/models/project.py +170 -0
- graphxr_database_proxy/proxy.py +495 -0
- graphxr_database_proxy/proxyForDev.py +290 -0
- graphxr_database_proxy/services/__init__.py +1 -0
- graphxr_database_proxy/services/project_service.py +150 -0
- graphxr_database_proxy/static/favicon.ico +0 -0
- graphxr_database_proxy/static/index.html +1 -0
- graphxr_database_proxy/static/main.7391ee9773c403483393.css +175 -0
- graphxr_database_proxy/static/main.7391ee9773c403483393.css.map +1 -0
- graphxr_database_proxy/static/main.ce3fbb85a7bc9452edb9.js +2 -0
- graphxr_database_proxy/static/main.ce3fbb85a7bc9452edb9.js.map +1 -0
- graphxr_database_proxy/static/vendors.70542a99f336a8021013.js +3 -0
- graphxr_database_proxy/static/vendors.70542a99f336a8021013.js.LICENSE.txt +95 -0
- graphxr_database_proxy/static/vendors.70542a99f336a8021013.js.map +1 -0
- graphxr_database_proxy-1.0.0.dist-info/METADATA +180 -0
- graphxr_database_proxy-1.0.0.dist-info/RECORD +33 -0
- graphxr_database_proxy-1.0.0.dist-info/WHEEL +5 -0
- graphxr_database_proxy-1.0.0.dist-info/entry_points.txt +2 -0
- graphxr_database_proxy-1.0.0.dist-info/licenses/LICENSE +21 -0
- graphxr_database_proxy-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,814 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Google Cloud Spanner driver
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from hashlib import new
|
|
7
|
+
import time
|
|
8
|
+
from typing import Any, Dict, List, Optional
|
|
9
|
+
|
|
10
|
+
from google.cloud import spanner
|
|
11
|
+
from google.cloud.spanner_v1 import Client, data_types
|
|
12
|
+
from google.oauth2 import service_account
|
|
13
|
+
from google.auth.transport.requests import Request
|
|
14
|
+
from google_auth_oauthlib.flow import Flow
|
|
15
|
+
from google.cloud import spanner
|
|
16
|
+
import google.oauth2.credentials
|
|
17
|
+
import re
|
|
18
|
+
|
|
19
|
+
from .base import BaseDatabaseDriver
|
|
20
|
+
from ..models.project import Project, GraphSchema, QueryData, QueryResponse, SchemaResponse, GraphSchemaResponse, SampleDataResponse, AuthType, Category, Relationship
|
|
21
|
+
from ..common.util import get_default_oauth_config
|
|
22
|
+
from ..services.project_service import ProjectService
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class SpannerDriver(BaseDatabaseDriver):
|
|
28
|
+
"""Google Cloud Spanner driver"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, project: Project):
|
|
31
|
+
super().__init__(project)
|
|
32
|
+
self.client: Optional[Client] = None
|
|
33
|
+
self.instance = None
|
|
34
|
+
self.database = None
|
|
35
|
+
|
|
36
|
+
async def connect(self) -> None:
|
|
37
|
+
"""Establish connection to Spanner"""
|
|
38
|
+
try:
|
|
39
|
+
print(f"[INFO] Connecting to Spanner with auth type: {self.config.auth_type}")
|
|
40
|
+
|
|
41
|
+
# Initialize client based on auth type
|
|
42
|
+
if self.config.auth_type == AuthType.OAUTH2:
|
|
43
|
+
print("[INFO] Using OAuth2 authentication")
|
|
44
|
+
self.client = await self._get_oauth_client()
|
|
45
|
+
elif self.config.auth_type == AuthType.SERVICE_ACCOUNT:
|
|
46
|
+
print("[INFO] Using Service Account authentication")
|
|
47
|
+
self.client = await self._get_service_account_client()
|
|
48
|
+
else:
|
|
49
|
+
raise ValueError(f"Unsupported auth type: {self.config.auth_type}")
|
|
50
|
+
|
|
51
|
+
print(f"[INFO] Project: {self.config.project_id}")
|
|
52
|
+
print(f"[INFO] Instance: {self.config.instance_id}")
|
|
53
|
+
print(f"[INFO] Database: {self.config.database_id}")
|
|
54
|
+
|
|
55
|
+
# Get instance and database
|
|
56
|
+
self.instance = self.client.instance(self.config.instance_id)
|
|
57
|
+
self.database = self.instance.database(self.config.database_id)
|
|
58
|
+
|
|
59
|
+
print("[OK] Spanner connection established")
|
|
60
|
+
|
|
61
|
+
except Exception as e:
|
|
62
|
+
print(f"[ERROR] Failed to connect to Spanner: {str(e)}")
|
|
63
|
+
raise ConnectionError(f"Failed to connect to Spanner: {str(e)}")
|
|
64
|
+
|
|
65
|
+
async def _refresh_oauth_token(self, credentials: google.oauth2.credentials.Credentials) -> google.oauth2.credentials.Credentials:
|
|
66
|
+
"""Refresh OAuth token"""
|
|
67
|
+
try:
|
|
68
|
+
print("[INFO] Refreshing OAuth token...")
|
|
69
|
+
request = Request()
|
|
70
|
+
credentials.refresh(request)
|
|
71
|
+
|
|
72
|
+
# Update the project with new token information
|
|
73
|
+
current_time = time.time()
|
|
74
|
+
project_service = ProjectService()
|
|
75
|
+
await project_service.update_project_token(
|
|
76
|
+
project_id=self.project.id,
|
|
77
|
+
token=credentials.token,
|
|
78
|
+
last_refreshed=current_time,
|
|
79
|
+
expires_in=getattr(credentials, 'expires_in', 3600)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# Update local config
|
|
83
|
+
self.project.database_config.oauth_config.token = credentials.token
|
|
84
|
+
self.project.database_config.oauth_config.last_refreshed = current_time
|
|
85
|
+
self.config.oauth_config = self.project.database_config.oauth_config
|
|
86
|
+
|
|
87
|
+
print("[OK] OAuth token refreshed successfully")
|
|
88
|
+
return credentials
|
|
89
|
+
|
|
90
|
+
except Exception as e:
|
|
91
|
+
print(f"[ERROR] Failed to refresh OAuth token: {e}")
|
|
92
|
+
raise e
|
|
93
|
+
|
|
94
|
+
def get_token_status(self) -> Dict[str, Any]:
|
|
95
|
+
"""Get current token status information"""
|
|
96
|
+
if not self.config.oauth_config:
|
|
97
|
+
return {"status": "no_oauth_config"}
|
|
98
|
+
|
|
99
|
+
oauth_info = self.config.oauth_config
|
|
100
|
+
current_time = time.time()
|
|
101
|
+
|
|
102
|
+
status = {
|
|
103
|
+
"has_token": bool(oauth_info.token),
|
|
104
|
+
"has_refresh_token": bool(oauth_info.refresh_token),
|
|
105
|
+
"expires_in": oauth_info.expires_in,
|
|
106
|
+
"last_refreshed": oauth_info.last_refreshed,
|
|
107
|
+
"current_time": current_time
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if oauth_info.last_refreshed and oauth_info.expires_in:
|
|
111
|
+
time_since_refresh = current_time - oauth_info.last_refreshed
|
|
112
|
+
time_until_expiry = oauth_info.expires_in - time_since_refresh
|
|
113
|
+
status.update({
|
|
114
|
+
"time_since_refresh": time_since_refresh,
|
|
115
|
+
"time_until_expiry": time_until_expiry,
|
|
116
|
+
"is_expired": time_until_expiry <= 0,
|
|
117
|
+
"expires_soon": time_until_expiry <= 300 # 5 minutes
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
return status
|
|
121
|
+
|
|
122
|
+
async def _get_oauth_client(self) -> Client:
|
|
123
|
+
"""Get Spanner client using OAuth2"""
|
|
124
|
+
if not self.config.oauth_config:
|
|
125
|
+
raise ValueError("OAuth config is required for OAuth2 authentication")
|
|
126
|
+
|
|
127
|
+
if not self.config.oauth_config.client_id or not self.config.oauth_config.client_secret:
|
|
128
|
+
default_oauth = get_default_oauth_config()
|
|
129
|
+
self.config.oauth_config.client_id = default_oauth.get("client_id")
|
|
130
|
+
self.config.oauth_config.client_secret = default_oauth.get("client_secret")
|
|
131
|
+
|
|
132
|
+
oauth_info = self.config.oauth_config
|
|
133
|
+
|
|
134
|
+
# print(f"🔑 OAuth Client ID: {oauth_info.client_id[:10]}..." if oauth_info.client_id else "None")
|
|
135
|
+
# print(f"🎫 Token available: {'Yes' if oauth_info.token else 'No'}")
|
|
136
|
+
# print(f"🔄 Refresh token available: {'Yes' if oauth_info.refresh_token else 'No'}")
|
|
137
|
+
|
|
138
|
+
# Create credentials with proper scopes for Spanner
|
|
139
|
+
credentials = google.oauth2.credentials.Credentials(
|
|
140
|
+
token=oauth_info.token,
|
|
141
|
+
refresh_token=oauth_info.refresh_token or None,
|
|
142
|
+
token_uri="https://oauth2.googleapis.com/token",
|
|
143
|
+
client_id=oauth_info.client_id,
|
|
144
|
+
client_secret=oauth_info.client_secret,
|
|
145
|
+
scopes=[
|
|
146
|
+
"https://www.googleapis.com/auth/cloudplatformprojects.readonly",
|
|
147
|
+
"https://www.googleapis.com/auth/spanner.admin",
|
|
148
|
+
"https://www.googleapis.com/auth/spanner.data",
|
|
149
|
+
"https://www.googleapis.com/auth/userinfo.profile",
|
|
150
|
+
"https://www.googleapis.com/auth/userinfo.email",
|
|
151
|
+
]
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
# Initialize last_refreshed if not set
|
|
155
|
+
if oauth_info.last_refreshed is None:
|
|
156
|
+
oauth_info.last_refreshed = time.time()
|
|
157
|
+
# Save the initialized timestamp
|
|
158
|
+
project_service = ProjectService()
|
|
159
|
+
await project_service.update_project_token(
|
|
160
|
+
project_id=self.project.id,
|
|
161
|
+
token=oauth_info.token,
|
|
162
|
+
last_refreshed=oauth_info.last_refreshed,
|
|
163
|
+
expires_in=oauth_info.expires_in
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
# Check if token needs refresh (expires in 5 minutes or less)
|
|
167
|
+
needs_refresh = False
|
|
168
|
+
if oauth_info.expires_in and oauth_info.last_refreshed:
|
|
169
|
+
time_since_refresh = time.time() - oauth_info.last_refreshed
|
|
170
|
+
expires_soon = time_since_refresh >= (oauth_info.expires_in - 300) # 5 minutes buffer
|
|
171
|
+
needs_refresh = expires_soon
|
|
172
|
+
|
|
173
|
+
if needs_refresh:
|
|
174
|
+
print(f"[INFO] Token expires soon (refreshed {time_since_refresh:.0f}s ago, expires in {oauth_info.expires_in}s)")
|
|
175
|
+
|
|
176
|
+
if needs_refresh and oauth_info.refresh_token:
|
|
177
|
+
print("[INFO] Refreshing OAuth token...")
|
|
178
|
+
credentials = await self._refresh_oauth_token(credentials)
|
|
179
|
+
elif needs_refresh and not oauth_info.refresh_token:
|
|
180
|
+
print("[WARN] Token expires soon but no refresh token available")
|
|
181
|
+
|
|
182
|
+
return spanner.Client(project=self.config.project_id, credentials=credentials)
|
|
183
|
+
|
|
184
|
+
async def _get_service_account_client(self) -> Client:
|
|
185
|
+
"""Get Spanner client using service account"""
|
|
186
|
+
if not self.config.oauth_config:
|
|
187
|
+
raise ValueError("OAuth config with service account data is required for service account authentication")
|
|
188
|
+
|
|
189
|
+
# Check if we have service account fields in oauth_config
|
|
190
|
+
sa_config = self.config.oauth_config
|
|
191
|
+
if sa_config.type == "service_account" and sa_config.private_key and sa_config.client_email:
|
|
192
|
+
# Create credentials from service account info in oauth_config
|
|
193
|
+
service_account_info = {
|
|
194
|
+
"type": sa_config.type,
|
|
195
|
+
"project_id": sa_config.project_id or self.config.project_id,
|
|
196
|
+
"private_key_id": sa_config.private_key_id,
|
|
197
|
+
"private_key": sa_config.private_key,
|
|
198
|
+
"client_email": sa_config.client_email,
|
|
199
|
+
"client_id": sa_config.client_id,
|
|
200
|
+
"auth_uri": sa_config.auth_uri or "https://accounts.google.com/o/oauth2/auth",
|
|
201
|
+
"token_uri": sa_config.token_uri or "https://oauth2.googleapis.com/token",
|
|
202
|
+
"auth_provider_x509_cert_url": sa_config.auth_provider_x509_cert_url or "https://www.googleapis.com/oauth2/v1/certs",
|
|
203
|
+
"client_x509_cert_url": sa_config.client_x509_cert_url
|
|
204
|
+
}
|
|
205
|
+
credentials = service_account.Credentials.from_service_account_info(
|
|
206
|
+
service_account_info,
|
|
207
|
+
scopes=[
|
|
208
|
+
"https://www.googleapis.com/auth/cloudplatformprojects.readonly",
|
|
209
|
+
"https://www.googleapis.com/auth/spanner.admin",
|
|
210
|
+
"https://www.googleapis.com/auth/spanner.data"
|
|
211
|
+
]
|
|
212
|
+
)
|
|
213
|
+
else:
|
|
214
|
+
raise ValueError("Service account information is incomplete in oauth_config")
|
|
215
|
+
|
|
216
|
+
return spanner.Client(project=self.config.project_id, credentials=credentials)
|
|
217
|
+
|
|
218
|
+
async def disconnect(self) -> None:
|
|
219
|
+
"""Close connection to Spanner"""
|
|
220
|
+
# Spanner client doesn't need explicit disconnection
|
|
221
|
+
self.client = None
|
|
222
|
+
self.instance = None
|
|
223
|
+
self.database = None
|
|
224
|
+
|
|
225
|
+
async def test_connection(self) -> bool:
|
|
226
|
+
"""Test Spanner connection"""
|
|
227
|
+
try:
|
|
228
|
+
if not self.database:
|
|
229
|
+
await self.connect()
|
|
230
|
+
|
|
231
|
+
# Test database existence first
|
|
232
|
+
if not self.database.exists():
|
|
233
|
+
print(f"Database {self.config.database_id} does not exist")
|
|
234
|
+
return False
|
|
235
|
+
|
|
236
|
+
# Simple test query using snapshot for read-only operation
|
|
237
|
+
with self.database.snapshot() as snapshot:
|
|
238
|
+
results = snapshot.execute_sql("SELECT 1 as test_value")
|
|
239
|
+
list(results) # Consume results
|
|
240
|
+
return True
|
|
241
|
+
except Exception as e:
|
|
242
|
+
print(f"Connection test failed: {e}")
|
|
243
|
+
return False
|
|
244
|
+
|
|
245
|
+
async def execute_query(self, query: str, parameters: Dict[str, Any] = None) -> QueryResponse:
|
|
246
|
+
"""Execute a Spanner query"""
|
|
247
|
+
start_time = time.time()
|
|
248
|
+
|
|
249
|
+
try:
|
|
250
|
+
if not self.database:
|
|
251
|
+
await self.connect()
|
|
252
|
+
|
|
253
|
+
query = query.strip().rstrip(';') # Clean up query
|
|
254
|
+
# Check if it's a graph query or SQL query
|
|
255
|
+
# Skip MATCH (n) RETURN n.propName LIMIT 25
|
|
256
|
+
# Skip MATCH (n) RETURN count(*) LIMIT 23
|
|
257
|
+
# Skip MATCH (n) RETURN SAFE_TO_JSON(*) LIMIT 23
|
|
258
|
+
test_query = re.sub(r'SAFE_TO_JSON|TO_JSON', '', query, flags=re.IGNORECASE).strip()
|
|
259
|
+
isGraphQuery = (re.search(r'MATCH', test_query, re.IGNORECASE) and
|
|
260
|
+
not re.search(r'RETURN\s*[a-z0-9]+\.', test_query, re.IGNORECASE) and
|
|
261
|
+
not re.search(r'RETURN\s*[a-z0-9]+\(.+\)', test_query, re.IGNORECASE))
|
|
262
|
+
if isGraphQuery and self.config.graph_name:
|
|
263
|
+
# MATCH (n)-[r:HasOyster]-(m) RETURN * LIMIT 25, split to three parts: MATCH, RETURN, LIMIT
|
|
264
|
+
|
|
265
|
+
# Replace keywords with flags and split
|
|
266
|
+
command_with_flags = query
|
|
267
|
+
command_with_flags = re.sub(r'MATCH(\s+[a-z]+=)?', '_flag_', command_with_flags, flags=re.IGNORECASE)
|
|
268
|
+
command_with_flags = re.sub(r'RETURN', '_flag_', command_with_flags, flags=re.IGNORECASE)
|
|
269
|
+
command_with_flags = re.sub(r'LIMIT', '_flag_', command_with_flags, flags=re.IGNORECASE)
|
|
270
|
+
command_parts = [part.strip() for part in command_with_flags.split('_flag_')]
|
|
271
|
+
graph_namespace = command_parts[0] if command_parts[0] else f"GRAPH {self.config.graph_name}"
|
|
272
|
+
limit = command_parts[3] if len(command_parts) > 3 and command_parts[3] else ""
|
|
273
|
+
match_command = command_parts[1] if len(command_parts) > 1 else ""
|
|
274
|
+
|
|
275
|
+
query = f"""
|
|
276
|
+
{graph_namespace}
|
|
277
|
+
MATCH __p={match_command}
|
|
278
|
+
RETURN SAFE_TO_JSON(__p) as thePath {f"LIMIT {limit}" if limit else "" }"""
|
|
279
|
+
|
|
280
|
+
print(f"[DEBUG] Transformed graph query:\n{query}")
|
|
281
|
+
query = query.strip().rstrip(';') # Clean up query again
|
|
282
|
+
|
|
283
|
+
# Auto append the graph namespace
|
|
284
|
+
if self.config.graph_name and not re.search(r'^GRAPH', query, re.IGNORECASE) and re.search(r'^MATCH', query, re.IGNORECASE):
|
|
285
|
+
query = f"""
|
|
286
|
+
GRAPH {self.config.graph_name}
|
|
287
|
+
{query}
|
|
288
|
+
"""
|
|
289
|
+
|
|
290
|
+
# Property Graph query
|
|
291
|
+
results = self._execute_graph_query(query, parameters)
|
|
292
|
+
else:
|
|
293
|
+
# SQL query
|
|
294
|
+
results = self._execute_sql_query(query, parameters)
|
|
295
|
+
|
|
296
|
+
execution_time = time.time() - start_time
|
|
297
|
+
|
|
298
|
+
return QueryResponse(
|
|
299
|
+
success=True,
|
|
300
|
+
data=results,
|
|
301
|
+
execution_time=execution_time
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
except Exception as e:
|
|
305
|
+
execution_time = time.time() - start_time
|
|
306
|
+
return QueryResponse(
|
|
307
|
+
success=False,
|
|
308
|
+
error=str(e),
|
|
309
|
+
execution_time=execution_time
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
def _execute_graph_query(self, query: str, parameters: Dict[str, Any] = None) -> QueryData:
|
|
313
|
+
"""Execute a Property Graph query"""
|
|
314
|
+
with self.database.snapshot() as snapshot:
|
|
315
|
+
results = snapshot.execute_sql(query, params=parameters or {})
|
|
316
|
+
|
|
317
|
+
# Parse graph data similar to the Node.js implementation
|
|
318
|
+
graph = {
|
|
319
|
+
"nodes": {},
|
|
320
|
+
"relationships": {}
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
if not results:
|
|
324
|
+
return QueryData(
|
|
325
|
+
type="GRAPH",
|
|
326
|
+
data={
|
|
327
|
+
"nodes": [],
|
|
328
|
+
"relationships": []
|
|
329
|
+
}
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
for row in results:
|
|
333
|
+
# Get the first value from the row (thePath data)
|
|
334
|
+
the_path_data = row[0] if row and row[0] else row
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
if not the_path_data:
|
|
338
|
+
continue
|
|
339
|
+
|
|
340
|
+
if isinstance(the_path_data, list):
|
|
341
|
+
the_path_data = the_path_data[0]
|
|
342
|
+
|
|
343
|
+
if isinstance(the_path_data, data_types.JsonObject) and hasattr(the_path_data, '_array_value'):
|
|
344
|
+
the_path_data = the_path_data._array_value
|
|
345
|
+
|
|
346
|
+
if isinstance(the_path_data, str):
|
|
347
|
+
try:
|
|
348
|
+
the_path_data = json.loads(the_path_data)
|
|
349
|
+
except json.JSONDecodeError as err:
|
|
350
|
+
print(f"Invalid parseGraph thePathData: {err}")
|
|
351
|
+
the_path_data = []
|
|
352
|
+
|
|
353
|
+
if not isinstance(the_path_data, list):
|
|
354
|
+
the_path_data=[the_path_data]
|
|
355
|
+
|
|
356
|
+
for node_or_edge in the_path_data:
|
|
357
|
+
# Support schema-less graph where properties is directly in the nodeOrEdge object
|
|
358
|
+
# Use the last label as the main label for compatibility with schema-less graph
|
|
359
|
+
is_dynamic_label = node_or_edge.get("properties", {}).get("label")
|
|
360
|
+
labels = node_or_edge.get("labels", [])
|
|
361
|
+
last_label = is_dynamic_label or (labels[-1] if labels else "")
|
|
362
|
+
|
|
363
|
+
properties = (node_or_edge.get("properties", {}).get("properties") or
|
|
364
|
+
node_or_edge.get("properties", {}))
|
|
365
|
+
|
|
366
|
+
if node_or_edge.get("kind") == "node":
|
|
367
|
+
graph["nodes"][node_or_edge.get("identifier")] = {
|
|
368
|
+
"id": node_or_edge.get("identifier"),
|
|
369
|
+
"labels": [last_label] if last_label else [],
|
|
370
|
+
"properties": properties
|
|
371
|
+
}
|
|
372
|
+
elif node_or_edge.get("kind") == "edge":
|
|
373
|
+
graph["relationships"][node_or_edge.get("identifier")] = {
|
|
374
|
+
"id": node_or_edge.get("identifier"),
|
|
375
|
+
"type": last_label,
|
|
376
|
+
"startNodeId": node_or_edge.get("source_node_identifier"),
|
|
377
|
+
"endNodeId": node_or_edge.get("destination_node_identifier"),
|
|
378
|
+
"properties": properties
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
return QueryData(
|
|
382
|
+
type="GRAPH",
|
|
383
|
+
data={
|
|
384
|
+
"nodes": list(graph["nodes"].values()),
|
|
385
|
+
"relationships": list(graph["relationships"].values())
|
|
386
|
+
}
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
def _execute_sql_query(self, query: str, parameters: Dict[str, Any] = None) -> QueryData:
|
|
390
|
+
"""Execute a SQL query"""
|
|
391
|
+
with self.database.snapshot() as snapshot:
|
|
392
|
+
results = snapshot.execute_sql(query, params=parameters or {})
|
|
393
|
+
rows = []
|
|
394
|
+
for row in results:
|
|
395
|
+
row_dict = {}
|
|
396
|
+
for i, field in enumerate(results.fields):
|
|
397
|
+
val = row[i]
|
|
398
|
+
# Convert Array to string
|
|
399
|
+
if isinstance(val, list):
|
|
400
|
+
val = ", ".join(str(item) for item in val)
|
|
401
|
+
row_dict[field.name] = val
|
|
402
|
+
rows.append(row_dict)
|
|
403
|
+
return QueryData(
|
|
404
|
+
type="TABLE",
|
|
405
|
+
data=rows
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
def _is_schema_less(self, meta_json) -> bool:
|
|
409
|
+
"""Check if the graph schema is schema-less (has dynamic properties)"""
|
|
410
|
+
if not meta_json or (not meta_json.get("nodeTables") and not meta_json.get("edgeTables")):
|
|
411
|
+
return False
|
|
412
|
+
|
|
413
|
+
node_tables = meta_json.get("nodeTables", [])
|
|
414
|
+
edge_tables = meta_json.get("edgeTables", [])
|
|
415
|
+
|
|
416
|
+
has_dynamic_node_table = any(
|
|
417
|
+
table.get("dynamicLabelExpr") or table.get("dynamicPropertyExpr")
|
|
418
|
+
for table in node_tables
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
has_dynamic_edge_table = any(
|
|
422
|
+
table.get("dynamicLabelExpr") or table.get("dynamicPropertyExpr")
|
|
423
|
+
for table in edge_tables
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
return has_dynamic_node_table or has_dynamic_edge_table
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _getSchemaForSchemaLessGraphs(self) -> GraphSchema:
|
|
430
|
+
"""Get schema for schema-less graphs"""
|
|
431
|
+
try:
|
|
432
|
+
# First query to get labels for nodes and relationships
|
|
433
|
+
meta_query = f"""
|
|
434
|
+
GRAPH {self.config.graph_name}
|
|
435
|
+
MATCH (n)
|
|
436
|
+
RETURN DISTINCT ARRAY_TO_STRING(LABELS(n),",") AS name,"" as startCategory, "" as endCategory , "category" as type
|
|
437
|
+
LIMIT 1000
|
|
438
|
+
UNION ALL
|
|
439
|
+
MATCH (n)-[r]->(m)
|
|
440
|
+
RETURN DISTINCT ARRAY_TO_STRING(LABELS(r),",") AS name , ARRAY_TO_STRING(LABELS(n),",") as startCategory, ARRAY_TO_STRING(LABELS(m),",") as endCategory ,"relationship" as type
|
|
441
|
+
LIMIT 1000
|
|
442
|
+
"""
|
|
443
|
+
|
|
444
|
+
rows = self._execute_sql_query(meta_query)
|
|
445
|
+
|
|
446
|
+
meta = {
|
|
447
|
+
"categories": {},
|
|
448
|
+
"relationships": {}
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
# Process labels
|
|
452
|
+
for row in rows.data:
|
|
453
|
+
names = row["name"].split(",")
|
|
454
|
+
row_type = row["type"]
|
|
455
|
+
names = [name.strip() for name in names if name.strip()]
|
|
456
|
+
|
|
457
|
+
for name in names:
|
|
458
|
+
if row_type == "category" and name not in meta["categories"]:
|
|
459
|
+
meta["categories"][name] = {
|
|
460
|
+
"name": name,
|
|
461
|
+
"props": [],
|
|
462
|
+
"keys": [],
|
|
463
|
+
"propsTypes": {}
|
|
464
|
+
}
|
|
465
|
+
elif row_type == "relationship" and name not in meta["relationships"]:
|
|
466
|
+
start_categories = [cat.strip() for cat in row["startCategory"].split(",") if cat.strip()]
|
|
467
|
+
end_categories = [cat.strip() for cat in row["endCategory"].split(",") if cat.strip()]
|
|
468
|
+
common_categories = [cat for cat in start_categories if cat in end_categories]
|
|
469
|
+
|
|
470
|
+
start_category = next((cat for cat in start_categories if cat not in common_categories),
|
|
471
|
+
start_categories[0] if start_categories else "")
|
|
472
|
+
end_category = next((cat for cat in end_categories if cat not in common_categories),
|
|
473
|
+
end_categories[0] if end_categories else "")
|
|
474
|
+
|
|
475
|
+
meta["relationships"][name] = {
|
|
476
|
+
"name": name,
|
|
477
|
+
"props": [],
|
|
478
|
+
"keys": [],
|
|
479
|
+
"propsTypes": {},
|
|
480
|
+
"startCategory": start_category,
|
|
481
|
+
"endCategory": end_category
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
# Second query to get properties
|
|
485
|
+
categories = list(meta["categories"].values())
|
|
486
|
+
relationships = list(meta["relationships"].values())
|
|
487
|
+
|
|
488
|
+
prop_queries = []
|
|
489
|
+
|
|
490
|
+
# Add category property queries
|
|
491
|
+
for category in categories:
|
|
492
|
+
prop_queries.append(f"""
|
|
493
|
+
MATCH (n:`{category["name"]}`)
|
|
494
|
+
RETURN SAFE_TO_JSON(n) as props, SAFE_TO_JSON(n) as startN, SAFE_TO_JSON(n) as endN
|
|
495
|
+
LIMIT 1
|
|
496
|
+
""")
|
|
497
|
+
|
|
498
|
+
# Add relationship property queries
|
|
499
|
+
for relationship in relationships:
|
|
500
|
+
prop_queries.append(f"""
|
|
501
|
+
MATCH (n:`{relationship["startCategory"]}`)-[r:`{relationship["name"]}`]->(m:`{relationship["endCategory"]}`)
|
|
502
|
+
RETURN SAFE_TO_JSON(r) as props, SAFE_TO_JSON(n) as startN, SAFE_TO_JSON(m) as endN
|
|
503
|
+
LIMIT 1
|
|
504
|
+
""")
|
|
505
|
+
|
|
506
|
+
if prop_queries:
|
|
507
|
+
prop_query = f"""
|
|
508
|
+
GRAPH {self.config.graph_name}
|
|
509
|
+
{" UNION ALL ".join(prop_queries)}
|
|
510
|
+
"""
|
|
511
|
+
|
|
512
|
+
prop_rows = self._execute_sql_query(prop_query)
|
|
513
|
+
|
|
514
|
+
# Process property results
|
|
515
|
+
for row in prop_rows.data:
|
|
516
|
+
props_data = row["props"]
|
|
517
|
+
if isinstance(props_data, str):
|
|
518
|
+
data = json.loads(props_data)
|
|
519
|
+
else:
|
|
520
|
+
# Handle JsonObject from Spanner
|
|
521
|
+
data = props_data
|
|
522
|
+
|
|
523
|
+
if data.get("kind") == "node":
|
|
524
|
+
# Remove element_definition_name from categories if it exists
|
|
525
|
+
if data.get("element_definition_name") in meta["categories"]:
|
|
526
|
+
del meta["categories"][data["element_definition_name"]]
|
|
527
|
+
|
|
528
|
+
# Get the category name (last label that's not element_definition_name)
|
|
529
|
+
labels = data.get("labels", [])
|
|
530
|
+
category = next((label for label in reversed(labels)
|
|
531
|
+
if label != data.get("element_definition_name")), None)
|
|
532
|
+
|
|
533
|
+
if category and category in meta["categories"]:
|
|
534
|
+
properties = data.get("properties", {}).get("properties", {})
|
|
535
|
+
props_type_map = {}
|
|
536
|
+
for key, value in properties.items():
|
|
537
|
+
props_type_map[key] = type(value).__name__.upper()
|
|
538
|
+
|
|
539
|
+
meta["categories"][category]["props"] = list(properties.keys())
|
|
540
|
+
meta["categories"][category]["propsTypes"] = props_type_map
|
|
541
|
+
|
|
542
|
+
elif data.get("kind") == "edge":
|
|
543
|
+
start_data_raw = row["startN"]
|
|
544
|
+
end_data_raw = row["endN"]
|
|
545
|
+
|
|
546
|
+
if isinstance(start_data_raw, str):
|
|
547
|
+
start_data = json.loads(start_data_raw)
|
|
548
|
+
else:
|
|
549
|
+
start_data = start_data_raw
|
|
550
|
+
|
|
551
|
+
if isinstance(end_data_raw, str):
|
|
552
|
+
end_data = json.loads(end_data_raw)
|
|
553
|
+
else:
|
|
554
|
+
end_data = end_data_raw
|
|
555
|
+
|
|
556
|
+
# Remove element_definition_name from relationships if it exists
|
|
557
|
+
if data.get("element_definition_name") in meta["relationships"]:
|
|
558
|
+
del meta["relationships"][data["element_definition_name"]]
|
|
559
|
+
|
|
560
|
+
# Get relationship and category names
|
|
561
|
+
relationship_labels = data.get("labels", [])
|
|
562
|
+
relationship = next((label for label in reversed(relationship_labels)
|
|
563
|
+
if label != data.get("element_definition_name")), None)
|
|
564
|
+
|
|
565
|
+
start_labels = start_data.get("labels", [])
|
|
566
|
+
start_category = next((label for label in reversed(start_labels)
|
|
567
|
+
if label != start_data.get("element_definition_name")), None)
|
|
568
|
+
|
|
569
|
+
end_labels = end_data.get("labels", [])
|
|
570
|
+
end_category = next((label for label in reversed(end_labels)
|
|
571
|
+
if label != end_data.get("element_definition_name")), None)
|
|
572
|
+
|
|
573
|
+
if relationship and relationship in meta["relationships"]:
|
|
574
|
+
properties = data.get("properties", {}).get("properties", {})
|
|
575
|
+
props_type_map = {}
|
|
576
|
+
for key, value in properties.items():
|
|
577
|
+
props_type_map[key] = type(value).__name__.upper()
|
|
578
|
+
|
|
579
|
+
meta["relationships"][relationship]["startCategory"] = start_category or ""
|
|
580
|
+
meta["relationships"][relationship]["endCategory"] = end_category or ""
|
|
581
|
+
meta["relationships"][relationship]["props"] = list(properties.keys())
|
|
582
|
+
meta["relationships"][relationship]["propsTypes"] = props_type_map
|
|
583
|
+
|
|
584
|
+
return GraphSchema(
|
|
585
|
+
categories=[Category(**cat) for cat in meta["categories"].values()],
|
|
586
|
+
relationships=[Relationship(**rel) for rel in meta["relationships"].values()]
|
|
587
|
+
)
|
|
588
|
+
|
|
589
|
+
except Exception as e:
|
|
590
|
+
print(f"Error getting schema for schema-less graphs: {e}")
|
|
591
|
+
return GraphSchema(
|
|
592
|
+
categories=[],
|
|
593
|
+
relationships=[]
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
async def get_graph_schema(self) -> GraphSchemaResponse:
|
|
597
|
+
"""Get Spanner Property Graph schema"""
|
|
598
|
+
try:
|
|
599
|
+
start_time = time.time()
|
|
600
|
+
if not self.database:
|
|
601
|
+
await self.connect()
|
|
602
|
+
|
|
603
|
+
# Get Property Graph information
|
|
604
|
+
graph_schema_query = f"""
|
|
605
|
+
SELECT
|
|
606
|
+
PG.PROPERTY_GRAPH_NAME as graphDB, PG.PROPERTY_GRAPH_METADATA_JSON as metaJSON
|
|
607
|
+
FROM
|
|
608
|
+
INFORMATION_SCHEMA.PROPERTY_GRAPHS as PG
|
|
609
|
+
WHERE PG.PROPERTY_GRAPH_NAME = "{self.config.graph_name}"
|
|
610
|
+
"""
|
|
611
|
+
schema_results = self._execute_sql_query(graph_schema_query)
|
|
612
|
+
meta = {
|
|
613
|
+
"categories": {},
|
|
614
|
+
"relationships": {}
|
|
615
|
+
}
|
|
616
|
+
if not schema_results or not schema_results.data or len(schema_results.data) == 0:
|
|
617
|
+
return GraphSchemaResponse(
|
|
618
|
+
success=False,
|
|
619
|
+
error="No schema results found",
|
|
620
|
+
execution_time= time.time() - start_time
|
|
621
|
+
)
|
|
622
|
+
|
|
623
|
+
meta_json = schema_results.data[0].get("metaJSON")
|
|
624
|
+
|
|
625
|
+
if not meta_json:
|
|
626
|
+
return GraphSchemaResponse(
|
|
627
|
+
success=False,
|
|
628
|
+
error="No metadata found",
|
|
629
|
+
execution_time= time.time() - start_time
|
|
630
|
+
)
|
|
631
|
+
|
|
632
|
+
if self._is_schema_less(meta_json):
|
|
633
|
+
meta = self._getSchemaForSchemaLessGraphs()
|
|
634
|
+
return GraphSchemaResponse(
|
|
635
|
+
success=True,
|
|
636
|
+
data=meta,
|
|
637
|
+
execution_time= time.time() - start_time
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
# Build property declarations map
|
|
641
|
+
property_declarations_map = {}
|
|
642
|
+
for prop_decl in meta_json.get("propertyDeclarations", []):
|
|
643
|
+
property_declarations_map[prop_decl.get("name")] = prop_decl.get("type")
|
|
644
|
+
|
|
645
|
+
# Build node table label map
|
|
646
|
+
node_table_label_map = {}
|
|
647
|
+
|
|
648
|
+
# Process nodes as categories
|
|
649
|
+
for node_table in meta_json.get("nodeTables", []):
|
|
650
|
+
category_name = node_table.get("labelNames", [])[0] if node_table.get("labelNames") else None
|
|
651
|
+
if not category_name:
|
|
652
|
+
continue
|
|
653
|
+
|
|
654
|
+
props_types = {}
|
|
655
|
+
for prop_def in node_table.get("propertyDefinitions", []):
|
|
656
|
+
prop_name = prop_def.get("propertyDeclarationName")
|
|
657
|
+
prop_type = (property_declarations_map.get(prop_name) or
|
|
658
|
+
property_declarations_map.get(prop_def.get("valueExpressionSql")) or
|
|
659
|
+
"string")
|
|
660
|
+
props_types[prop_name] = prop_type
|
|
661
|
+
|
|
662
|
+
table_name = (node_table.get("baseTableName") or
|
|
663
|
+
node_table.get("name") or
|
|
664
|
+
category_name)
|
|
665
|
+
node_table_label_map[table_name] = category_name
|
|
666
|
+
|
|
667
|
+
meta["categories"][category_name] = {
|
|
668
|
+
"name": category_name,
|
|
669
|
+
"props": list(props_types.keys()),
|
|
670
|
+
"propsTypes": props_types or {},
|
|
671
|
+
"keys": node_table.get("keyColumns", [])
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
# Process edges as relationships
|
|
675
|
+
for edge_table in meta_json.get("edgeTables", []):
|
|
676
|
+
relationship_name = edge_table.get("labelNames", [])[0] if edge_table.get("labelNames") else None
|
|
677
|
+
if not relationship_name:
|
|
678
|
+
continue
|
|
679
|
+
|
|
680
|
+
props_types = {}
|
|
681
|
+
for prop_def in edge_table.get("propertyDefinitions", []):
|
|
682
|
+
prop_name = prop_def.get("propertyDeclarationName")
|
|
683
|
+
prop_type = (property_declarations_map.get(prop_name) or
|
|
684
|
+
property_declarations_map.get(prop_def.get("valueExpressionSql")) or
|
|
685
|
+
"string")
|
|
686
|
+
props_types[prop_name] = prop_type
|
|
687
|
+
|
|
688
|
+
source_node_table = edge_table.get("sourceNodeTable", {}).get("nodeTableName")
|
|
689
|
+
dest_node_table = edge_table.get("destinationNodeTable", {}).get("nodeTableName")
|
|
690
|
+
|
|
691
|
+
meta["relationships"][relationship_name] = {
|
|
692
|
+
"name": relationship_name,
|
|
693
|
+
"propsTypes": props_types or {},
|
|
694
|
+
"props": list(props_types.keys()),
|
|
695
|
+
"keys": edge_table.get("keyColumns", []),
|
|
696
|
+
"startCategory": node_table_label_map.get(source_node_table, source_node_table),
|
|
697
|
+
"endCategory": node_table_label_map.get(dest_node_table, dest_node_table)
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
return GraphSchemaResponse(success=True, data={
|
|
701
|
+
"categories": list(meta["categories"].values()),
|
|
702
|
+
"relationships": list(meta["relationships"].values())
|
|
703
|
+
})
|
|
704
|
+
|
|
705
|
+
except Exception as e:
|
|
706
|
+
return GraphSchemaResponse(success=False, error=str(e))
|
|
707
|
+
|
|
708
|
+
async def get_schema(self) -> SchemaResponse:
|
|
709
|
+
"""Get Spanner database schema"""
|
|
710
|
+
try:
|
|
711
|
+
if not self.database:
|
|
712
|
+
await self.connect()
|
|
713
|
+
|
|
714
|
+
# Get table information
|
|
715
|
+
schema_query = """
|
|
716
|
+
SELECT
|
|
717
|
+
table_name as tableName,
|
|
718
|
+
column_name as columnName,
|
|
719
|
+
spanner_type as spannerType
|
|
720
|
+
FROM
|
|
721
|
+
INFORMATION_SCHEMA.COLUMNS
|
|
722
|
+
WHERE
|
|
723
|
+
table_schema NOT IN ('INFORMATION_SCHEMA', 'SPANNER_SYS')
|
|
724
|
+
"""
|
|
725
|
+
|
|
726
|
+
results = self._execute_sql_query(schema_query)
|
|
727
|
+
|
|
728
|
+
schema = {}
|
|
729
|
+
for row in results.data:
|
|
730
|
+
table_name = row["tableName"]
|
|
731
|
+
column_name = row["columnName"]
|
|
732
|
+
column_type = row["spannerType"]
|
|
733
|
+
|
|
734
|
+
if table_name not in schema:
|
|
735
|
+
schema[table_name] = {}
|
|
736
|
+
|
|
737
|
+
schema[table_name][column_name] = column_type
|
|
738
|
+
|
|
739
|
+
return SchemaResponse(success=True, data=schema)
|
|
740
|
+
|
|
741
|
+
except Exception as e:
|
|
742
|
+
return SchemaResponse(success=False, error=str(e))
|
|
743
|
+
|
|
744
|
+
async def get_sample_data(self) -> SampleDataResponse:
|
|
745
|
+
"""Get sample data from Spanner database"""
|
|
746
|
+
try:
|
|
747
|
+
limit = 10 # Number of rows to sample from each table/graph
|
|
748
|
+
if not self.database:
|
|
749
|
+
await self.connect()
|
|
750
|
+
|
|
751
|
+
sample_data = {
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
# Get list of tables
|
|
755
|
+
tables_query = """
|
|
756
|
+
SELECT
|
|
757
|
+
table_name
|
|
758
|
+
FROM
|
|
759
|
+
INFORMATION_SCHEMA.TABLES
|
|
760
|
+
WHERE
|
|
761
|
+
table_type = 'BASE TABLE'
|
|
762
|
+
"""
|
|
763
|
+
|
|
764
|
+
table_results = self._execute_sql_query(tables_query)
|
|
765
|
+
|
|
766
|
+
def get_table_sample(table_name: str):
|
|
767
|
+
"""Get sample data for a single table"""
|
|
768
|
+
try:
|
|
769
|
+
sample_query = f"SELECT * FROM `{table_name}` LIMIT {limit}"
|
|
770
|
+
sample_rows = self._execute_sql_query(sample_query)
|
|
771
|
+
return table_name, sample_rows.data
|
|
772
|
+
|
|
773
|
+
except Exception as table_error:
|
|
774
|
+
# Return empty list for tables that can't be queried
|
|
775
|
+
return table_name, []
|
|
776
|
+
|
|
777
|
+
# Extract table names
|
|
778
|
+
table_names = [row['table_name'] for row in table_results.data]
|
|
779
|
+
|
|
780
|
+
# Execute queries for each table
|
|
781
|
+
for table_name in table_names:
|
|
782
|
+
table_name, sample_rows = get_table_sample(table_name)
|
|
783
|
+
sample_data[table_name] = sample_rows
|
|
784
|
+
|
|
785
|
+
return SampleDataResponse(success=True, data=sample_data)
|
|
786
|
+
|
|
787
|
+
except Exception as e:
|
|
788
|
+
return SampleDataResponse(success=False, error=str(e))
|
|
789
|
+
|
|
790
|
+
def get_api_info(self, project_name: str) -> Dict[str, Any]:
|
|
791
|
+
"""Get API information for Spanner"""
|
|
792
|
+
base_url = f"/api/spanner/{project_name}"
|
|
793
|
+
return {
|
|
794
|
+
"type": "spanner",
|
|
795
|
+
"api_urls": {
|
|
796
|
+
"info": base_url,
|
|
797
|
+
"query": f"{base_url}/query",
|
|
798
|
+
"schema": f"{base_url}/schema",
|
|
799
|
+
"graphSchema": f"{base_url}/graphSchema",
|
|
800
|
+
"sampleData": f"{base_url}/sampleData",
|
|
801
|
+
"tokenStatus": f"{base_url}/token-status",
|
|
802
|
+
"test": f"{base_url}/test"
|
|
803
|
+
},
|
|
804
|
+
"version": "1.0",
|
|
805
|
+
"features": {
|
|
806
|
+
"property_graph": True,
|
|
807
|
+
"sql": True,
|
|
808
|
+
"schema": True,
|
|
809
|
+
"graph_schema": True,
|
|
810
|
+
"sample_data": True,
|
|
811
|
+
"token_management": True,
|
|
812
|
+
"transactions": True
|
|
813
|
+
}
|
|
814
|
+
}
|