ownapi 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli.py +1017 -0
- compiler.py +948 -0
- core/__init__.py +19 -0
- core/api_keys.py +523 -0
- core/auto_repair.py +682 -0
- core/base_client.py +165 -0
- core/consent.py +169 -0
- core/form_automation.py +1044 -0
- core/hitl_queue.py +886 -0
- core/onboarding.py +1457 -0
- core/token_store.py +223 -0
- healer.py +2074 -0
- master_mcp.py +193 -0
- ownapi/__init__.py +94 -0
- ownapi/cli.py +309 -0
- ownapi/sdk.py +436 -0
- ownapi-1.0.0.dist-info/METADATA +287 -0
- ownapi-1.0.0.dist-info/RECORD +62 -0
- ownapi-1.0.0.dist-info/WHEEL +5 -0
- ownapi-1.0.0.dist-info/entry_points.txt +4 -0
- ownapi-1.0.0.dist-info/top_level.txt +10 -0
- recorder.py +486 -0
- registry/coingecko/__init__.py +5 -0
- registry/coingecko/client.py +84 -0
- registry/coingecko/schema.json +23 -0
- registry/edevlet/__init__.py +5 -0
- registry/edevlet/client.py +54 -0
- registry/edevlet/schema.json +14 -0
- registry/enuygun/__init__.py +5 -0
- registry/enuygun/client.py +78 -0
- registry/enuygun/schema.json +13 -0
- registry/github_trending/__init__.py +5 -0
- registry/github_trending/client.py +90 -0
- registry/github_trending/schema.json +31 -0
- registry/hackernews/__init__.py +5 -0
- registry/hackernews/client.py +90 -0
- registry/hackernews/schema.json +38 -0
- registry/hepsiburada/__init__.py +5 -0
- registry/hepsiburada/client.py +32 -0
- registry/hepsiburada/schema.json +23 -0
- registry/linkedin_jobs/__init__.py +5 -0
- registry/linkedin_jobs/client.py +241 -0
- registry/linkedin_jobs/schema.json +46 -0
- registry/pegasus/__init__.py +5 -0
- registry/pegasus/client.py +141 -0
- registry/pegasus/schema.json +76 -0
- registry/thy/__init__.py +5 -0
- registry/thy/client.py +1796 -0
- registry/thy/schema.json +46 -0
- registry/trendyol/__init__.py +5 -0
- registry/trendyol/client.py +228 -0
- registry/trendyol/schema.json +30 -0
- registry/ucak_bileti/client.py +94 -0
- registry/ucak_bileti/schema.json +47 -0
- registry/wikipedia/__init__.py +5 -0
- registry/wikipedia/client.py +64 -0
- registry/wikipedia/schema.json +31 -0
- registry/yahoo_finance/__init__.py +5 -0
- registry/yahoo_finance/client.py +98 -0
- registry/yahoo_finance/schema.json +22 -0
- server.py +891 -0
- static/index.html +2401 -0
core/onboarding.py
ADDED
|
@@ -0,0 +1,1457 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import copy
|
|
5
|
+
import importlib.util
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
import traceback
|
|
12
|
+
import uuid
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
from urllib.parse import parse_qs, urlparse
|
|
17
|
+
|
|
18
|
+
from dotenv import load_dotenv
|
|
19
|
+
from openai import OpenAI
|
|
20
|
+
from playwright.async_api import Page, Request, Response, async_playwright
|
|
21
|
+
from rich.console import Console
|
|
22
|
+
from rich.panel import Panel
|
|
23
|
+
from rich.table import Table
|
|
24
|
+
|
|
25
|
+
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
26
|
+
if str(PROJECT_ROOT) not in sys.path:
|
|
27
|
+
sys.path.insert(0, str(PROJECT_ROOT))
|
|
28
|
+
|
|
29
|
+
from compiler import DynamicParamSpec, RegistrySchema
|
|
30
|
+
from core.consent import ConsentManager
|
|
31
|
+
from core.token_store import TokenStore
|
|
32
|
+
from recorder import IGNORED_DOMAIN_KEYWORDS, RELEVANT_HEADER_PREFIXES
|
|
33
|
+
|
|
34
|
+
console = Console(stderr=True, legacy_windows=False)
|
|
35
|
+
|
|
36
|
+
MAX_LLM_FIELD_CHARS = 2000
|
|
37
|
+
|
|
38
|
+
STATIC_FILE_EXTENSIONS: tuple[str, ...] = (
|
|
39
|
+
".css",
|
|
40
|
+
".js",
|
|
41
|
+
".mjs",
|
|
42
|
+
".cjs",
|
|
43
|
+
".map",
|
|
44
|
+
".png",
|
|
45
|
+
".jpg",
|
|
46
|
+
".jpeg",
|
|
47
|
+
".gif",
|
|
48
|
+
".svg",
|
|
49
|
+
".ico",
|
|
50
|
+
".webp",
|
|
51
|
+
".avif",
|
|
52
|
+
".bmp",
|
|
53
|
+
".woff",
|
|
54
|
+
".woff2",
|
|
55
|
+
".ttf",
|
|
56
|
+
".eot",
|
|
57
|
+
".otf",
|
|
58
|
+
".mp4",
|
|
59
|
+
".webm",
|
|
60
|
+
".mp3",
|
|
61
|
+
".wav",
|
|
62
|
+
".ogg",
|
|
63
|
+
".pdf",
|
|
64
|
+
".zip",
|
|
65
|
+
".gz",
|
|
66
|
+
".tar",
|
|
67
|
+
".wasm",
|
|
68
|
+
".webmanifest",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
EXCLUDED_NOISE_URL_KEYWORDS: tuple[str, ...] = (
|
|
73
|
+
"allowlist",
|
|
74
|
+
"/litms/",
|
|
75
|
+
"static.licdn.com",
|
|
76
|
+
"/aero-v1/",
|
|
77
|
+
"/sc/h/",
|
|
78
|
+
"sensorcollect",
|
|
79
|
+
"realtimefrontend",
|
|
80
|
+
"realtime/connect",
|
|
81
|
+
"demdex.net",
|
|
82
|
+
"protechts.net",
|
|
83
|
+
"ns1p.net",
|
|
84
|
+
"ccm/collect",
|
|
85
|
+
"rmkt/collect",
|
|
86
|
+
"gmp/conversion",
|
|
87
|
+
"getthirdpartytrackingpixels",
|
|
88
|
+
"actions/app-config",
|
|
89
|
+
"chameleonconfig",
|
|
90
|
+
"voyagerglobalalerts",
|
|
91
|
+
"badgingitemcounts",
|
|
92
|
+
"presencestatuses",
|
|
93
|
+
"awaystatus",
|
|
94
|
+
"affiliatedmailboxes",
|
|
95
|
+
"voyagerdashmysettings",
|
|
96
|
+
"policy/notices",
|
|
97
|
+
"cdn-cgi",
|
|
98
|
+
"measurement",
|
|
99
|
+
"analytics",
|
|
100
|
+
"telemetry",
|
|
101
|
+
"doubleclick",
|
|
102
|
+
"clarity.ms",
|
|
103
|
+
"sentry.io",
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
PRIORITY_ENDPOINT_KEYWORDS: tuple[str, ...] = (
|
|
107
|
+
"search",
|
|
108
|
+
"jobs",
|
|
109
|
+
"query",
|
|
110
|
+
"voyager",
|
|
111
|
+
"api",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _is_static_asset_url(url: str) -> bool:
|
|
116
|
+
"""
|
|
117
|
+
URL yolunun statik bir dosyaya (.css, .js, resim, font, static CDN) veya
|
|
118
|
+
izin listesi / telemetri (`allowlist`, `/litms/`, `sensorCollect` vb.) adresine
|
|
119
|
+
ait olup olmadığını denetler.
|
|
120
|
+
"""
|
|
121
|
+
if not url:
|
|
122
|
+
return False
|
|
123
|
+
try:
|
|
124
|
+
url_l = url.lower().strip()
|
|
125
|
+
if any(noise in url_l for noise in EXCLUDED_NOISE_URL_KEYWORDS):
|
|
126
|
+
return True
|
|
127
|
+
parsed = urlparse(url_l)
|
|
128
|
+
host = (parsed.hostname or "").lower()
|
|
129
|
+
path = (parsed.path or "").lower().strip()
|
|
130
|
+
if host.startswith("static.") or "cdn." in host:
|
|
131
|
+
return True
|
|
132
|
+
if any(path.endswith(ext) for ext in STATIC_FILE_EXTENSIONS):
|
|
133
|
+
return True
|
|
134
|
+
if any(seg in path for seg in ("/static/", "/assets/", "/fonts/", "/images/", "/img/", "/favicon", "/aero-v1/")):
|
|
135
|
+
if not any(api_kw in path for api_kw in ("/api/", "/v1/", "/v2/", "/graphql", ".json")):
|
|
136
|
+
return True
|
|
137
|
+
return False
|
|
138
|
+
except Exception:
|
|
139
|
+
return False
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _extract_url_search_params(url: str) -> dict[str, str]:
|
|
143
|
+
"""
|
|
144
|
+
Hedef veya yakalanan URL içindeki `keywords`, `query`, `q`, `location` gibi
|
|
145
|
+
dinamik arama parametrelerini URL-decode edilmiş temiz metin olarak çıkarır.
|
|
146
|
+
"""
|
|
147
|
+
from urllib.parse import unquote_plus
|
|
148
|
+
|
|
149
|
+
extracted: dict[str, str] = {}
|
|
150
|
+
if not url:
|
|
151
|
+
return extracted
|
|
152
|
+
try:
|
|
153
|
+
parsed = urlparse(url)
|
|
154
|
+
qs = parse_qs(parsed.query)
|
|
155
|
+
for key in ("keywords", "query", "q", "search", "location", "geoId"):
|
|
156
|
+
if key in qs and qs[key]:
|
|
157
|
+
val = unquote_plus(str(qs[key][0])).strip()
|
|
158
|
+
if val:
|
|
159
|
+
extracted[key] = val
|
|
160
|
+
except Exception:
|
|
161
|
+
pass
|
|
162
|
+
return extracted
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _score_endpoint_priority(ep: dict[str, Any], target_url: str = "") -> int:
|
|
166
|
+
"""
|
|
167
|
+
Yakalanan 200+ XHR/Fetch isteği arasından rastgele ilk GET isteğini seçmek yerine;
|
|
168
|
+
URL, query_params ve gövde içinde `search`, `jobs`, `query`, `voyager`, `api` ve
|
|
169
|
+
`keywords` geçen anlamlı veri uç noktalarını puanlayarak önceliklendirir.
|
|
170
|
+
"""
|
|
171
|
+
full_u = str(ep.get("full_url") or ep.get("url") or "").strip()
|
|
172
|
+
url_l = full_u.lower()
|
|
173
|
+
if _is_static_asset_url(full_u):
|
|
174
|
+
return -1000
|
|
175
|
+
|
|
176
|
+
score = 0
|
|
177
|
+
path_l = str(ep.get("path") or urlparse(full_u).path or "").lower()
|
|
178
|
+
q_params = ep.get("query_params") or {}
|
|
179
|
+
payload_str = str(ep.get("request_payload") or "").lower()
|
|
180
|
+
resp_str = str(ep.get("response_sample") or "").strip()
|
|
181
|
+
|
|
182
|
+
# 1. Öncelikli URL kelimeleri: search, jobs, query, voyager, api
|
|
183
|
+
if "search" in url_l or "search" in payload_str:
|
|
184
|
+
score += 45
|
|
185
|
+
if "jobs" in url_l or "jobsearch" in url_l or "jobseeker" in url_l:
|
|
186
|
+
score += 40
|
|
187
|
+
if "query" in url_l or "graphql" in url_l:
|
|
188
|
+
score += 30
|
|
189
|
+
if "voyager" in url_l:
|
|
190
|
+
score += 25
|
|
191
|
+
if "/api/" in path_l or "api." in url_l or "server-request" in path_l:
|
|
192
|
+
score += 20
|
|
193
|
+
|
|
194
|
+
# 2. Arama parametresi (`keywords`, `q`, `query`) eşleşmesi
|
|
195
|
+
q_keys_lower = {str(k).lower() for k in q_params.keys()}
|
|
196
|
+
if "keywords" in q_keys_lower or "keywords=" in url_l or '"keywords"' in payload_str:
|
|
197
|
+
score += 55
|
|
198
|
+
elif q_keys_lower.intersection({"q", "query", "search", "keyword"}):
|
|
199
|
+
score += 35
|
|
200
|
+
|
|
201
|
+
target_params = _extract_url_search_params(target_url)
|
|
202
|
+
for tk, tv in target_params.items():
|
|
203
|
+
if tv and (tv.lower() in url_l or tv.lower() in payload_str):
|
|
204
|
+
score += 30
|
|
205
|
+
|
|
206
|
+
# 3. Yanıt içeriği kalitesi
|
|
207
|
+
if resp_str.startswith("{") or resp_str.startswith("["):
|
|
208
|
+
score += 25
|
|
209
|
+
elif "0:{" in resp_str or '"response":' in resp_str:
|
|
210
|
+
score += 20
|
|
211
|
+
if len(resp_str) > 200:
|
|
212
|
+
score += 10
|
|
213
|
+
|
|
214
|
+
# Arka plan/prefech veya ilgisiz profil/mesajlaşma uç noktalarını düşür
|
|
215
|
+
if any(low_kw in url_l for low_kw in ("messaging", "notifications", "mailbox", "/feed/", "peoplewhocanhelp", "jobalerttoggle")):
|
|
216
|
+
score -= 35
|
|
217
|
+
if '"isprefetch": true' in payload_str or '"isprefetch":true' in payload_str:
|
|
218
|
+
score -= 25
|
|
219
|
+
|
|
220
|
+
return score
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _truncate_for_llm(value: Any, max_chars: int = MAX_LLM_FIELD_CHARS) -> Any:
|
|
224
|
+
"""post_data / request_payload veya yanıt metnini maksimum `max_chars` (2000) karakterle sınırlar."""
|
|
225
|
+
if value is None:
|
|
226
|
+
return None
|
|
227
|
+
if isinstance(value, str):
|
|
228
|
+
return value[:max_chars]
|
|
229
|
+
try:
|
|
230
|
+
serialized = json.dumps(value, ensure_ascii=False)
|
|
231
|
+
if len(serialized) <= max_chars:
|
|
232
|
+
return value
|
|
233
|
+
return serialized[:max_chars]
|
|
234
|
+
except Exception:
|
|
235
|
+
return str(value)[:max_chars]
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _sanitize_endpoint_for_llm(ep: dict[str, Any], max_chars: int = MAX_LLM_FIELD_CHARS) -> dict[str, Any]:
|
|
239
|
+
"""
|
|
240
|
+
LLM'e gönderilecek tekil uç nokta kaydının post_data (request_payload), response_sample
|
|
241
|
+
ve büyük header değerlerini 2000 karakter sınırı altında tutarak 128k token aşımını engeller.
|
|
242
|
+
"""
|
|
243
|
+
cleaned = dict(ep)
|
|
244
|
+
cleaned["request_payload"] = _truncate_for_llm(ep.get("request_payload"), max_chars=max_chars)
|
|
245
|
+
resp_sample = str(ep.get("response_sample", "") or "")
|
|
246
|
+
cleaned["response_sample"] = resp_sample[:max_chars]
|
|
247
|
+
cleaned["response_truncated"] = len(resp_sample) > max_chars or bool(ep.get("response_truncated"))
|
|
248
|
+
|
|
249
|
+
raw_hdrs = ep.get("headers")
|
|
250
|
+
if isinstance(raw_hdrs, dict):
|
|
251
|
+
cleaned["headers"] = {
|
|
252
|
+
str(k): (str(v)[:500] if len(str(v)) > 500 else str(v))
|
|
253
|
+
for k, v in raw_hdrs.items()
|
|
254
|
+
}
|
|
255
|
+
return cleaned
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _clean_and_validate_python_code(code: str) -> str:
|
|
259
|
+
"""
|
|
260
|
+
LLM tarafından üretilen Python kodundaki olası Markdown çitlerini (```python ... ```)
|
|
261
|
+
temizler ve `ast.parse` ile Python sözdizimi (syntax) doğrulaması yapar.
|
|
262
|
+
"""
|
|
263
|
+
import ast
|
|
264
|
+
|
|
265
|
+
cleaned = (code or "").strip()
|
|
266
|
+
if cleaned.startswith("```"):
|
|
267
|
+
lines = cleaned.splitlines()
|
|
268
|
+
if lines and lines[0].startswith("```"):
|
|
269
|
+
lines = lines[1:]
|
|
270
|
+
if lines and lines[-1].strip().startswith("```"):
|
|
271
|
+
lines = lines[:-1]
|
|
272
|
+
cleaned = "\n".join(lines).strip()
|
|
273
|
+
|
|
274
|
+
# Syntax doğrulaması (geçersizse SyntaxError fırlatır ve çağıran katman onarım/fallback uygular)
|
|
275
|
+
ast.parse(cleaned)
|
|
276
|
+
return cleaned
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _to_camel_case(snake_str: str) -> str:
|
|
280
|
+
"""snake_case metni CamelCase formatına dönüştürür."""
|
|
281
|
+
components = re.sub(r"[^a-zA-Z0-9_]", "_", snake_str).split("_")
|
|
282
|
+
return "".join(x.capitalize() for x in components if x)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _sanitize_service_name(name: str) -> str:
|
|
286
|
+
"""Servis adını geçerli bir klasör ve Python tanımlayıcısına (snake_case) çevirir."""
|
|
287
|
+
clean = re.sub(r"[^a-zA-Z0-9_]", "_", name.strip().lower())
|
|
288
|
+
clean = re.sub(r"_+", "_", clean).strip("_")
|
|
289
|
+
if not clean:
|
|
290
|
+
clean = f"service_{uuid.uuid4().hex[:6]}"
|
|
291
|
+
if clean[0].isdigit():
|
|
292
|
+
clean = f"svc_{clean}"
|
|
293
|
+
return clean
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
async def _record_traffic_async(
|
|
297
|
+
start_url: str,
|
|
298
|
+
output_path: Path,
|
|
299
|
+
service_name: str,
|
|
300
|
+
tenant_id: str = "default",
|
|
301
|
+
headless: bool = False,
|
|
302
|
+
wait_seconds: int | None = None,
|
|
303
|
+
) -> dict[str, Any]:
|
|
304
|
+
"""
|
|
305
|
+
Playwright ile temiz Chrome penceresi açarak kullanıcının oturumundaki
|
|
306
|
+
XHR/Fetch JSON API isteklerini, yanıt örneklerini ve çerezlerini kaydeder.
|
|
307
|
+
- wait_seconds None ise: CLI modunda terminalde ENTER basılmasını bekler.
|
|
308
|
+
- wait_seconds belirtilmişse: Web Dashboard modunda tarayıcı içine yüzen bir
|
|
309
|
+
'Kaydı Tamamla & Derle' butonu enjekte eder ve kullanıcı butona basana veya
|
|
310
|
+
süre dolana kadar dinler.
|
|
311
|
+
"""
|
|
312
|
+
profile_dir = PROJECT_ROOT / ".browser_profile"
|
|
313
|
+
profile_dir.mkdir(parents=True, exist_ok=True)
|
|
314
|
+
|
|
315
|
+
captured_endpoints: list[dict[str, Any]] = []
|
|
316
|
+
endpoint_counter = 0
|
|
317
|
+
token_store = TokenStore()
|
|
318
|
+
|
|
319
|
+
async with async_playwright() as p:
|
|
320
|
+
is_headless_env = bool(
|
|
321
|
+
headless or (sys.platform.startswith("linux") and not os.getenv("DISPLAY"))
|
|
322
|
+
)
|
|
323
|
+
launch_kwargs: dict[str, Any] = {
|
|
324
|
+
"user_data_dir": str(profile_dir.resolve()),
|
|
325
|
+
"headless": is_headless_env,
|
|
326
|
+
"args": [
|
|
327
|
+
"--disable-blink-features=AutomationControlled",
|
|
328
|
+
"--no-sandbox",
|
|
329
|
+
"--disable-setuid-sandbox",
|
|
330
|
+
"--disable-dev-shm-usage",
|
|
331
|
+
],
|
|
332
|
+
}
|
|
333
|
+
context = await p.chromium.launch_persistent_context(**launch_kwargs)
|
|
334
|
+
|
|
335
|
+
page = context.pages[0] if context.pages else await context.new_page()
|
|
336
|
+
if not headless:
|
|
337
|
+
try:
|
|
338
|
+
await page.bring_to_front()
|
|
339
|
+
except Exception:
|
|
340
|
+
pass
|
|
341
|
+
|
|
342
|
+
async def on_response(res: Response) -> None:
|
|
343
|
+
nonlocal endpoint_counter
|
|
344
|
+
try:
|
|
345
|
+
req = res.request
|
|
346
|
+
resource_type = req.resource_type
|
|
347
|
+
if resource_type not in ("xhr", "fetch"):
|
|
348
|
+
return
|
|
349
|
+
|
|
350
|
+
req_url = req.url
|
|
351
|
+
if _is_static_asset_url(req_url):
|
|
352
|
+
return
|
|
353
|
+
|
|
354
|
+
try:
|
|
355
|
+
parsed_url = urlparse(req_url)
|
|
356
|
+
hostname = (parsed_url.hostname or "").lower()
|
|
357
|
+
except Exception:
|
|
358
|
+
return
|
|
359
|
+
|
|
360
|
+
if any(ign in hostname for ign in IGNORED_DOMAIN_KEYWORDS):
|
|
361
|
+
return
|
|
362
|
+
|
|
363
|
+
try:
|
|
364
|
+
raw_headers = dict(req.headers)
|
|
365
|
+
except (UnicodeDecodeError, Exception):
|
|
366
|
+
raw_headers = {}
|
|
367
|
+
|
|
368
|
+
filtered_headers: dict[str, str] = {}
|
|
369
|
+
for k, v in raw_headers.items():
|
|
370
|
+
lk = k.lower().strip()
|
|
371
|
+
if any(lk == p or lk.startswith(p) for p in RELEVANT_HEADER_PREFIXES):
|
|
372
|
+
filtered_headers[lk] = v
|
|
373
|
+
|
|
374
|
+
# post_data okuma: sıkıştırılmış (gzip) veya binary (octet-stream) verilerde
|
|
375
|
+
# UnicodeDecodeError fırlatılmasını try-except ile güvenle yakala ve 2000 karakterle sınırla
|
|
376
|
+
payload: Any = None
|
|
377
|
+
try:
|
|
378
|
+
raw_post = req.post_data
|
|
379
|
+
if raw_post:
|
|
380
|
+
clipped_post = raw_post[:MAX_LLM_FIELD_CHARS]
|
|
381
|
+
try:
|
|
382
|
+
payload = _truncate_for_llm(json.loads(raw_post), max_chars=MAX_LLM_FIELD_CHARS)
|
|
383
|
+
except Exception:
|
|
384
|
+
payload = clipped_post
|
|
385
|
+
except (UnicodeDecodeError, Exception):
|
|
386
|
+
try:
|
|
387
|
+
buf = req.post_data_buffer
|
|
388
|
+
if buf:
|
|
389
|
+
import gzip
|
|
390
|
+
|
|
391
|
+
raw_bytes = gzip.decompress(buf) if buf[:2] == b"\x1f\x8b" else buf
|
|
392
|
+
decoded_post = raw_bytes.decode("utf-8", errors="ignore").strip()
|
|
393
|
+
if decoded_post:
|
|
394
|
+
try:
|
|
395
|
+
payload = _truncate_for_llm(json.loads(decoded_post), max_chars=MAX_LLM_FIELD_CHARS)
|
|
396
|
+
except Exception:
|
|
397
|
+
payload = decoded_post[:MAX_LLM_FIELD_CHARS]
|
|
398
|
+
except (UnicodeDecodeError, Exception):
|
|
399
|
+
payload = None
|
|
400
|
+
|
|
401
|
+
status = res.status
|
|
402
|
+
sample = ""
|
|
403
|
+
is_truncated = False
|
|
404
|
+
try:
|
|
405
|
+
text = await res.text()
|
|
406
|
+
if text:
|
|
407
|
+
sample = text[:MAX_LLM_FIELD_CHARS]
|
|
408
|
+
is_truncated = len(text) > MAX_LLM_FIELD_CHARS
|
|
409
|
+
except (UnicodeDecodeError, Exception):
|
|
410
|
+
try:
|
|
411
|
+
body_bytes = await res.body()
|
|
412
|
+
if body_bytes:
|
|
413
|
+
import gzip
|
|
414
|
+
|
|
415
|
+
raw_body = gzip.decompress(body_bytes) if body_bytes[:2] == b"\x1f\x8b" else body_bytes
|
|
416
|
+
decoded_body = raw_body.decode("utf-8", errors="ignore").strip()
|
|
417
|
+
if decoded_body:
|
|
418
|
+
sample = decoded_body[:MAX_LLM_FIELD_CHARS]
|
|
419
|
+
is_truncated = len(decoded_body) > MAX_LLM_FIELD_CHARS
|
|
420
|
+
except (UnicodeDecodeError, Exception):
|
|
421
|
+
sample = ""
|
|
422
|
+
|
|
423
|
+
endpoint_counter += 1
|
|
424
|
+
captured_endpoints.append({
|
|
425
|
+
"id": endpoint_counter,
|
|
426
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
427
|
+
"method": req.method.upper(),
|
|
428
|
+
"url": f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}",
|
|
429
|
+
"full_url": req_url,
|
|
430
|
+
"domain": hostname,
|
|
431
|
+
"path": parsed_url.path,
|
|
432
|
+
"status_code": status,
|
|
433
|
+
"resource_type": resource_type,
|
|
434
|
+
"query_params": parse_qs(parsed_url.query),
|
|
435
|
+
"headers": filtered_headers,
|
|
436
|
+
"request_payload": payload,
|
|
437
|
+
"response_sample": sample,
|
|
438
|
+
"response_truncated": is_truncated,
|
|
439
|
+
})
|
|
440
|
+
except (UnicodeDecodeError, Exception):
|
|
441
|
+
# Binary / gzip / octet-stream kaynaklı decode hatalarını akışı bozmadan yoksay
|
|
442
|
+
return
|
|
443
|
+
|
|
444
|
+
page.on("response", on_response)
|
|
445
|
+
|
|
446
|
+
console.print(
|
|
447
|
+
Panel.fit(
|
|
448
|
+
f"[bold green]🚀 Tarayıcı Açılıyor:[/bold green] [white]{start_url}[/white]\n"
|
|
449
|
+
"[yellow]Hedef işlemi (arama, form doldurma, gezinme) kendi oturumunuzla yapın.[/yellow]\n"
|
|
450
|
+
+ (
|
|
451
|
+
f"[bold cyan]Web Dashboard Modu: {wait_seconds} sn içinde işlemi yapın veya tarayıcıdaki 'Kaydı Bitir' butonuna tıklayın...[/bold cyan]"
|
|
452
|
+
if wait_seconds is not None
|
|
453
|
+
else "[bold white]Trafik kaydını tamamlamak için terminalde ENTER'a basın...[/bold white]"
|
|
454
|
+
),
|
|
455
|
+
title="OwnAPI User-Delegated Traffic Recorder",
|
|
456
|
+
border_style="green",
|
|
457
|
+
)
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
try:
|
|
461
|
+
await page.goto(start_url, wait_until="domcontentloaded")
|
|
462
|
+
except Exception as e:
|
|
463
|
+
console.print(f"[dim]Sayfa gezinme uyarısı: {e}[/dim]")
|
|
464
|
+
|
|
465
|
+
if wait_seconds is not None:
|
|
466
|
+
total_steps = max(2, int(wait_seconds * 2))
|
|
467
|
+
for step_i in range(total_steps):
|
|
468
|
+
remaining = max(0, int(wait_seconds - (step_i * 0.5)))
|
|
469
|
+
try:
|
|
470
|
+
if page.is_closed():
|
|
471
|
+
break
|
|
472
|
+
stop_clicked = await page.evaluate(
|
|
473
|
+
"""({ count, rem, svc }) => {
|
|
474
|
+
if (window.__ownapi_stop_recording) return true;
|
|
475
|
+
let bar = document.getElementById('__ownapi_rec_banner');
|
|
476
|
+
if (!bar && document.body) {
|
|
477
|
+
bar = document.createElement('div');
|
|
478
|
+
bar.id = '__ownapi_rec_banner';
|
|
479
|
+
bar.style.cssText = 'position:fixed;top:14px;right:14px;z-index:2147483647;background:#11131C;color:#F3F4F6;border:2px solid #6366F1;border-radius:12px;padding:12px 16px;font-family:system-ui,sans-serif;font-size:13px;box-shadow:0 12px 32px rgba(0,0,0,0.65);display:flex;align-items:center;gap:12px;';
|
|
480
|
+
bar.innerHTML = `
|
|
481
|
+
<div>
|
|
482
|
+
<div style="font-weight:700;color:#818CF8;">🔴 OwnAPI Recording: ${svc}</div>
|
|
483
|
+
<div id="__ownapi_rec_stat" style="font-size:11px;color:#9CA3AF;margin-top:2px;">Yakalanan XHR/Fetch: ${count} | Kalan: ${rem}s</div>
|
|
484
|
+
</div>
|
|
485
|
+
<button id="__ownapi_rec_btn" style="background:#10B981;color:#fff;border:none;border-radius:8px;padding:7px 12px;font-weight:700;cursor:pointer;font-size:12px;">
|
|
486
|
+
✅ Kaydı Bitir & Derle
|
|
487
|
+
</button>
|
|
488
|
+
`;
|
|
489
|
+
document.body.appendChild(bar);
|
|
490
|
+
document.getElementById('__ownapi_rec_btn').onclick = () => {
|
|
491
|
+
window.__ownapi_stop_recording = true;
|
|
492
|
+
bar.style.borderColor = '#10B981';
|
|
493
|
+
bar.innerHTML = '<div style="font-weight:700;color:#10B981;">✔ Kayıt tamamlandı, GPT-4o ile derleniyor...</div>';
|
|
494
|
+
};
|
|
495
|
+
} else if (bar) {
|
|
496
|
+
const st = document.getElementById('__ownapi_rec_stat');
|
|
497
|
+
if (st) st.textContent = `Yakalanan XHR/Fetch: ${count} | Otomatik Bitiş: ${rem}s`;
|
|
498
|
+
}
|
|
499
|
+
return Boolean(window.__ownapi_stop_recording);
|
|
500
|
+
}""",
|
|
501
|
+
{"count": len(captured_endpoints), "rem": remaining, "svc": service_name},
|
|
502
|
+
)
|
|
503
|
+
if stop_clicked:
|
|
504
|
+
break
|
|
505
|
+
except Exception:
|
|
506
|
+
pass
|
|
507
|
+
await asyncio.sleep(0.5)
|
|
508
|
+
else:
|
|
509
|
+
await asyncio.to_thread(input, "\n👉 İşlem tamamlandığında ENTER'a basın...")
|
|
510
|
+
|
|
511
|
+
# Eğer sayfada hiç XHR/Fetch tetiklenmediyse (örn. doğrudan JSON API URL'si veya statik sayfa),
|
|
512
|
+
# hedef URL'nin kendisini varsayılan GET uç noktası olarak kaydet ki derleme kesintisiz çalışsın.
|
|
513
|
+
if not captured_endpoints:
|
|
514
|
+
try:
|
|
515
|
+
parsed_start = urlparse(start_url)
|
|
516
|
+
body_preview = ""
|
|
517
|
+
try:
|
|
518
|
+
body_preview = (await page.evaluate("() => document.body ? document.body.innerText.slice(0, 1500) : ''")) or ""
|
|
519
|
+
except Exception:
|
|
520
|
+
pass
|
|
521
|
+
captured_endpoints.append({
|
|
522
|
+
"id": 1,
|
|
523
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
524
|
+
"method": "GET",
|
|
525
|
+
"url": f"{parsed_start.scheme}://{parsed_start.netloc}{parsed_start.path or '/'}",
|
|
526
|
+
"full_url": start_url,
|
|
527
|
+
"domain": (parsed_start.hostname or "").lower(),
|
|
528
|
+
"path": parsed_start.path or "/",
|
|
529
|
+
"status_code": 200,
|
|
530
|
+
"resource_type": "fetch",
|
|
531
|
+
"query_params": parse_qs(parsed_start.query),
|
|
532
|
+
"headers": {"accept": "application/json, text/plain, */*"},
|
|
533
|
+
"request_payload": None,
|
|
534
|
+
"response_sample": body_preview[:1500],
|
|
535
|
+
"response_truncated": False,
|
|
536
|
+
})
|
|
537
|
+
except Exception:
|
|
538
|
+
pass
|
|
539
|
+
|
|
540
|
+
console.print(f"[bold green]✔ Dinleme sonlandırıldı. Toplam {len(captured_endpoints)} XHR/Fetch isteği yakalandı.[/bold green]")
|
|
541
|
+
|
|
542
|
+
# Oturum çerezlerini TokenStore'a kaydet
|
|
543
|
+
try:
|
|
544
|
+
cookies = await context.cookies()
|
|
545
|
+
user_agent = await page.evaluate("() => navigator.userAgent")
|
|
546
|
+
token_store.save_session(
|
|
547
|
+
tenant_id=tenant_id,
|
|
548
|
+
service=service_name,
|
|
549
|
+
cookies=cookies,
|
|
550
|
+
user_agent=user_agent,
|
|
551
|
+
metadata={"target_url": start_url},
|
|
552
|
+
)
|
|
553
|
+
except Exception:
|
|
554
|
+
pass
|
|
555
|
+
|
|
556
|
+
try:
|
|
557
|
+
await context.close()
|
|
558
|
+
except Exception:
|
|
559
|
+
pass
|
|
560
|
+
|
|
561
|
+
traffic_session = {
|
|
562
|
+
"target_url": start_url,
|
|
563
|
+
"recorded_at": datetime.now(timezone.utc).isoformat(),
|
|
564
|
+
"total_endpoints": len(captured_endpoints),
|
|
565
|
+
"endpoints": captured_endpoints,
|
|
566
|
+
}
|
|
567
|
+
output_path.write_text(json.dumps(traffic_session, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
568
|
+
return traffic_session
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def _normalize_query_params(raw_params: dict[str, Any]) -> dict[str, Any]:
|
|
572
|
+
"""parse_qs tarafından üretilen {'key': ['val']} yapısını {'key': 'val'} yapısına sadeleştirir."""
|
|
573
|
+
normalized: dict[str, Any] = {}
|
|
574
|
+
for k, v in (raw_params or {}).items():
|
|
575
|
+
if isinstance(v, list) and len(v) == 1:
|
|
576
|
+
normalized[k] = v[0]
|
|
577
|
+
else:
|
|
578
|
+
normalized[k] = v
|
|
579
|
+
return normalized
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _build_fallback_code(
|
|
583
|
+
service_name: str,
|
|
584
|
+
target_url: str,
|
|
585
|
+
endpoint: dict[str, Any],
|
|
586
|
+
description: str,
|
|
587
|
+
) -> tuple[str, RegistrySchema]:
|
|
588
|
+
"""LLM erişimi olmadığında veya hata aldığında çalışan standart BaseOwnClient kodunu üretir."""
|
|
589
|
+
camel_name = _to_camel_case(service_name)
|
|
590
|
+
target_search_params = _extract_url_search_params(target_url)
|
|
591
|
+
ep_search_params = _extract_url_search_params(str(endpoint.get("full_url") or endpoint.get("url") or ""))
|
|
592
|
+
merged_search_params = {**target_search_params, **ep_search_params}
|
|
593
|
+
|
|
594
|
+
is_search = bool(merged_search_params) or any(
|
|
595
|
+
k in (description or "").lower() or k in service_name.lower()
|
|
596
|
+
for k in ("search", "ara", "bul", "query", "list", "get", "jobs", "ilan")
|
|
597
|
+
)
|
|
598
|
+
fn_name = f"search_{service_name}" if is_search else f"execute_{service_name}"
|
|
599
|
+
method = endpoint.get("method", "GET").upper()
|
|
600
|
+
ep_url = endpoint.get("url", target_url)
|
|
601
|
+
headers = endpoint.get("headers", {}) or {}
|
|
602
|
+
query_params = _normalize_query_params(endpoint.get("query_params", {}) or {})
|
|
603
|
+
payload = endpoint.get("request_payload")
|
|
604
|
+
|
|
605
|
+
dynamic_params: list[DynamicParamSpec] = []
|
|
606
|
+
if "keywords" in query_params or "keywords" in merged_search_params:
|
|
607
|
+
kw_default = str(query_params.get("keywords") or merged_search_params.get("keywords") or "AI Engineer")
|
|
608
|
+
if "keywords" not in query_params:
|
|
609
|
+
query_params["keywords"] = kw_default
|
|
610
|
+
dynamic_params.append(
|
|
611
|
+
DynamicParamSpec(
|
|
612
|
+
name="keywords",
|
|
613
|
+
original_key="keywords",
|
|
614
|
+
location="query",
|
|
615
|
+
param_type="str",
|
|
616
|
+
default_value=kw_default,
|
|
617
|
+
description="Aranacak iş/pozisyon veya anahtar kelime (örn. 'AI Engineer').",
|
|
618
|
+
)
|
|
619
|
+
)
|
|
620
|
+
elif query_params:
|
|
621
|
+
first_k = next(iter(query_params.keys()))
|
|
622
|
+
first_v = query_params[first_k][0] if isinstance(query_params[first_k], list) else query_params[first_k]
|
|
623
|
+
param_arg_name = "keywords" if first_k.lower() == "keywords" else "query"
|
|
624
|
+
dynamic_params.append(
|
|
625
|
+
DynamicParamSpec(
|
|
626
|
+
name=param_arg_name,
|
|
627
|
+
original_key=first_k,
|
|
628
|
+
location="query",
|
|
629
|
+
param_type="str",
|
|
630
|
+
default_value=str(first_v),
|
|
631
|
+
description=f"{first_k} sorgu parametresi",
|
|
632
|
+
)
|
|
633
|
+
)
|
|
634
|
+
elif payload and isinstance(payload, dict):
|
|
635
|
+
first_k = next(iter(payload.keys()))
|
|
636
|
+
dynamic_params.append(
|
|
637
|
+
DynamicParamSpec(
|
|
638
|
+
name="query",
|
|
639
|
+
original_key=first_k,
|
|
640
|
+
location="body",
|
|
641
|
+
param_type="str",
|
|
642
|
+
default_value=str(payload[first_k]),
|
|
643
|
+
description=f"{first_k} gövde parametresi",
|
|
644
|
+
)
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
param_var = dynamic_params[0].name if dynamic_params else "query"
|
|
648
|
+
default_q_repr = repr(dynamic_params[0].default_value) if dynamic_params else "'test'"
|
|
649
|
+
sig_params = [f"{param_var}: str = {default_q_repr}"] if dynamic_params else []
|
|
650
|
+
method_sig = ", ".join(["self"] + sig_params)
|
|
651
|
+
module_fn_sig = ", ".join(sig_params + ["tenant_id: str = 'default'", "mode: str = 'local'"])
|
|
652
|
+
call_args = ", ".join([f"{param_var}={param_var}"] if dynamic_params else [])
|
|
653
|
+
|
|
654
|
+
query_update_line = ""
|
|
655
|
+
body_update_line = ""
|
|
656
|
+
if dynamic_params and dynamic_params[0].location == "query":
|
|
657
|
+
orig_key_repr = repr(dynamic_params[0].original_key)
|
|
658
|
+
query_update_line = f"if params is not None:\n params[{orig_key_repr}] = {param_var}"
|
|
659
|
+
elif dynamic_params and dynamic_params[0].location == "body":
|
|
660
|
+
orig_key_repr = repr(dynamic_params[0].original_key)
|
|
661
|
+
body_update_line = f"if isinstance(payload, dict):\n payload[{orig_key_repr}] = {param_var}"
|
|
662
|
+
|
|
663
|
+
client_code = f'''from __future__ import annotations
|
|
664
|
+
|
|
665
|
+
import copy
|
|
666
|
+
import pathlib
|
|
667
|
+
import sys
|
|
668
|
+
from typing import Any
|
|
669
|
+
|
|
670
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
671
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
672
|
+
|
|
673
|
+
from core.base_client import BaseOwnClient
|
|
674
|
+
from core.token_store import TokenStore
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
class {camel_name}Client(BaseOwnClient):
|
|
678
|
+
"""{service_name} servisi için OwnAPI User-Delegated Browser Middleware istemcisi."""
|
|
679
|
+
|
|
680
|
+
def __init__(
|
|
681
|
+
self,
|
|
682
|
+
tenant_id: str = "default",
|
|
683
|
+
token_store: TokenStore | None = None,
|
|
684
|
+
mode: str = "local",
|
|
685
|
+
) -> None:
|
|
686
|
+
super().__init__(
|
|
687
|
+
service_name={repr(service_name)},
|
|
688
|
+
base_url={repr(target_url)},
|
|
689
|
+
tenant_id=tenant_id,
|
|
690
|
+
token_store=token_store,
|
|
691
|
+
mode=mode,
|
|
692
|
+
)
|
|
693
|
+
|
|
694
|
+
def {fn_name}({method_sig}) -> dict[str, Any]:
|
|
695
|
+
endpoint_url = {repr(ep_url)}
|
|
696
|
+
headers: dict[str, str] = {repr(headers)}
|
|
697
|
+
params: dict[str, Any] = {repr(query_params)}
|
|
698
|
+
payload: Any = {repr(payload) if payload is not None else "None"}
|
|
699
|
+
|
|
700
|
+
{query_update_line}
|
|
701
|
+
{body_update_line}
|
|
702
|
+
|
|
703
|
+
res = self.request(
|
|
704
|
+
method={repr(method)},
|
|
705
|
+
url=endpoint_url,
|
|
706
|
+
headers=headers,
|
|
707
|
+
params=params if params else None,
|
|
708
|
+
json_payload=payload if payload is not None else None,
|
|
709
|
+
)
|
|
710
|
+
return res if isinstance(res, dict) else {{"data": res}}
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def {fn_name}({module_fn_sig}) -> dict[str, Any]:
|
|
714
|
+
"""
|
|
715
|
+
{(description or service_name + ' servisini kullanıcının gerçek tarayıcı oturumu üzerinden çağırır.').replace('"', "'")}
|
|
716
|
+
"""
|
|
717
|
+
client = {camel_name}Client(tenant_id=tenant_id, mode=mode)
|
|
718
|
+
return client.{fn_name}({call_args})
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
if __name__ == "__main__":
|
|
722
|
+
result = {fn_name}()
|
|
723
|
+
print("[OwnAPI] Sonuç:", str(result)[:500])
|
|
724
|
+
'''
|
|
725
|
+
|
|
726
|
+
schema_obj = RegistrySchema(
|
|
727
|
+
service_name=service_name,
|
|
728
|
+
target_url=target_url,
|
|
729
|
+
goal=description or f"{service_name} API çağrısı",
|
|
730
|
+
function_name=fn_name,
|
|
731
|
+
explanation=f"{service_name} servisi için otomatik derlenmiş OwnAPI istemcisi.",
|
|
732
|
+
method=method,
|
|
733
|
+
endpoint_url=ep_url,
|
|
734
|
+
dynamic_params=dynamic_params,
|
|
735
|
+
created_at=datetime.now(timezone.utc).isoformat(),
|
|
736
|
+
)
|
|
737
|
+
|
|
738
|
+
return client_code, schema_obj
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def _select_meaningful_endpoints(endpoints: list[dict[str, Any]], target_url: str = "") -> list[dict[str, Any]]:
|
|
742
|
+
"""
|
|
743
|
+
Yakalanan 200+ XHR/Fetch isteği arasından:
|
|
744
|
+
1. Statik dosyaları (`.css`, `.js`, resimler, `static.licdn.com`, `/aero-v1/`) ve
|
|
745
|
+
izin listesi/telemetri (`allowlist`, `/litms/`, `sensorCollect`, `realtimeFrontend`) URL'lerini eler.
|
|
746
|
+
2. URL ve parametrelerde `search`, `jobs`, `query`, `voyager`, `api`, `keywords` geçenleri
|
|
747
|
+
`_score_endpoint_priority` ile puanlayarak en alakalı veri uç noktalarını en üste taşır.
|
|
748
|
+
3. Tüm istek gövdelerini ve yanıt örneklerini maksimum 2000 karakterle sınırlar (truncate eder).
|
|
749
|
+
"""
|
|
750
|
+
scored_candidates: list[tuple[int, int, dict[str, Any]]] = []
|
|
751
|
+
fallback_non_static: list[tuple[int, int, dict[str, Any]]] = []
|
|
752
|
+
|
|
753
|
+
for idx, ep in enumerate(endpoints):
|
|
754
|
+
full_u = str(ep.get("full_url") or ep.get("url") or "").strip()
|
|
755
|
+
if _is_static_asset_url(full_u):
|
|
756
|
+
continue
|
|
757
|
+
|
|
758
|
+
sanitized_ep = _sanitize_endpoint_for_llm(ep, max_chars=MAX_LLM_FIELD_CHARS)
|
|
759
|
+
status = sanitized_ep.get("status_code", 0)
|
|
760
|
+
if not (200 <= status < 400):
|
|
761
|
+
continue
|
|
762
|
+
|
|
763
|
+
priority_score = _score_endpoint_priority(sanitized_ep, target_url=target_url)
|
|
764
|
+
if priority_score < -100:
|
|
765
|
+
continue
|
|
766
|
+
|
|
767
|
+
fallback_non_static.append((priority_score, -idx, sanitized_ep))
|
|
768
|
+
resp = str(sanitized_ep.get("response_sample", "")).strip()
|
|
769
|
+
url_l = full_u.lower()
|
|
770
|
+
has_priority_kw = any(kw in url_l for kw in PRIORITY_ENDPOINT_KEYWORDS) or "keywords" in url_l
|
|
771
|
+
if resp.startswith("{") or resp.startswith("[") or "0:{" in resp or has_priority_kw:
|
|
772
|
+
scored_candidates.append((priority_score, -idx, sanitized_ep))
|
|
773
|
+
|
|
774
|
+
if scored_candidates:
|
|
775
|
+
scored_candidates.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
|
776
|
+
return [item[2] for item in scored_candidates]
|
|
777
|
+
|
|
778
|
+
if fallback_non_static:
|
|
779
|
+
fallback_non_static.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
|
780
|
+
return [item[2] for item in fallback_non_static[:10]]
|
|
781
|
+
|
|
782
|
+
if endpoints:
|
|
783
|
+
return [_sanitize_endpoint_for_llm(ep, max_chars=MAX_LLM_FIELD_CHARS) for ep in endpoints[:5]]
|
|
784
|
+
return []
|
|
785
|
+
|
|
786
|
+
|
|
787
|
+
def _compile_with_llm(
|
|
788
|
+
service_name: str,
|
|
789
|
+
target_url: str,
|
|
790
|
+
description: str,
|
|
791
|
+
endpoints: list[dict[str, Any]],
|
|
792
|
+
) -> tuple[str, RegistrySchema, dict[str, Any]]:
|
|
793
|
+
"""
|
|
794
|
+
OpenAI GPT-4o kullanarak statik dosyalardan/allowlist'lerden arındırılmış,
|
|
795
|
+
`search`/`jobs`/`query`/`voyager`/`api` önceliklendirilmiş ve 2000 karakterle
|
|
796
|
+
truncate edilmiş API uç noktalarını analiz edip BaseOwnClient kodunu derler.
|
|
797
|
+
"""
|
|
798
|
+
load_dotenv(override=False)
|
|
799
|
+
api_key = os.getenv("OPENAI_API_KEY", "").strip()
|
|
800
|
+
if not api_key:
|
|
801
|
+
raise RuntimeError("OPENAI_API_KEY bulunamadı!")
|
|
802
|
+
|
|
803
|
+
client = OpenAI(api_key=api_key)
|
|
804
|
+
model = os.getenv("OPENAI_MODEL", "gpt-4o").strip()
|
|
805
|
+
|
|
806
|
+
filtered_endpoints = _select_meaningful_endpoints(endpoints, target_url=target_url)
|
|
807
|
+
if not filtered_endpoints:
|
|
808
|
+
raise ValueError("Yakalanan trafikte anlamlı bir JSON API uç noktası bulunamadı.")
|
|
809
|
+
|
|
810
|
+
# LLM'e gönderilecek uç noktaları tekrar 2000 karakter sınırından geçir (128k token koruması)
|
|
811
|
+
llm_ready_endpoints = [_sanitize_endpoint_for_llm(ep, max_chars=MAX_LLM_FIELD_CHARS) for ep in filtered_endpoints[:8]]
|
|
812
|
+
camel_name = _to_camel_case(service_name)
|
|
813
|
+
target_search_params = _extract_url_search_params(target_url)
|
|
814
|
+
|
|
815
|
+
prompt = f"""\
|
|
816
|
+
Aşağıda kullanıcının `{start_url_to_domain(target_url)}` sitesinde yaptığı işlem sırasında yakalanan (statik `.css`/`.js`/resim/allowlist dosyalarından arındırılmış, `search`/`jobs`/`query`/`voyager`/`api` önceliğine göre sıralanmış ve gövdeleri maksimum 2000 karakterle sınırlandırılmış) XHR/Fetch API uç noktaları verilmiştir.
|
|
817
|
+
|
|
818
|
+
Kullanıcı Hedefi/Açıklaması: {description or service_name}
|
|
819
|
+
Servis Adı: {service_name}
|
|
820
|
+
Hedef URL: {target_url}
|
|
821
|
+
Hedef URL Arama Parametreleri: {json.dumps(target_search_params, ensure_ascii=False)}
|
|
822
|
+
|
|
823
|
+
Yakalanan Öncelikli Uç Noktalar:
|
|
824
|
+
{json.dumps(llm_ready_endpoints, ensure_ascii=False, indent=2)}
|
|
825
|
+
|
|
826
|
+
GÖREVİN:
|
|
827
|
+
1. AKILLI ENDPOINT SEÇİMİ: Rastgele bir meta veri (`litms`, `allowlist`, `metadata/user`) veya bildirim uç noktası SEÇME! URL, `query_params` veya `request_payload` içinde `search`, `jobs`, `query`, `voyager`, `api` veya `keywords` geçen EN UYGUN ASIL iş/arama/veri uç noktasını (`matched_endpoint_id`) seç.
|
|
828
|
+
2. PARAMETRE EŞLEME (`keywords` / `query`): Hedef URL'de veya seçilen uç noktada `keywords` (örneğin `"AI Engineer"`), `query`, `q` veya `location` gibi arama/filtreleme parametreleri varsa, bunları `client.py` fonksiyon imzasına dinamik argüman (örn. `keywords: str = "AI Engineer"`) olarak enjekte et ve `dynamic_params` şemasına ekle ki Live Playground veya AI Ajanı üzerinden arama kelimesi dinamik olarak değiştirilebilsin.
|
|
829
|
+
3. Uç noktanın `query_params` ve `headers` alanlarındaki ZORUNLU parametreleri (örneğin `referer`, `csrf-token` veya `x-` başlıkları) eksiksiz koru. `query_params` içindeki tek elemanlı listeleri (`["deger"]`) düz string (`"deger"`) olarak gönder.
|
|
830
|
+
4. `core.base_client.BaseOwnClient` sınıfından miras alan, `self.request(...)` metodunu kullanan, `tenant_id: str = "default"` ve `mode: str = "local"` parametrelerini destekleyen eksiksiz, temiz ve hatasız bir `client.py` Python kodu üret.
|
|
831
|
+
|
|
832
|
+
KESİN PYTHON SÖZDİZİMİ (STRICT PYTHON SYNTAX) KURALLARI:
|
|
833
|
+
- Ürettiğin `client_code` doğrudan `ast.parse()` ile derlenecektir; kesinlikle `SyntaxError` veya `IndentationError` içermemelidir.
|
|
834
|
+
- Fonksiyon imzalarında ASLA boş veya fazladan virgül bırakma (örneğin dinamik parametre yoksa `def fn(self) -> dict[str, Any]:` yaz, ASLA `def fn(self, ,):` yazma).
|
|
835
|
+
- Python sözlük/liste tanımlamalarında JSON sabitleri olan `true`, `false`, `null` KULLANMA; mutlaka geçerli Python sabitleri olan `True`, `False`, `None` kullan.
|
|
836
|
+
- `client_code` alanına Markdown kod bloğu işaretleri (```python ... ```) KOYMA, yalnızca saf Python kaynak kodu döndür.
|
|
837
|
+
- Tüm parantez `()`, köşeli parantez `[]`, süslü parantez `{{}}` ve tırnak işaretlerinin eksiksiz kapandığından emin ol.
|
|
838
|
+
- `self.request(...)` zaten ayrıştırılmış bir `dict` (sözlük) döndürür; gelen yanıt üzerinde `.text` özniteliği çağırma, doğrudan `res` döndür veya `res.get("text", json.dumps(res, ensure_ascii=False))` kullan.
|
|
839
|
+
|
|
840
|
+
Kod yapısı MUTLAKA şu standartta olmalıdır:
|
|
841
|
+
from __future__ import annotations
|
|
842
|
+
import copy
|
|
843
|
+
import json
|
|
844
|
+
import pathlib
|
|
845
|
+
import sys
|
|
846
|
+
from typing import Any
|
|
847
|
+
|
|
848
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
849
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
850
|
+
|
|
851
|
+
from core.base_client import BaseOwnClient
|
|
852
|
+
from core.token_store import TokenStore
|
|
853
|
+
|
|
854
|
+
class {camel_name}Client(BaseOwnClient):
|
|
855
|
+
def __init__(self, tenant_id: str = "default", token_store: TokenStore | None = None, mode: str = "local") -> None:
|
|
856
|
+
super().__init__(service_name="{service_name}", base_url="{target_url}", tenant_id=tenant_id, token_store=token_store, mode=mode)
|
|
857
|
+
|
|
858
|
+
def <fonksiyon_adi>(self, <varsa_dinamik_parametreler_varsayilan_degerli>) -> dict[str, Any]:
|
|
859
|
+
res = self.request(method=..., url=..., headers=..., params=..., json_payload=...)
|
|
860
|
+
return res if isinstance(res, dict) else {{"data": res}}
|
|
861
|
+
|
|
862
|
+
def <fonksiyon_adi>(<varsa_dinamik_parametreler_varsayilan_degerli>, tenant_id: str = "default", mode: str = "local") -> dict[str, Any]:
|
|
863
|
+
\"\"\"Docstring\"\"\"
|
|
864
|
+
client = {camel_name}Client(tenant_id=tenant_id, mode=mode)
|
|
865
|
+
return client.<fonksiyon_adi>(...)
|
|
866
|
+
|
|
867
|
+
Yanıtını SADECE aşağıdaki JSON formatında ver:
|
|
868
|
+
{{
|
|
869
|
+
"matched_endpoint_id": <int>,
|
|
870
|
+
"function_name": "<snake_case_fonksiyon_adi>",
|
|
871
|
+
"explanation": "<Kısa Türkçe açıklama>",
|
|
872
|
+
"dynamic_params": [
|
|
873
|
+
{{
|
|
874
|
+
"name": "<arguman_adi>",
|
|
875
|
+
"original_key": "<orijinal_anahtar>",
|
|
876
|
+
"location": "query | body | path",
|
|
877
|
+
"param_type": "str | int | float | bool",
|
|
878
|
+
"default_value": <varsayilan_deger>,
|
|
879
|
+
"description": "<açıklama>"
|
|
880
|
+
}}
|
|
881
|
+
],
|
|
882
|
+
"client_code": "<tam python kodu>"
|
|
883
|
+
}}
|
|
884
|
+
"""
|
|
885
|
+
|
|
886
|
+
resp = client.chat.completions.create(
|
|
887
|
+
model=model,
|
|
888
|
+
messages=[{"role": "user", "content": prompt}],
|
|
889
|
+
temperature=0.1,
|
|
890
|
+
response_format={"type": "json_object"},
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
content = resp.choices[0].message.content or "{}"
|
|
894
|
+
parsed = json.loads(content)
|
|
895
|
+
|
|
896
|
+
matched_id = parsed.get("matched_endpoint_id", 1)
|
|
897
|
+
matched_ep = next((e for e in filtered_endpoints if e.get("id") == matched_id), filtered_endpoints[0])
|
|
898
|
+
|
|
899
|
+
dyn_params: list[DynamicParamSpec] = []
|
|
900
|
+
for dp in parsed.get("dynamic_params", []):
|
|
901
|
+
try:
|
|
902
|
+
dyn_params.append(DynamicParamSpec(**dp))
|
|
903
|
+
except Exception:
|
|
904
|
+
pass
|
|
905
|
+
|
|
906
|
+
# Hedef URL'de `keywords` varsa ve LLM dynamic_params listesine eklemediyse otomatik tamamla
|
|
907
|
+
if "keywords" in target_search_params and not any(p.name == "keywords" for p in dyn_params):
|
|
908
|
+
dyn_params.insert(
|
|
909
|
+
0,
|
|
910
|
+
DynamicParamSpec(
|
|
911
|
+
name="keywords",
|
|
912
|
+
original_key="keywords",
|
|
913
|
+
location="query",
|
|
914
|
+
param_type="str",
|
|
915
|
+
default_value=target_search_params["keywords"],
|
|
916
|
+
description="Aranacak pozisyon, iş unvanı veya anahtar kelime.",
|
|
917
|
+
),
|
|
918
|
+
)
|
|
919
|
+
|
|
920
|
+
fn_name = parsed.get("function_name", f"search_{service_name}")
|
|
921
|
+
raw_client_code = parsed.get("client_code", "")
|
|
922
|
+
client_code = _clean_and_validate_python_code(raw_client_code)
|
|
923
|
+
|
|
924
|
+
schema_obj = RegistrySchema(
|
|
925
|
+
service_name=service_name,
|
|
926
|
+
target_url=target_url,
|
|
927
|
+
goal=description or f"{service_name} API sorgusu",
|
|
928
|
+
function_name=fn_name,
|
|
929
|
+
explanation=parsed.get("explanation", f"{service_name} için otomatik derlenmiş OwnAPI aracı."),
|
|
930
|
+
method=matched_ep.get("method", "GET").upper(),
|
|
931
|
+
endpoint_url=matched_ep.get("url", target_url),
|
|
932
|
+
dynamic_params=dyn_params,
|
|
933
|
+
created_at=datetime.now(timezone.utc).isoformat(),
|
|
934
|
+
)
|
|
935
|
+
|
|
936
|
+
return client_code, schema_obj, matched_ep
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
def _repair_client_with_llm(
|
|
940
|
+
service_name: str,
|
|
941
|
+
target_url: str,
|
|
942
|
+
description: str,
|
|
943
|
+
matched_endpoint: dict[str, Any],
|
|
944
|
+
all_endpoints: list[dict[str, Any]],
|
|
945
|
+
broken_code: str,
|
|
946
|
+
function_name: str,
|
|
947
|
+
error_detail: str,
|
|
948
|
+
) -> tuple[str, str]:
|
|
949
|
+
"""
|
|
950
|
+
Auto-Test (Validator Loop) sırasında hata alındığında, hatayı, mevcut kodu ve
|
|
951
|
+
2000 karakterle sınırlandırılmış orijinal trafik şemasını GPT-4o'ya geri besleyerek kodu otomatik onarır.
|
|
952
|
+
"""
|
|
953
|
+
load_dotenv(override=False)
|
|
954
|
+
api_key = os.getenv("OPENAI_API_KEY", "").strip()
|
|
955
|
+
if not api_key:
|
|
956
|
+
fallback_code, fallback_schema = _build_fallback_code(service_name, target_url, matched_endpoint, description)
|
|
957
|
+
return fallback_code, fallback_schema.function_name
|
|
958
|
+
|
|
959
|
+
client = OpenAI(api_key=api_key)
|
|
960
|
+
model = os.getenv("OPENAI_MODEL", "gpt-4o").strip()
|
|
961
|
+
camel_name = _to_camel_case(service_name)
|
|
962
|
+
filtered_endpoints = [_sanitize_endpoint_for_llm(ep, max_chars=MAX_LLM_FIELD_CHARS) for ep in _select_meaningful_endpoints(all_endpoints)[:6]]
|
|
963
|
+
sanitized_matched = _sanitize_endpoint_for_llm(matched_endpoint, max_chars=MAX_LLM_FIELD_CHARS)
|
|
964
|
+
|
|
965
|
+
repair_prompt = f"""\
|
|
966
|
+
Ürettiğin `registry/{service_name}/client.py` kodu Auto-Test (Validator Loop) aşamasında çalıştırıldığında HATA verdi veya geçersiz yanıt döndürdü.
|
|
967
|
+
|
|
968
|
+
Servis Adı: {service_name}
|
|
969
|
+
Hedef URL: {target_url}
|
|
970
|
+
Beklenen Ana Fonksiyon Adı: {function_name}
|
|
971
|
+
|
|
972
|
+
ALINAN HATA / BAŞARISIZ YANIT:
|
|
973
|
+
{str(error_detail)[:MAX_LLM_FIELD_CHARS]}
|
|
974
|
+
|
|
975
|
+
MEVCUT HATALI KOD (`client.py`):
|
|
976
|
+
```python
|
|
977
|
+
{broken_code[:4000]}
|
|
978
|
+
```
|
|
979
|
+
|
|
980
|
+
SEÇİLEN ORİJİNAL UÇ NOKTA KAYDI (`captured_traffic.json` - 2000 karakter truncate):
|
|
981
|
+
{json.dumps(sanitized_matched, ensure_ascii=False, indent=2)}
|
|
982
|
+
|
|
983
|
+
ALTERNATİF YAKALANAN UÇ NOKTALAR:
|
|
984
|
+
{json.dumps(filtered_endpoints, ensure_ascii=False, indent=2)}
|
|
985
|
+
|
|
986
|
+
GÖREVİN:
|
|
987
|
+
1. Hatayı analiz et (örneğin SyntaxError, eksik query_params, yanlış URL, eksik header, yanlış JSON body, argümanların varsayılan değerinin olmaması veya yanlış uç nokta seçimi).
|
|
988
|
+
2. DİKKAT: `self.request(...)` metodu HTTP Response nesnesi DEĞİL, doğrudan `dict` (sözlük) döndürür! Bu nedenle `response.text` ÇAĞIRMA; gelen yanıt zaten `dict` ise `.text` aramaya kalkmadan doğrudan `res` (sözlük) döndür veya string gerekiyorsa `res.get("text", json.dumps(res, ensure_ascii=False)) if isinstance(res, dict) else str(res)` olarak işle.
|
|
989
|
+
3. KESİN PYTHON SÖZDİZİMİ (STRICT PYTHON SYNTAX) KURALLARI:
|
|
990
|
+
- Fonksiyon imzalarında ASLA boş virgül (`def fn(self, ,):`) bırakma.
|
|
991
|
+
- Python sözlüklerinde JSON sabitleri (`true`, `false`, `null`) DEĞİL, Python sabitleri (`True`, `False`, `None`) kullan.
|
|
992
|
+
- `client_code` içine ```python çitleri koyma; doğrudan `ast.parse()` ile hatasız derlenen saf Python kodu döndür.
|
|
993
|
+
4. Tüm argümanlara yakalanan trafikten geçerli varsayılan değerler (`default_value`) ver ki `{function_name}()` parametresiz çağrıldığında doğrudan 200 OK geçerli JSON döndürsün.
|
|
994
|
+
5. `BaseOwnClient` sınıfından miras alan (`{camel_name}Client(BaseOwnClient)`) ve modül seviyesinde `{function_name}(..., tenant_id: str = "default", mode: str = "local")` fonksiyonunu tanımlayan düzeltilmiş tam Python kodunu üret.
|
|
995
|
+
|
|
996
|
+
Yanıtını SADECE aşağıdaki JSON formatında ver:
|
|
997
|
+
{{
|
|
998
|
+
"function_name": "{function_name}",
|
|
999
|
+
"fix_explanation": "<Yapılan düzeltmenin kısa Türkçe açıklaması>",
|
|
1000
|
+
"client_code": "<düzeltilmiş tam python kodu>"
|
|
1001
|
+
}}
|
|
1002
|
+
"""
|
|
1003
|
+
|
|
1004
|
+
try:
|
|
1005
|
+
resp = client.chat.completions.create(
|
|
1006
|
+
model=model,
|
|
1007
|
+
messages=[{"role": "user", "content": repair_prompt}],
|
|
1008
|
+
temperature=0.1,
|
|
1009
|
+
response_format={"type": "json_object"},
|
|
1010
|
+
)
|
|
1011
|
+
parsed = json.loads(resp.choices[0].message.content or "{}")
|
|
1012
|
+
repaired_code = _clean_and_validate_python_code(parsed.get("client_code", ""))
|
|
1013
|
+
repaired_fn = parsed.get("function_name", function_name).strip() or function_name
|
|
1014
|
+
fix_exp = parsed.get("fix_explanation", "Otomatik LLM onarımı uygulandı.")
|
|
1015
|
+
console.print(f"[bold cyan]🔧 GPT-4o Onarım Açıklaması:[/bold cyan] {fix_exp}")
|
|
1016
|
+
if repaired_code:
|
|
1017
|
+
return repaired_code, repaired_fn
|
|
1018
|
+
except Exception as exc:
|
|
1019
|
+
console.print(f"[dim yellow]⚠️ LLM onarım çağrısı başarısız ({exc}), deterministik şablona dönülüyor...[/dim yellow]")
|
|
1020
|
+
|
|
1021
|
+
fallback_code, fallback_schema = _build_fallback_code(service_name, target_url, matched_endpoint, description)
|
|
1022
|
+
return fallback_code, fallback_schema.function_name
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def _is_valid_json_response(result: Any) -> tuple[bool, str]:
|
|
1026
|
+
"""
|
|
1027
|
+
Test çağrısından dönen yanıtın geçerli ve hatasız bir JSON/veri yanıtı olup olmadığını denetler.
|
|
1028
|
+
"""
|
|
1029
|
+
if result is None:
|
|
1030
|
+
return False, "Yanıt None döndü."
|
|
1031
|
+
if isinstance(result, str):
|
|
1032
|
+
cleaned = result.strip()
|
|
1033
|
+
if not cleaned:
|
|
1034
|
+
return False, "Yanıt boş string ('') döndü."
|
|
1035
|
+
return True, f"Geçerli metin/JSON yanıtı ({len(cleaned)} karakter)"
|
|
1036
|
+
if not isinstance(result, (dict, list)):
|
|
1037
|
+
return False, f"Yanıt JSON (dict/list/str) değil: {type(result).__name__}"
|
|
1038
|
+
if isinstance(result, list):
|
|
1039
|
+
return True, f"Geçerli JSON listesi ({len(result)} öğe)"
|
|
1040
|
+
if not result:
|
|
1041
|
+
return False, "Yanıt boş sözlük ({}) döndü."
|
|
1042
|
+
|
|
1043
|
+
# SessionHealer ayrıştırma hatası veya HTTP 4xx/5xx kontrolü
|
|
1044
|
+
if result.get("error") == "Yanıt JSON formatında ayrıştırılamadı":
|
|
1045
|
+
return False, f"JSON ayrıştırma hatası (status={result.get('status_code')}): {str(result.get('text', ''))[:250]}"
|
|
1046
|
+
|
|
1047
|
+
status_code = result.get("status_code") or result.get("statusCode") or result.get("status")
|
|
1048
|
+
if isinstance(status_code, int) and status_code >= 400:
|
|
1049
|
+
return False, f"HTTP Hata Kodu ({status_code}): {json.dumps(result, ensure_ascii=False)[:300]}"
|
|
1050
|
+
|
|
1051
|
+
if result.get("success") is False and (result.get("error") or result.get("errors")):
|
|
1052
|
+
return False, f"API Başarısız Yanıtı: {json.dumps(result, ensure_ascii=False)[:300]}"
|
|
1053
|
+
|
|
1054
|
+
return True, f"Geçerli JSON nesnesi (Anahtarlar: {list(result.keys())[:6]})"
|
|
1055
|
+
|
|
1056
|
+
|
|
1057
|
+
def _load_and_call_client(
|
|
1058
|
+
client_path: Path,
|
|
1059
|
+
function_name: str,
|
|
1060
|
+
tenant_id: str = "default",
|
|
1061
|
+
mode: str = "local",
|
|
1062
|
+
) -> Any:
|
|
1063
|
+
"""Üretilen client.py modülünü dinamik yükler ve ana fonksiyonu varsayılan argümanlarla çağırır."""
|
|
1064
|
+
mod_name = f"own_api_validator_{client_path.parent.name}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:4]}"
|
|
1065
|
+
spec = importlib.util.spec_from_file_location(mod_name, str(client_path.resolve()))
|
|
1066
|
+
if spec is None or spec.loader is None:
|
|
1067
|
+
raise RuntimeError(f"{client_path} yüklenemedi.")
|
|
1068
|
+
module = importlib.util.module_from_spec(spec)
|
|
1069
|
+
sys.modules[mod_name] = module
|
|
1070
|
+
spec.loader.exec_module(module)
|
|
1071
|
+
|
|
1072
|
+
fn = getattr(module, function_name, None)
|
|
1073
|
+
if fn is None:
|
|
1074
|
+
# Modül içindeki ilk public fonksiyonu bul
|
|
1075
|
+
import inspect
|
|
1076
|
+
funcs = [
|
|
1077
|
+
obj for name, obj in inspect.getmembers(module, inspect.isfunction)
|
|
1078
|
+
if obj.__module__ == mod_name and not name.startswith("_")
|
|
1079
|
+
]
|
|
1080
|
+
if not funcs:
|
|
1081
|
+
raise AttributeError(f"`{function_name}` fonksiyonu {client_path.name} içinde bulunamadı.")
|
|
1082
|
+
fn = funcs[0]
|
|
1083
|
+
|
|
1084
|
+
import inspect
|
|
1085
|
+
sig = inspect.signature(fn)
|
|
1086
|
+
kwargs: dict[str, Any] = {}
|
|
1087
|
+
if "tenant_id" in sig.parameters:
|
|
1088
|
+
kwargs["tenant_id"] = tenant_id
|
|
1089
|
+
if "mode" in sig.parameters:
|
|
1090
|
+
kwargs["mode"] = mode
|
|
1091
|
+
|
|
1092
|
+
return fn(**kwargs)
|
|
1093
|
+
|
|
1094
|
+
|
|
1095
|
+
def validate_and_repair_client(
|
|
1096
|
+
service_name: str,
|
|
1097
|
+
target_url: str,
|
|
1098
|
+
description: str,
|
|
1099
|
+
client_path: Path,
|
|
1100
|
+
schema_obj: RegistrySchema,
|
|
1101
|
+
matched_endpoint: dict[str, Any],
|
|
1102
|
+
all_endpoints: list[dict[str, Any]],
|
|
1103
|
+
validation_runs: int = 3,
|
|
1104
|
+
max_repair_attempts: int = 3,
|
|
1105
|
+
tenant_id: str = "default",
|
|
1106
|
+
mode: str = "local",
|
|
1107
|
+
) -> dict[str, Any]:
|
|
1108
|
+
"""
|
|
1109
|
+
Compiler + Validator Loop (Auto-Test Katmanı):
|
|
1110
|
+
Üretilen `client.py` dosyasını otomatik olarak `validation_runs` (3) kez test çağrısına tabi tutar.
|
|
1111
|
+
Eğer 3 çağrının 3'ü de başarılı JSON döndürürse 'BAŞARILI' olarak kaydeder.
|
|
1112
|
+
Hata alınırsa hatayı GPT-4o'ya geri besleyip kodu düzelttirir ve 3'lü testi tekrarlar.
|
|
1113
|
+
"""
|
|
1114
|
+
console.print(
|
|
1115
|
+
Panel.fit(
|
|
1116
|
+
f"[bold cyan]🧪 Compiler + Validator Loop (Auto-Test Katmanı) Başlatıldı[/bold cyan]\n"
|
|
1117
|
+
f"[white]Servis:[/white] [magenta]{service_name}[/magenta] | "
|
|
1118
|
+
f"[white]Fonksiyon:[/white] [yellow]{schema_obj.function_name}()[/yellow]\n"
|
|
1119
|
+
f"[dim]Hedef: {validation_runs}/{validation_runs} ardışık başarılı JSON yanıtı doğrulaması[/dim]",
|
|
1120
|
+
title="OwnAPI Auto-Validator",
|
|
1121
|
+
border_style="cyan",
|
|
1122
|
+
)
|
|
1123
|
+
)
|
|
1124
|
+
|
|
1125
|
+
current_code = client_path.read_text(encoding="utf-8")
|
|
1126
|
+
current_fn_name = schema_obj.function_name
|
|
1127
|
+
repair_count = 0
|
|
1128
|
+
last_error = ""
|
|
1129
|
+
|
|
1130
|
+
for attempt in range(max_repair_attempts + 1):
|
|
1131
|
+
passed_runs = 0
|
|
1132
|
+
run_logs: list[dict[str, Any]] = []
|
|
1133
|
+
failed_reason = ""
|
|
1134
|
+
|
|
1135
|
+
for run_idx in range(1, validation_runs + 1):
|
|
1136
|
+
console.print(
|
|
1137
|
+
f"[bold yellow]▶ Auto-Test Çağrısı [{run_idx}/{validation_runs}][/bold yellow] "
|
|
1138
|
+
f"(Onarım Turu #{attempt})..."
|
|
1139
|
+
)
|
|
1140
|
+
start_t = time.perf_counter()
|
|
1141
|
+
try:
|
|
1142
|
+
res = _load_and_call_client(
|
|
1143
|
+
client_path=client_path,
|
|
1144
|
+
function_name=current_fn_name,
|
|
1145
|
+
tenant_id=tenant_id,
|
|
1146
|
+
mode=mode,
|
|
1147
|
+
)
|
|
1148
|
+
elapsed_ms = (time.perf_counter() - start_t) * 1000.0
|
|
1149
|
+
is_ok, detail = _is_valid_json_response(res)
|
|
1150
|
+
if is_ok:
|
|
1151
|
+
passed_runs += 1
|
|
1152
|
+
run_logs.append({
|
|
1153
|
+
"run": run_idx,
|
|
1154
|
+
"status": "BAŞARILI",
|
|
1155
|
+
"elapsed_ms": round(elapsed_ms, 2),
|
|
1156
|
+
"detail": detail,
|
|
1157
|
+
})
|
|
1158
|
+
console.print(
|
|
1159
|
+
f" [bold green]✔ Test {run_idx}/{validation_runs} BAŞARILI[/bold green] "
|
|
1160
|
+
f"({elapsed_ms:.0f} ms) — [dim]{detail}[/dim]"
|
|
1161
|
+
)
|
|
1162
|
+
else:
|
|
1163
|
+
failed_reason = f"Test {run_idx}/{validation_runs} geçersiz yanıt döndürdü: {detail}"
|
|
1164
|
+
run_logs.append({
|
|
1165
|
+
"run": run_idx,
|
|
1166
|
+
"status": "BAŞARISIZ",
|
|
1167
|
+
"elapsed_ms": round(elapsed_ms, 2),
|
|
1168
|
+
"detail": detail,
|
|
1169
|
+
})
|
|
1170
|
+
console.print(f" [bold red]✖ {failed_reason}[/bold red]")
|
|
1171
|
+
break
|
|
1172
|
+
except Exception as exc:
|
|
1173
|
+
elapsed_ms = (time.perf_counter() - start_t) * 1000.0
|
|
1174
|
+
tb_str = traceback.format_exc(limit=4)
|
|
1175
|
+
failed_reason = f"Test {run_idx}/{validation_runs} çalışma zamanı hatası: {exc}\n{tb_str}"
|
|
1176
|
+
run_logs.append({
|
|
1177
|
+
"run": run_idx,
|
|
1178
|
+
"status": "HATA",
|
|
1179
|
+
"elapsed_ms": round(elapsed_ms, 2),
|
|
1180
|
+
"detail": str(exc),
|
|
1181
|
+
})
|
|
1182
|
+
console.print(f" [bold red]✖ Test {run_idx}/{validation_runs} HATA:[/bold red] {exc}")
|
|
1183
|
+
break
|
|
1184
|
+
|
|
1185
|
+
if passed_runs == validation_runs:
|
|
1186
|
+
table = Table(
|
|
1187
|
+
title=f"✅ OwnAPI Auto-Test Doğrulama Raporu ({service_name} — {validation_runs}/{validation_runs} BAŞARILI)",
|
|
1188
|
+
show_header=True,
|
|
1189
|
+
header_style="bold green",
|
|
1190
|
+
show_lines=True,
|
|
1191
|
+
)
|
|
1192
|
+
table.add_column("Test Çağrısı", justify="center", style="bold cyan")
|
|
1193
|
+
table.add_column("Durum", justify="center", style="bold green")
|
|
1194
|
+
table.add_column("Süre (ms)", justify="right", style="yellow")
|
|
1195
|
+
table.add_column("JSON Doğrulama Özeti", style="white")
|
|
1196
|
+
for rlog in run_logs:
|
|
1197
|
+
table.add_row(
|
|
1198
|
+
f"#{rlog['run']}/{validation_runs}",
|
|
1199
|
+
f"🟢 {rlog['status']}",
|
|
1200
|
+
f"{rlog['elapsed_ms']:.1f} ms",
|
|
1201
|
+
rlog["detail"],
|
|
1202
|
+
)
|
|
1203
|
+
console.print()
|
|
1204
|
+
console.print(table)
|
|
1205
|
+
console.print()
|
|
1206
|
+
return {
|
|
1207
|
+
"validation_status": "BAŞARILI",
|
|
1208
|
+
"passed_runs": passed_runs,
|
|
1209
|
+
"required_runs": validation_runs,
|
|
1210
|
+
"repair_attempts": repair_count,
|
|
1211
|
+
"function_name": current_fn_name,
|
|
1212
|
+
"runs": run_logs,
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
last_error = failed_reason
|
|
1216
|
+
if attempt < max_repair_attempts:
|
|
1217
|
+
repair_count += 1
|
|
1218
|
+
console.print(
|
|
1219
|
+
f"\n[bold yellow]🔄 Auto-Repair #{repair_count}/{max_repair_attempts}: "
|
|
1220
|
+
f"Hata GPT-4o'ya geri besleniyor ve `client.py` yeniden derleniyor...[/bold yellow]"
|
|
1221
|
+
)
|
|
1222
|
+
repaired_code, repaired_fn = _repair_client_with_llm(
|
|
1223
|
+
service_name=service_name,
|
|
1224
|
+
target_url=target_url,
|
|
1225
|
+
description=description,
|
|
1226
|
+
matched_endpoint=matched_endpoint,
|
|
1227
|
+
all_endpoints=all_endpoints,
|
|
1228
|
+
broken_code=current_code,
|
|
1229
|
+
function_name=current_fn_name,
|
|
1230
|
+
error_detail=last_error,
|
|
1231
|
+
)
|
|
1232
|
+
current_code = repaired_code
|
|
1233
|
+
current_fn_name = repaired_fn
|
|
1234
|
+
schema_obj.function_name = current_fn_name
|
|
1235
|
+
client_path.write_text(current_code.strip() + "\n", encoding="utf-8")
|
|
1236
|
+
|
|
1237
|
+
console.print(
|
|
1238
|
+
f"[bold red]⚠️ Auto-Test Katmanı {max_repair_attempts} onarım denemesine rağmen "
|
|
1239
|
+
f"{validation_runs}/{validation_runs} doğrulamayı tamamlayamadı.[/bold red]"
|
|
1240
|
+
)
|
|
1241
|
+
return {
|
|
1242
|
+
"validation_status": "KISMİ_VEYA_HATALI",
|
|
1243
|
+
"passed_runs": 0,
|
|
1244
|
+
"required_runs": validation_runs,
|
|
1245
|
+
"repair_attempts": repair_count,
|
|
1246
|
+
"function_name": current_fn_name,
|
|
1247
|
+
"last_error": last_error,
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
|
|
1251
|
+
def start_url_to_domain(url: str) -> str:
|
|
1252
|
+
try:
|
|
1253
|
+
return urlparse(url).netloc or url
|
|
1254
|
+
except Exception:
|
|
1255
|
+
return url
|
|
1256
|
+
|
|
1257
|
+
|
|
1258
|
+
def onboard_new_site(
|
|
1259
|
+
service_name: str,
|
|
1260
|
+
start_url: str,
|
|
1261
|
+
description: str = "",
|
|
1262
|
+
headless: bool = False,
|
|
1263
|
+
registry_root: Path | str = PROJECT_ROOT / "registry",
|
|
1264
|
+
tenant_id: str = "default",
|
|
1265
|
+
mode: str = "local",
|
|
1266
|
+
auto_validate: bool = True,
|
|
1267
|
+
validation_runs: int = 3,
|
|
1268
|
+
interactive_tos: bool = False,
|
|
1269
|
+
wait_seconds: int | None = None,
|
|
1270
|
+
) -> dict[str, Any]:
|
|
1271
|
+
"""
|
|
1272
|
+
Kullanıcı kod yazmadan yeni bir siteyi OwnAPI mimarisine ekler:
|
|
1273
|
+
1. ToS / Kullanıcı Delegasyon Onay Mekanizmasını (`ConsentManager`) çalıştırır ve loglar.
|
|
1274
|
+
2. Temiz Chrome açıp trafiği ve oturum çerezlerini dinler.
|
|
1275
|
+
3. `registry/{service_name}/captured_traffic.json` dosyasına kaydeder.
|
|
1276
|
+
4. OpenAI GPT-4o (veya fallback şablonu) ile `BaseOwnClient` uyumlu `client.py`,
|
|
1277
|
+
`schema.json` ve `__init__.py` dosyalarını derler.
|
|
1278
|
+
5. Compiler + Validator Loop (Auto-Test Katmanı): Üretilen `client.py` dosyasını otomatik
|
|
1279
|
+
olarak 3 kez test çağrısına tabi tutar; hata alınırsa GPT-4o'ya hatayı geri verip
|
|
1280
|
+
kodu düzelttirir ve 3/3 başarılı JSON alındığında 'BAŞARILI' olarak kaydeder.
|
|
1281
|
+
"""
|
|
1282
|
+
reg_root = Path(registry_root)
|
|
1283
|
+
clean_name = _sanitize_service_name(service_name)
|
|
1284
|
+
service_dir = reg_root / clean_name
|
|
1285
|
+
service_dir.mkdir(parents=True, exist_ok=True)
|
|
1286
|
+
|
|
1287
|
+
traffic_path = service_dir / "captured_traffic.json"
|
|
1288
|
+
client_path = service_dir / "client.py"
|
|
1289
|
+
schema_path = service_dir / "schema.json"
|
|
1290
|
+
init_path = service_dir / "__init__.py"
|
|
1291
|
+
|
|
1292
|
+
# 1. ToS / Kullanıcı Delegasyon Onayı
|
|
1293
|
+
consent_mgr = ConsentManager()
|
|
1294
|
+
consent_record = consent_mgr.ensure_consent(
|
|
1295
|
+
tenant_id=tenant_id,
|
|
1296
|
+
service=clean_name,
|
|
1297
|
+
action="onboarding",
|
|
1298
|
+
mode=mode,
|
|
1299
|
+
interactive=interactive_tos,
|
|
1300
|
+
force_banner=True,
|
|
1301
|
+
)
|
|
1302
|
+
|
|
1303
|
+
console.print(
|
|
1304
|
+
Panel.fit(
|
|
1305
|
+
f"[bold cyan]🌟 Yeni Servis Onboarding Başlatıldı:[/bold cyan] [white]{clean_name}[/white]\n"
|
|
1306
|
+
f"[dim]Hedef URL:[/dim] [green]{start_url}[/green]\n"
|
|
1307
|
+
f"[dim]Açıklama:[/dim] [yellow]{description or 'Belirtilmedi'}[/yellow]\n"
|
|
1308
|
+
f"[dim]Çalışma Modu:[/dim] [cyan]{mode.upper()}[/cyan]\n"
|
|
1309
|
+
f"[dim]Hedef Dizin:[/dim] [magenta]{service_dir}[/magenta]",
|
|
1310
|
+
title="OwnAPI Onboarding Engine",
|
|
1311
|
+
border_style="cyan",
|
|
1312
|
+
)
|
|
1313
|
+
)
|
|
1314
|
+
|
|
1315
|
+
# 2. Trafiği Kaydet (Eğer zaten captured_traffic.json yoksa veya boşsa tarayıcı aç)
|
|
1316
|
+
try:
|
|
1317
|
+
traffic_data = asyncio.run(
|
|
1318
|
+
_record_traffic_async(
|
|
1319
|
+
start_url=start_url,
|
|
1320
|
+
output_path=traffic_path,
|
|
1321
|
+
service_name=clean_name,
|
|
1322
|
+
tenant_id=tenant_id,
|
|
1323
|
+
headless=headless,
|
|
1324
|
+
wait_seconds=wait_seconds,
|
|
1325
|
+
)
|
|
1326
|
+
)
|
|
1327
|
+
except RuntimeError:
|
|
1328
|
+
import concurrent.futures
|
|
1329
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
|
1330
|
+
traffic_data = pool.submit(
|
|
1331
|
+
asyncio.run,
|
|
1332
|
+
_record_traffic_async(
|
|
1333
|
+
start_url=start_url,
|
|
1334
|
+
output_path=traffic_path,
|
|
1335
|
+
service_name=clean_name,
|
|
1336
|
+
tenant_id=tenant_id,
|
|
1337
|
+
headless=headless,
|
|
1338
|
+
wait_seconds=wait_seconds,
|
|
1339
|
+
),
|
|
1340
|
+
).result()
|
|
1341
|
+
|
|
1342
|
+
endpoints = traffic_data.get("endpoints", [])
|
|
1343
|
+
if not endpoints:
|
|
1344
|
+
console.print("[bold red]❌ Ağ trafiği yakalanamadı veya oturum erken sonlandırıldı.[/bold red]")
|
|
1345
|
+
return {
|
|
1346
|
+
"success": False,
|
|
1347
|
+
"error": "Ağ trafiği yakalanamadı.",
|
|
1348
|
+
"service_name": clean_name,
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
# 3. LLM veya Fallback ile Derle
|
|
1352
|
+
console.print("\n[bold yellow]🧠 Yakalanan API uç noktaları analiz ediliyor ve kod derleniyor...[/bold yellow]")
|
|
1353
|
+
client_code = ""
|
|
1354
|
+
schema_obj: RegistrySchema | None = None
|
|
1355
|
+
matched_ep: dict[str, Any] = {}
|
|
1356
|
+
|
|
1357
|
+
try:
|
|
1358
|
+
client_code, schema_obj, matched_ep = _compile_with_llm(clean_name, start_url, description, endpoints)
|
|
1359
|
+
console.print("[bold green]✔ OpenAI GPT-4o ile akıllı derleme tamamlandı![/bold green]")
|
|
1360
|
+
except Exception as exc:
|
|
1361
|
+
console.print(f"[dim yellow]⚠️ OpenAI API çağrısı atlandı/hata verdi ({exc}), standart BaseOwnClient şablonu kullanılıyor...[/dim yellow]")
|
|
1362
|
+
meaningful = _select_meaningful_endpoints(endpoints)
|
|
1363
|
+
matched_ep = meaningful[0] if meaningful else endpoints[0]
|
|
1364
|
+
client_code, schema_obj = _build_fallback_code(clean_name, start_url, matched_ep, description)
|
|
1365
|
+
|
|
1366
|
+
# 4. Dosyaları Diske Yaz
|
|
1367
|
+
client_path.write_text(client_code.strip() + "\n", encoding="utf-8")
|
|
1368
|
+
|
|
1369
|
+
# 5. Compiler + Validator Loop (3x Auto-Test & GPT-4o Self-Repair)
|
|
1370
|
+
validation_report: dict[str, Any] = {
|
|
1371
|
+
"validation_status": "ATLANDI",
|
|
1372
|
+
"passed_runs": 0,
|
|
1373
|
+
"required_runs": validation_runs,
|
|
1374
|
+
}
|
|
1375
|
+
if auto_validate:
|
|
1376
|
+
validation_report = validate_and_repair_client(
|
|
1377
|
+
service_name=clean_name,
|
|
1378
|
+
target_url=start_url,
|
|
1379
|
+
description=description,
|
|
1380
|
+
client_path=client_path,
|
|
1381
|
+
schema_obj=schema_obj,
|
|
1382
|
+
matched_endpoint=matched_ep,
|
|
1383
|
+
all_endpoints=endpoints,
|
|
1384
|
+
validation_runs=validation_runs,
|
|
1385
|
+
max_repair_attempts=3,
|
|
1386
|
+
tenant_id=tenant_id,
|
|
1387
|
+
mode=mode,
|
|
1388
|
+
)
|
|
1389
|
+
|
|
1390
|
+
# Şema JSON dosyasına doğrulama durumunu da ekle
|
|
1391
|
+
schema_dict = schema_obj.model_dump()
|
|
1392
|
+
schema_dict["validation_status"] = validation_report.get("validation_status", "BAŞARILI")
|
|
1393
|
+
schema_dict["validation_passed_runs"] = validation_report.get("passed_runs", validation_runs)
|
|
1394
|
+
schema_path.write_text(json.dumps(schema_dict, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
1395
|
+
|
|
1396
|
+
camel_name = _to_camel_case(clean_name)
|
|
1397
|
+
init_content = f'''"""OwnAPI Registry Module: {clean_name}."""
|
|
1398
|
+
|
|
1399
|
+
from .client import {schema_obj.function_name}, {camel_name}Client
|
|
1400
|
+
|
|
1401
|
+
__all__ = ["{schema_obj.function_name}", "{camel_name}Client"]
|
|
1402
|
+
'''
|
|
1403
|
+
init_path.write_text(init_content, encoding="utf-8")
|
|
1404
|
+
|
|
1405
|
+
val_badge = (
|
|
1406
|
+
f"[bold green]🟢 {validation_report['validation_status']} "
|
|
1407
|
+
f"({validation_report.get('passed_runs', 0)}/{validation_runs} Test Başarılı)[/bold green]"
|
|
1408
|
+
if validation_report.get("validation_status") == "BAŞARILI"
|
|
1409
|
+
else f"[bold yellow]🟡 {validation_report.get('validation_status')}[/bold yellow]"
|
|
1410
|
+
)
|
|
1411
|
+
|
|
1412
|
+
console.print(
|
|
1413
|
+
Panel.fit(
|
|
1414
|
+
f"[bold green]✨ Servis Başarıyla Derlendi, Doğrulandı ve Kaydedildi![/bold green]\n"
|
|
1415
|
+
f"[white]Servis Adı:[/white] [cyan]{clean_name}[/cyan]\n"
|
|
1416
|
+
f"[white]Fonksiyon:[/white] [green]{schema_obj.function_name}[/green]\n"
|
|
1417
|
+
f"[white]Endpoint:[/white] [yellow]{schema_obj.method} {schema_obj.endpoint_url}[/yellow]\n"
|
|
1418
|
+
f"[white]Auto-Test Durumu:[/white] {val_badge}\n"
|
|
1419
|
+
f"[white]Parametreler:[/white] {', '.join(p.name for p in schema_obj.dynamic_params) or 'Yok'}\n\n"
|
|
1420
|
+
f"[dim]Oluşturulan Dosyalar:[/dim]\n"
|
|
1421
|
+
f" • {client_path.name}\n"
|
|
1422
|
+
f" • {schema_path.name}\n"
|
|
1423
|
+
f" • {init_path.name}\n"
|
|
1424
|
+
f" • {traffic_path.name}",
|
|
1425
|
+
title="OwnAPI Onboarding Tamamlandı",
|
|
1426
|
+
border_style="green",
|
|
1427
|
+
)
|
|
1428
|
+
)
|
|
1429
|
+
|
|
1430
|
+
return {
|
|
1431
|
+
"success": True,
|
|
1432
|
+
"service_name": clean_name,
|
|
1433
|
+
"function_name": schema_obj.function_name,
|
|
1434
|
+
"endpoint_url": schema_obj.endpoint_url,
|
|
1435
|
+
"method": schema_obj.method,
|
|
1436
|
+
"validation_status": validation_report.get("validation_status"),
|
|
1437
|
+
"validation_report": validation_report,
|
|
1438
|
+
"consent_record": consent_record,
|
|
1439
|
+
"dynamic_params": [p.model_dump() for p in schema_obj.dynamic_params],
|
|
1440
|
+
"files_created": [
|
|
1441
|
+
str(traffic_path),
|
|
1442
|
+
str(client_path),
|
|
1443
|
+
str(schema_path),
|
|
1444
|
+
str(init_path),
|
|
1445
|
+
],
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
if __name__ == "__main__":
|
|
1450
|
+
import argparse
|
|
1451
|
+
parser = argparse.ArgumentParser(description="OwnAPI Otomatik Site Ekleme (Onboarding + Auto-Test)")
|
|
1452
|
+
parser.add_argument("service_name", help="Eklenecek servis adı (örn: biletix)")
|
|
1453
|
+
parser.add_argument("start_url", help="Giriş sayfası URL'si")
|
|
1454
|
+
parser.add_argument("--desc", default="", help="İsteğe bağlı servis açıklaması")
|
|
1455
|
+
parser.add_argument("--mode", default="local", choices=["local", "cloud"], help="Çalışma modu (local veya cloud)")
|
|
1456
|
+
args = parser.parse_args()
|
|
1457
|
+
onboard_new_site(args.service_name, args.start_url, args.desc, mode=args.mode)
|