ownapi 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli.py +1017 -0
- compiler.py +948 -0
- core/__init__.py +19 -0
- core/api_keys.py +523 -0
- core/auto_repair.py +682 -0
- core/base_client.py +165 -0
- core/consent.py +169 -0
- core/form_automation.py +1044 -0
- core/hitl_queue.py +886 -0
- core/onboarding.py +1457 -0
- core/token_store.py +223 -0
- healer.py +2074 -0
- master_mcp.py +193 -0
- ownapi/__init__.py +94 -0
- ownapi/cli.py +309 -0
- ownapi/sdk.py +436 -0
- ownapi-1.0.0.dist-info/METADATA +287 -0
- ownapi-1.0.0.dist-info/RECORD +62 -0
- ownapi-1.0.0.dist-info/WHEEL +5 -0
- ownapi-1.0.0.dist-info/entry_points.txt +4 -0
- ownapi-1.0.0.dist-info/top_level.txt +10 -0
- recorder.py +486 -0
- registry/coingecko/__init__.py +5 -0
- registry/coingecko/client.py +84 -0
- registry/coingecko/schema.json +23 -0
- registry/edevlet/__init__.py +5 -0
- registry/edevlet/client.py +54 -0
- registry/edevlet/schema.json +14 -0
- registry/enuygun/__init__.py +5 -0
- registry/enuygun/client.py +78 -0
- registry/enuygun/schema.json +13 -0
- registry/github_trending/__init__.py +5 -0
- registry/github_trending/client.py +90 -0
- registry/github_trending/schema.json +31 -0
- registry/hackernews/__init__.py +5 -0
- registry/hackernews/client.py +90 -0
- registry/hackernews/schema.json +38 -0
- registry/hepsiburada/__init__.py +5 -0
- registry/hepsiburada/client.py +32 -0
- registry/hepsiburada/schema.json +23 -0
- registry/linkedin_jobs/__init__.py +5 -0
- registry/linkedin_jobs/client.py +241 -0
- registry/linkedin_jobs/schema.json +46 -0
- registry/pegasus/__init__.py +5 -0
- registry/pegasus/client.py +141 -0
- registry/pegasus/schema.json +76 -0
- registry/thy/__init__.py +5 -0
- registry/thy/client.py +1796 -0
- registry/thy/schema.json +46 -0
- registry/trendyol/__init__.py +5 -0
- registry/trendyol/client.py +228 -0
- registry/trendyol/schema.json +30 -0
- registry/ucak_bileti/client.py +94 -0
- registry/ucak_bileti/schema.json +47 -0
- registry/wikipedia/__init__.py +5 -0
- registry/wikipedia/client.py +64 -0
- registry/wikipedia/schema.json +31 -0
- registry/yahoo_finance/__init__.py +5 -0
- registry/yahoo_finance/client.py +98 -0
- registry/yahoo_finance/schema.json +22 -0
- server.py +891 -0
- static/index.html +2401 -0
recorder.py
ADDED
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
from urllib.parse import parse_qs, urlparse
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
from playwright.async_api import Page, Request, Response, async_playwright
|
|
12
|
+
from pydantic import BaseModel, Field
|
|
13
|
+
from rich.console import Console
|
|
14
|
+
from rich.panel import Panel
|
|
15
|
+
from rich.table import Table
|
|
16
|
+
|
|
17
|
+
console = Console(stderr=True, legacy_windows=False)
|
|
18
|
+
app = typer.Typer(help="Shadow-API Compiler - AÄŸ TrafiÄŸi Kaydedici (Recorder)")
|
|
19
|
+
|
|
20
|
+
# Filtrelenecek telemetri, analitik ve reklam domain parçaları
|
|
21
|
+
IGNORED_DOMAIN_KEYWORDS: tuple[str, ...] = (
|
|
22
|
+
"google-analytics.com",
|
|
23
|
+
"googletagmanager.com",
|
|
24
|
+
"doubleclick.net",
|
|
25
|
+
"googlesyndication.com",
|
|
26
|
+
"googleadservices.com",
|
|
27
|
+
"sentry.io",
|
|
28
|
+
"clarity.ms",
|
|
29
|
+
"hotjar.com",
|
|
30
|
+
"mixpanel.com",
|
|
31
|
+
"segment.io",
|
|
32
|
+
"segment.com",
|
|
33
|
+
"amplitude.com",
|
|
34
|
+
"datadoghq.com",
|
|
35
|
+
"newrelic.com",
|
|
36
|
+
"facebook.net",
|
|
37
|
+
"connect.facebook.net",
|
|
38
|
+
"analytics",
|
|
39
|
+
"telemetry",
|
|
40
|
+
"metrics",
|
|
41
|
+
"beacon",
|
|
42
|
+
"pixel",
|
|
43
|
+
"bugsnag.com",
|
|
44
|
+
"fullstory.com",
|
|
45
|
+
"intercom.io",
|
|
46
|
+
"crisp.chat",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# Yakalanması kritik olan kimlik doğrulama, oturum ve tarayıcı parmak izi (fingerprint) başlıkları
|
|
50
|
+
RELEVANT_HEADER_PREFIXES: tuple[str, ...] = (
|
|
51
|
+
"authorization",
|
|
52
|
+
"cookie",
|
|
53
|
+
"x-csrf",
|
|
54
|
+
"x-xsrf",
|
|
55
|
+
"csrf",
|
|
56
|
+
"xsrf",
|
|
57
|
+
"x-api-key",
|
|
58
|
+
"api-key",
|
|
59
|
+
"x-algolia",
|
|
60
|
+
"x-auth",
|
|
61
|
+
"x-session",
|
|
62
|
+
"x-token",
|
|
63
|
+
"x-requested-with",
|
|
64
|
+
"content-type",
|
|
65
|
+
"accept",
|
|
66
|
+
"user-agent",
|
|
67
|
+
"referer",
|
|
68
|
+
"origin",
|
|
69
|
+
"sec-ch-ua",
|
|
70
|
+
"sec-ch-ua-mobile",
|
|
71
|
+
"sec-ch-ua-platform",
|
|
72
|
+
"sec-fetch-dest",
|
|
73
|
+
"sec-fetch-mode",
|
|
74
|
+
"sec-fetch-site",
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class CapturedEndpoint(BaseModel):
|
|
79
|
+
"""Yakalanan tekil bir XHR/Fetch API çağrısının veri modeli."""
|
|
80
|
+
|
|
81
|
+
id: int = Field(..., description="Yakalanma sırası")
|
|
82
|
+
timestamp: str = Field(..., description="UTC ISO-8601 zaman damgası")
|
|
83
|
+
method: str = Field(..., description="HTTP Metodu (GET, POST, PUT, DELETE vb.)")
|
|
84
|
+
url: str = Field(..., description="Tam istek URL'i (sorgu parametreleri hariç baz URL)")
|
|
85
|
+
full_url: str = Field(..., description="Sorgu parametreleriyle birlikte tam URL")
|
|
86
|
+
domain: str = Field(..., description="Hedef sunucu domain adı")
|
|
87
|
+
path: str = Field(..., description="URL yolu (endpoint path)")
|
|
88
|
+
status_code: int = Field(..., description="HTTP yanıt durum kodu")
|
|
89
|
+
resource_type: str = Field(..., description="Playwright kaynak tipi (xhr veya fetch)")
|
|
90
|
+
query_params: dict[str, Any] = Field(default_factory=dict, description="Ayrıştırılmış URL sorgu parametreleri")
|
|
91
|
+
headers: dict[str, str] = Field(default_factory=dict, description="Filtrelenmiş kritik HTTP başlıkları")
|
|
92
|
+
request_payload: Any | None = Field(default=None, description="Varsa gönderilen JSON veya metin gövdesi")
|
|
93
|
+
response_sample: str = Field(..., description="Yanıt JSON örneği (maksimum 2000 karakter)")
|
|
94
|
+
response_truncated: bool = Field(default=False, description="Yanıtın 2000 karakterde kesilip kesilmediği")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class CaptureSession(BaseModel):
|
|
98
|
+
"""Tüm kayıt oturumunu temsil eden kök şema."""
|
|
99
|
+
|
|
100
|
+
target_url: str
|
|
101
|
+
captured_at: str
|
|
102
|
+
total_captured: int
|
|
103
|
+
endpoints: list[CapturedEndpoint]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class TrafficRecorder:
|
|
107
|
+
"""Playwright üzerinden XHR/Fetch trafiğini dinleyen ve filtreleyen sınıf."""
|
|
108
|
+
|
|
109
|
+
def __init__(
|
|
110
|
+
self,
|
|
111
|
+
target_url: str,
|
|
112
|
+
output_path: Path = Path("captured_traffic.json"),
|
|
113
|
+
max_response_chars: int = 2000,
|
|
114
|
+
profile_dir: Path = Path(".browser_profile"),
|
|
115
|
+
user_wait_sec: int = 5,
|
|
116
|
+
) -> None:
|
|
117
|
+
self.target_url = target_url
|
|
118
|
+
self.output_path = output_path
|
|
119
|
+
self.max_response_chars = max_response_chars
|
|
120
|
+
self.profile_dir = profile_dir
|
|
121
|
+
self.user_wait_sec = user_wait_sec
|
|
122
|
+
self.captured_endpoints: list[CapturedEndpoint] = []
|
|
123
|
+
self._counter: int = 0
|
|
124
|
+
self._lock = asyncio.Lock()
|
|
125
|
+
|
|
126
|
+
@staticmethod
|
|
127
|
+
def _is_ignored_domain(url: str) -> bool:
|
|
128
|
+
"""URL'in analitik, reklam veya telemetri servislerine ait olup olmadığını kontrol eder."""
|
|
129
|
+
try:
|
|
130
|
+
parsed = urlparse(url)
|
|
131
|
+
hostname = (parsed.hostname or "").lower()
|
|
132
|
+
path = (parsed.path or "").lower()
|
|
133
|
+
if not hostname:
|
|
134
|
+
return True
|
|
135
|
+
|
|
136
|
+
for keyword in IGNORED_DOMAIN_KEYWORDS:
|
|
137
|
+
if keyword in hostname or f"/{keyword}/" in path:
|
|
138
|
+
return True
|
|
139
|
+
return False
|
|
140
|
+
except Exception:
|
|
141
|
+
return True
|
|
142
|
+
|
|
143
|
+
@staticmethod
|
|
144
|
+
def _extract_relevant_headers(headers: dict[str, str]) -> dict[str, str]:
|
|
145
|
+
"""Sadece kimlik doğrulama, CSRF, Cookie ve içerik tipiyle ilgili başlıkları süzer."""
|
|
146
|
+
filtered: dict[str, str] = {}
|
|
147
|
+
for key, value in headers.items():
|
|
148
|
+
lower_key = key.lower()
|
|
149
|
+
if any(lower_key == prefix or lower_key.startswith(prefix) for prefix in RELEVANT_HEADER_PREFIXES):
|
|
150
|
+
filtered[key] = value
|
|
151
|
+
return filtered
|
|
152
|
+
|
|
153
|
+
@staticmethod
|
|
154
|
+
def _parse_query_params(url: str) -> dict[str, Any]:
|
|
155
|
+
"""URL içindeki query string parametrelerini temiz bir sözlüğe dönüştürür."""
|
|
156
|
+
parsed = urlparse(url)
|
|
157
|
+
raw_params = parse_qs(parsed.query, keep_blank_values=True)
|
|
158
|
+
# Tek elemanlı listeleri doğrudan değere indirge
|
|
159
|
+
return {k: v[0] if len(v) == 1 else v for k, v in raw_params.items()}
|
|
160
|
+
|
|
161
|
+
@staticmethod
|
|
162
|
+
def _extract_request_payload(request: Request) -> Any | None:
|
|
163
|
+
"""İstek gövdesini (varsa) JSON olarak, değilse düz metin olarak çıkarır."""
|
|
164
|
+
try:
|
|
165
|
+
post_data = request.post_data
|
|
166
|
+
if not post_data:
|
|
167
|
+
return None
|
|
168
|
+
try:
|
|
169
|
+
return json.loads(post_data)
|
|
170
|
+
except json.JSONDecodeError:
|
|
171
|
+
return post_data[:2000]
|
|
172
|
+
except Exception:
|
|
173
|
+
return None
|
|
174
|
+
|
|
175
|
+
async def _handle_response(self, response: Response) -> None:
|
|
176
|
+
"""Her ağ yanıtı geldiğinde tetiklenen asenkron dinleyici."""
|
|
177
|
+
try:
|
|
178
|
+
request = response.request
|
|
179
|
+
|
|
180
|
+
# 1. Sadece XHR ve Fetch isteklerini filtrele
|
|
181
|
+
if request.resource_type not in ("xhr", "fetch"):
|
|
182
|
+
return
|
|
183
|
+
|
|
184
|
+
full_url = request.url
|
|
185
|
+
|
|
186
|
+
# 2. Telemetri, reklam ve analitik domainlerini atla
|
|
187
|
+
if self._is_ignored_domain(full_url):
|
|
188
|
+
return
|
|
189
|
+
|
|
190
|
+
# 3. Yanıt Content-Type başlığının JSON olduğundan emin ol
|
|
191
|
+
content_type = response.headers.get("content-type", "").lower()
|
|
192
|
+
if "application/json" not in content_type and "+json" not in content_type:
|
|
193
|
+
return
|
|
194
|
+
|
|
195
|
+
# 4. Yanıt gövdesini güvenli şekilde oku ve geçerli bir JSON olduğunu doğrula
|
|
196
|
+
try:
|
|
197
|
+
raw_text = await response.text()
|
|
198
|
+
except Exception:
|
|
199
|
+
# Yönlendirmeler veya tarayıcı kapanırken yarım kalan istekler hata fırlatabilir
|
|
200
|
+
return
|
|
201
|
+
|
|
202
|
+
if not raw_text or not raw_text.strip():
|
|
203
|
+
return
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
parsed_json = json.loads(raw_text)
|
|
207
|
+
# Okunabilir/kompakt formatta yeniden serileÅŸtir
|
|
208
|
+
normalized_json_str = json.dumps(parsed_json, ensure_ascii=False)
|
|
209
|
+
except json.JSONDecodeError:
|
|
210
|
+
return
|
|
211
|
+
|
|
212
|
+
truncated = len(normalized_json_str) > self.max_response_chars
|
|
213
|
+
response_sample = normalized_json_str[: self.max_response_chars]
|
|
214
|
+
|
|
215
|
+
# İstek başlıklarını al (all_headers Cookie gibi hassas başlıkları da içerir)
|
|
216
|
+
try:
|
|
217
|
+
all_req_headers = await request.all_headers()
|
|
218
|
+
except Exception:
|
|
219
|
+
all_req_headers = request.headers
|
|
220
|
+
|
|
221
|
+
parsed_url = urlparse(full_url)
|
|
222
|
+
base_url = f"{parsed_url.scheme}://{parsed_url.netloc}{parsed_url.path}"
|
|
223
|
+
|
|
224
|
+
async with self._lock:
|
|
225
|
+
self._counter += 1
|
|
226
|
+
endpoint = CapturedEndpoint(
|
|
227
|
+
id=self._counter,
|
|
228
|
+
timestamp=datetime.now(timezone.utc).isoformat(),
|
|
229
|
+
method=request.method.upper(),
|
|
230
|
+
url=base_url,
|
|
231
|
+
full_url=full_url,
|
|
232
|
+
domain=parsed_url.netloc,
|
|
233
|
+
path=parsed_url.path or "/",
|
|
234
|
+
status_code=response.status,
|
|
235
|
+
resource_type=request.resource_type,
|
|
236
|
+
query_params=self._parse_query_params(full_url),
|
|
237
|
+
headers=self._extract_relevant_headers(all_req_headers),
|
|
238
|
+
request_payload=self._extract_request_payload(request),
|
|
239
|
+
response_sample=response_sample,
|
|
240
|
+
response_truncated=truncated,
|
|
241
|
+
)
|
|
242
|
+
self.captured_endpoints.append(endpoint)
|
|
243
|
+
|
|
244
|
+
console.print(
|
|
245
|
+
f"[bold green]✔ Yakalandı #{endpoint.id}[/bold green] "
|
|
246
|
+
f"[cyan]{endpoint.method}[/cyan] [white]{endpoint.url}[/white] "
|
|
247
|
+
f"([yellow]{endpoint.status_code}[/yellow])"
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
except Exception as exc:
|
|
251
|
+
console.print(f"[dim red]! İstek işlenirken hata oluştu: {exc}[/dim red]")
|
|
252
|
+
|
|
253
|
+
def save_to_disk(self) -> None:
|
|
254
|
+
"""Yakalanan tüm trafiği JSON şeması halinde diske yazar ve taze oturum başlıklarını session_tokens.json'a aktarır."""
|
|
255
|
+
now_iso = datetime.now(timezone.utc).isoformat()
|
|
256
|
+
session_data = CaptureSession(
|
|
257
|
+
target_url=self.target_url,
|
|
258
|
+
captured_at=now_iso,
|
|
259
|
+
total_captured=len(self.captured_endpoints),
|
|
260
|
+
endpoints=self.captured_endpoints,
|
|
261
|
+
)
|
|
262
|
+
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
263
|
+
self.output_path.write_text(
|
|
264
|
+
session_data.model_dump_json(indent=2),
|
|
265
|
+
encoding="utf-8",
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
# En son yakalanan uç noktanın dinamik başlıklarını session_tokens.json'a kaydet
|
|
269
|
+
if self.captured_endpoints:
|
|
270
|
+
latest_ep = self.captured_endpoints[-1]
|
|
271
|
+
merged_headers: dict[str, str] = {}
|
|
272
|
+
for ep in self.captured_endpoints:
|
|
273
|
+
for hk, hv in ep.headers.items():
|
|
274
|
+
if hk.lower() not in ("accept-encoding", "accept-language", "content-length"):
|
|
275
|
+
merged_headers[hk.lower()] = hv
|
|
276
|
+
merged_body_tokens: dict[str, Any] = {}
|
|
277
|
+
for ep in self.captured_endpoints:
|
|
278
|
+
if isinstance(ep.request_payload, dict):
|
|
279
|
+
merged_body_tokens.update(ep.request_payload)
|
|
280
|
+
|
|
281
|
+
token_bundle = {
|
|
282
|
+
"target_url": self.target_url,
|
|
283
|
+
"healed_at": now_iso,
|
|
284
|
+
"headers": merged_headers,
|
|
285
|
+
"query_params": latest_ep.query_params,
|
|
286
|
+
"body_tokens": merged_body_tokens,
|
|
287
|
+
**merged_headers,
|
|
288
|
+
}
|
|
289
|
+
Path("session_tokens.json").write_text(
|
|
290
|
+
json.dumps(token_bundle, ensure_ascii=False, indent=2),
|
|
291
|
+
encoding="utf-8",
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def print_summary_table(self) -> None:
|
|
295
|
+
"""Yakalanan uç noktaların özetini Rich tablosu olarak ekrana basar."""
|
|
296
|
+
if not self.captured_endpoints:
|
|
297
|
+
console.print(
|
|
298
|
+
Panel(
|
|
299
|
+
"[bold yellow]Hiçbir geçerli XHR/Fetch JSON isteği yakalanmadı.[/bold yellow]",
|
|
300
|
+
title="Shadow-API Özeti",
|
|
301
|
+
border_style="yellow",
|
|
302
|
+
)
|
|
303
|
+
)
|
|
304
|
+
return
|
|
305
|
+
|
|
306
|
+
table = Table(
|
|
307
|
+
title=f"Shadow-API Yakalanan Uç Noktalar ({len(self.captured_endpoints)} Adet)",
|
|
308
|
+
show_header=True,
|
|
309
|
+
header_style="bold magenta",
|
|
310
|
+
show_lines=True,
|
|
311
|
+
)
|
|
312
|
+
table.add_column("#", style="dim", width=4, justify="right")
|
|
313
|
+
table.add_column("Metot", style="bold cyan", width=8)
|
|
314
|
+
table.add_column("Durum", width=6, justify="center")
|
|
315
|
+
table.add_column("Domain", style="green", overflow="fold")
|
|
316
|
+
table.add_column("Endpoint (Path)", style="bold white", overflow="fold")
|
|
317
|
+
table.add_column("Sorgu / Payload", style="yellow", width=18)
|
|
318
|
+
table.add_column("Kritik Başlıklar", style="dim cyan", width=22)
|
|
319
|
+
|
|
320
|
+
for ep in self.captured_endpoints:
|
|
321
|
+
status_color = "green" if 200 <= ep.status_code < 300 else "yellow" if ep.status_code < 400 else "red"
|
|
322
|
+
has_query = f"Query({len(ep.query_params)})" if ep.query_params else "-"
|
|
323
|
+
has_body = "Body(✔)" if ep.request_payload is not None else "-"
|
|
324
|
+
meta_info = f"{has_query} / {has_body}"
|
|
325
|
+
header_keys = ", ".join(ep.headers.keys()) if ep.headers else "-"
|
|
326
|
+
|
|
327
|
+
table.add_row(
|
|
328
|
+
str(ep.id),
|
|
329
|
+
ep.method,
|
|
330
|
+
f"[{status_color}]{ep.status_code}[/{status_color}]",
|
|
331
|
+
ep.domain,
|
|
332
|
+
ep.path,
|
|
333
|
+
meta_info,
|
|
334
|
+
header_keys,
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
console.print()
|
|
338
|
+
console.print(table)
|
|
339
|
+
console.print(
|
|
340
|
+
f"\n[bold green]💾 Trafik verisi başarıyla kaydedildi:[/bold green] "
|
|
341
|
+
f"[underline]{self.output_path.resolve()}[/underline]\n"
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
async def run(self) -> list[CapturedEndpoint]:
|
|
345
|
+
"""Tarayıcıyı başlatır, trafiği dinler ve kullanıcı Enter'a basana kadar bekler."""
|
|
346
|
+
console.print(
|
|
347
|
+
Panel.fit(
|
|
348
|
+
f"[bold cyan]Hedef URL:[/bold cyan] {self.target_url}\n"
|
|
349
|
+
f"[bold cyan]Çıktı Dosyası:[/bold cyan] {self.output_path}\n\n"
|
|
350
|
+
"[dim]Tarayıcı açılıyor... İşlemlerinizi tamamladıktan sonra kaydı bitirmek için "
|
|
351
|
+
"bu terminalde [bold white]ENTER[/bold white] tuşuna basın.[/dim]",
|
|
352
|
+
title="🚀 Shadow-API Recorder",
|
|
353
|
+
border_style="cyan",
|
|
354
|
+
)
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
self.profile_dir.mkdir(parents=True, exist_ok=True)
|
|
358
|
+
# Headless çalışmalardan kalan ekran dışı (-32000) pencere konumunu sıfırla
|
|
359
|
+
prefs_file = self.profile_dir / "Default" / "Preferences"
|
|
360
|
+
if prefs_file.exists():
|
|
361
|
+
try:
|
|
362
|
+
prefs_data = json.loads(prefs_file.read_text(encoding="utf-8"))
|
|
363
|
+
if "browser" in prefs_data and "window_placement" in prefs_data["browser"]:
|
|
364
|
+
prefs_data["browser"].pop("window_placement", None)
|
|
365
|
+
prefs_file.write_text(json.dumps(prefs_data), encoding="utf-8")
|
|
366
|
+
except Exception:
|
|
367
|
+
pass
|
|
368
|
+
|
|
369
|
+
from healer import DEFAULT_DESKTOP_UA, STEALTH_INIT_SCRIPT
|
|
370
|
+
|
|
371
|
+
async with async_playwright() as p:
|
|
372
|
+
context = await p.chromium.launch_persistent_context(
|
|
373
|
+
user_data_dir=str(self.profile_dir.resolve()),
|
|
374
|
+
headless=False,
|
|
375
|
+
user_agent=DEFAULT_DESKTOP_UA,
|
|
376
|
+
locale="tr-TR",
|
|
377
|
+
timezone_id="Europe/Istanbul",
|
|
378
|
+
ignore_https_errors=True,
|
|
379
|
+
accept_downloads=True,
|
|
380
|
+
no_viewport=True,
|
|
381
|
+
args=[
|
|
382
|
+
"--disable-http2",
|
|
383
|
+
"--disable-blink-features=AutomationControlled",
|
|
384
|
+
"--no-sandbox",
|
|
385
|
+
"--disable-setuid-sandbox",
|
|
386
|
+
"--disable-dev-shm-usage",
|
|
387
|
+
"--disable-accelerated-2d-canvas",
|
|
388
|
+
"--no-first-run",
|
|
389
|
+
"--no-service-autorun",
|
|
390
|
+
"--password-store=basic",
|
|
391
|
+
"--use-mock-keychain",
|
|
392
|
+
"--window-position=80,60",
|
|
393
|
+
"--window-size=1366,850",
|
|
394
|
+
],
|
|
395
|
+
)
|
|
396
|
+
await context.add_init_script(STEALTH_INIT_SCRIPT)
|
|
397
|
+
|
|
398
|
+
page: Page = context.pages[0] if context.pages else await context.new_page()
|
|
399
|
+
await page.bring_to_front()
|
|
400
|
+
await page.add_init_script(STEALTH_INIT_SCRIPT)
|
|
401
|
+
# Tüm mevcut ve yeni açılacak sayfalarda yanıtları dinle
|
|
402
|
+
for existing_page in context.pages:
|
|
403
|
+
existing_page.on("response", lambda response: asyncio.create_task(self._handle_response(response)))
|
|
404
|
+
context.on(
|
|
405
|
+
"page",
|
|
406
|
+
lambda new_p: new_p.on("response", lambda response: asyncio.create_task(self._handle_response(response))),
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
try:
|
|
410
|
+
await page.goto(self.target_url, wait_until="commit", timeout=30_000)
|
|
411
|
+
except Exception as exc:
|
|
412
|
+
console.print(
|
|
413
|
+
f"[bold yellow]⚠Sayfa ilk yönlendirmede uyarı verdi ({exc}), "
|
|
414
|
+
"tarayıcıda manuel gezinebilirsiniz; trafik dinleme aktif...[/bold yellow]"
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
console.print(
|
|
418
|
+
Panel.fit(
|
|
419
|
+
"[bold cyan]ğŸ‘ï¸ Tarayıcı Penceresi Ekranda Açıldı (headless=False)[/bold cyan]\n"
|
|
420
|
+
"[dim]Eğer ekranda PerimeterX (Basılı Tutun), Cloudflare Turnstile veya Captcha çıkarsa lütfen tarayıcıda tamamlayın.[/dim]\n"
|
|
421
|
+
f"[dim]Kullanıcı kontrolü ve sayfa stabilizasyonu için {self.user_wait_sec} saniye bekleniyor...[/dim]",
|
|
422
|
+
title="Browser Hazır",
|
|
423
|
+
border_style="cyan",
|
|
424
|
+
)
|
|
425
|
+
)
|
|
426
|
+
if self.user_wait_sec > 0:
|
|
427
|
+
await page.wait_for_timeout(self.user_wait_sec * 1000)
|
|
428
|
+
|
|
429
|
+
console.print(
|
|
430
|
+
"\n[bold reverse green] KAYIT AKTİF [/bold reverse green] "
|
|
431
|
+
"Tarayıcıda etkileşimde bulunun. Durdurmak için [bold yellow]ENTER[/bold yellow]'a basın...\n"
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
# Bloklamadan kullanıcının Enter'a basmasını bekle
|
|
435
|
+
await asyncio.to_thread(input)
|
|
436
|
+
|
|
437
|
+
console.print("[dim]Oturum durumu (.browser_profile) kaydediliyor ve tarayıcı kapatılıyor...[/dim]")
|
|
438
|
+
try:
|
|
439
|
+
await context.storage_state(path=str(self.profile_dir / "storage_state.json"))
|
|
440
|
+
except Exception:
|
|
441
|
+
pass
|
|
442
|
+
await context.close()
|
|
443
|
+
|
|
444
|
+
self.save_to_disk()
|
|
445
|
+
self.print_summary_table()
|
|
446
|
+
return self.captured_endpoints
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
@app.command()
|
|
450
|
+
def record(
|
|
451
|
+
url: str = typer.Argument(..., help="Dinlenecek hedef web sitesinin URL'i (örn: https://example.com)"),
|
|
452
|
+
output: Path = typer.Option(
|
|
453
|
+
Path("captured_traffic.json"),
|
|
454
|
+
"--output",
|
|
455
|
+
"-o",
|
|
456
|
+
help="Yakalanan trafiğin kaydedileceği JSON dosyası",
|
|
457
|
+
),
|
|
458
|
+
max_chars: int = typer.Option(
|
|
459
|
+
2000,
|
|
460
|
+
"--max-chars",
|
|
461
|
+
"-m",
|
|
462
|
+
help="Yanıt JSON örneği için maksimum karakter sınırı",
|
|
463
|
+
),
|
|
464
|
+
user_wait: int = typer.Option(
|
|
465
|
+
5,
|
|
466
|
+
"--user-wait",
|
|
467
|
+
"-w",
|
|
468
|
+
help="Sayfa açıldıktan sonra kullanıcı doğrulaması (PerimeterX vb.) için tanınan başlangıç bekleme süresi (saniye)",
|
|
469
|
+
),
|
|
470
|
+
) -> None:
|
|
471
|
+
"""Verilen URL'yi görünür Chromium ile açar ve XHR/Fetch JSON trafiğini kaydeder."""
|
|
472
|
+
if not url.startswith(("http://", "https://")):
|
|
473
|
+
url = f"https://{url}"
|
|
474
|
+
|
|
475
|
+
recorder = TrafficRecorder(target_url=url, output_path=output, max_response_chars=max_chars, user_wait_sec=user_wait)
|
|
476
|
+
try:
|
|
477
|
+
asyncio.run(recorder.run())
|
|
478
|
+
except KeyboardInterrupt:
|
|
479
|
+
console.print("\n[yellow]İşlem kullanıcı tarafından kesildi. Mevcut veriler kaydediliyor...[/yellow]")
|
|
480
|
+
recorder.save_to_disk()
|
|
481
|
+
recorder.print_summary_table()
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
if __name__ == "__main__":
|
|
485
|
+
app()
|
|
486
|
+
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import pathlib
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
10
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
11
|
+
|
|
12
|
+
from core.base_client import BaseOwnClient
|
|
13
|
+
from core.token_store import TokenStore
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CoingeckoClient(BaseOwnClient):
|
|
17
|
+
"""CoinGecko cryptocurrency market & token search client via OwnAPI User-Delegated Browser Middleware."""
|
|
18
|
+
|
|
19
|
+
def __init__(
|
|
20
|
+
self,
|
|
21
|
+
tenant_id: str = "default",
|
|
22
|
+
token_store: TokenStore | None = None,
|
|
23
|
+
mode: str = "local",
|
|
24
|
+
) -> None:
|
|
25
|
+
super().__init__(
|
|
26
|
+
service_name="coingecko",
|
|
27
|
+
base_url="https://api.coingecko.com/",
|
|
28
|
+
tenant_id=tenant_id,
|
|
29
|
+
token_store=token_store,
|
|
30
|
+
mode=mode,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def search_crypto(self, query: str = "bitcoin") -> dict[str, Any]:
|
|
34
|
+
url = "https://api.coingecko.com/api/v3/search"
|
|
35
|
+
headers = {
|
|
36
|
+
"accept": "application/json, text/plain, */*",
|
|
37
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36",
|
|
38
|
+
}
|
|
39
|
+
params = {
|
|
40
|
+
"query": str(query).strip() or "bitcoin",
|
|
41
|
+
}
|
|
42
|
+
raw = self.request(method="GET", url=url, headers=headers, params=params)
|
|
43
|
+
if isinstance(raw, dict) and "coins" in raw:
|
|
44
|
+
coins = raw.get("coins") or []
|
|
45
|
+
return {
|
|
46
|
+
"success": True,
|
|
47
|
+
"service": "coingecko",
|
|
48
|
+
"query": query,
|
|
49
|
+
"total_coins_found": len(coins),
|
|
50
|
+
"top_coins": [
|
|
51
|
+
{
|
|
52
|
+
"id": c.get("id"),
|
|
53
|
+
"name": c.get("name"),
|
|
54
|
+
"symbol": c.get("symbol"),
|
|
55
|
+
"market_cap_rank": c.get("market_cap_rank"),
|
|
56
|
+
"thumb": c.get("thumb"),
|
|
57
|
+
}
|
|
58
|
+
for c in coins[:10]
|
|
59
|
+
if isinstance(c, dict)
|
|
60
|
+
],
|
|
61
|
+
"categories": (raw.get("categories") or [])[:5],
|
|
62
|
+
}
|
|
63
|
+
return raw
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def search_crypto(
|
|
67
|
+
query: str = "bitcoin",
|
|
68
|
+
tenant_id: str = "default",
|
|
69
|
+
mode: str = "local",
|
|
70
|
+
) -> dict[str, Any]:
|
|
71
|
+
"""
|
|
72
|
+
Searches cryptocurrencies, tokens, market cap ranks, and categories on CoinGecko
|
|
73
|
+
via OwnAPI User-Delegated Browser Middleware.
|
|
74
|
+
"""
|
|
75
|
+
client = CoingeckoClient(tenant_id=tenant_id, mode=mode)
|
|
76
|
+
return client.search_crypto(query=query)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
if __name__ == "__main__":
|
|
80
|
+
t0 = time.perf_counter()
|
|
81
|
+
res = search_crypto()
|
|
82
|
+
elapsed = (time.perf_counter() - t0) * 1000
|
|
83
|
+
print(f"[OwnAPI - CoinGecko] ({elapsed:.1f} ms)")
|
|
84
|
+
print(json.dumps(res, ensure_ascii=False, indent=2)[:800])
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"service_name": "coingecko",
|
|
3
|
+
"target_url": "https://www.coingecko.com/",
|
|
4
|
+
"goal": "Cryptocurrency market, token, and market cap rank search on CoinGecko",
|
|
5
|
+
"function_name": "search_crypto",
|
|
6
|
+
"explanation": "CoinGecko cryptocurrency search endpoint returning matching coins, symbols, market cap rankings, and categories.",
|
|
7
|
+
"method": "GET",
|
|
8
|
+
"endpoint_url": "https://api.coingecko.com/api/v3/search",
|
|
9
|
+
"dynamic_params": [
|
|
10
|
+
{
|
|
11
|
+
"name": "query",
|
|
12
|
+
"original_key": "query",
|
|
13
|
+
"location": "query",
|
|
14
|
+
"param_type": "str",
|
|
15
|
+
"default_value": "bitcoin",
|
|
16
|
+
"description": "Cryptocurrency name or ticker symbol (e.g. bitcoin, solana, ethereum)"
|
|
17
|
+
}
|
|
18
|
+
],
|
|
19
|
+
"created_at": "2026-11-06T17:48:00+00:00",
|
|
20
|
+
"validation_status": "BAŞARILI",
|
|
21
|
+
"validation_passed_runs": 3,
|
|
22
|
+
"last_validated_at": "2026-11-06T19:29:09.420490+00:00"
|
|
23
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import pathlib
|
|
5
|
+
import sys
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
9
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
10
|
+
|
|
11
|
+
from core.base_client import BaseOwnClient
|
|
12
|
+
from core.token_store import TokenStore
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class EdevletClient(BaseOwnClient):
|
|
16
|
+
"""e-Devlet Kapısı (turkiye.gov.tr) User-Delegated Browser Middleware istemcisi."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, tenant_id: str = "default", token_store: TokenStore | None = None, mode: str = "local"):
|
|
19
|
+
super().__init__(
|
|
20
|
+
service_name="edevlet",
|
|
21
|
+
base_url="https://www.turkiye.gov.tr",
|
|
22
|
+
tenant_id=tenant_id,
|
|
23
|
+
token_store=token_store,
|
|
24
|
+
mode=mode,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
def get_homepage_content(self) -> dict[str, Any]:
|
|
28
|
+
headers = {
|
|
29
|
+
"accept": "application/json, text/plain, */*",
|
|
30
|
+
}
|
|
31
|
+
res = self.request(method="GET", url="/", headers=headers)
|
|
32
|
+
if isinstance(res, dict):
|
|
33
|
+
text_val = res.get("text") or res.get("response_sample") or json.dumps(res, ensure_ascii=False)
|
|
34
|
+
return {
|
|
35
|
+
"service": "edevlet",
|
|
36
|
+
"url": "https://www.turkiye.gov.tr/",
|
|
37
|
+
"status_code": res.get("status_code", 200),
|
|
38
|
+
"ok": res.get("ok", True),
|
|
39
|
+
"content_preview": str(text_val)[:1500],
|
|
40
|
+
}
|
|
41
|
+
text_val = getattr(res, "text", str(res))
|
|
42
|
+
return {
|
|
43
|
+
"service": "edevlet",
|
|
44
|
+
"url": "https://www.turkiye.gov.tr/",
|
|
45
|
+
"status_code": 200,
|
|
46
|
+
"ok": True,
|
|
47
|
+
"content_preview": str(text_val)[:1500],
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def get_homepage_content(tenant_id: str = "default", mode: str = "local") -> dict[str, Any]:
|
|
52
|
+
"""e-Devlet ana sayfasının içeriğini kullanıcının gerçek tarayıcı oturumu üzerinden alır."""
|
|
53
|
+
client = EdevletClient(tenant_id=tenant_id, mode=mode)
|
|
54
|
+
return client.get_homepage_content()
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"service_name": "edevlet",
|
|
3
|
+
"target_url": "https://www.turkiye.gov.tr",
|
|
4
|
+
"goal": "edevlet test",
|
|
5
|
+
"function_name": "get_homepage_content",
|
|
6
|
+
"explanation": "Kullanıcının hedefi olan e-Devlet ana sayfasını almak için uygun uç nokta.",
|
|
7
|
+
"method": "GET",
|
|
8
|
+
"endpoint_url": "https://www.turkiye.gov.tr/",
|
|
9
|
+
"dynamic_params": [],
|
|
10
|
+
"created_at": "2026-11-06T19:25:14.919786+00:00",
|
|
11
|
+
"validation_status": "BAŞARILI",
|
|
12
|
+
"validation_passed_runs": 3,
|
|
13
|
+
"last_validated_at": "2026-11-06T19:30:22.403562+00:00"
|
|
14
|
+
}
|