fluidattacks_core_git 12.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fluidattacks_core/git/__init__.py +382 -0
- fluidattacks_core/git/classes.py +29 -0
- fluidattacks_core/git/clone.py +243 -0
- fluidattacks_core/git/codecommit_utils.py +129 -0
- fluidattacks_core/git/constants.py +7 -0
- fluidattacks_core/git/delete_files.py +47 -0
- fluidattacks_core/git/download_file.py +47 -0
- fluidattacks_core/git/download_repo.py +148 -0
- fluidattacks_core/git/https_utils.py +230 -0
- fluidattacks_core/git/py.typed +0 -0
- fluidattacks_core/git/remote.py +45 -0
- fluidattacks_core/git/show_file.py +408 -0
- fluidattacks_core/git/ssh_utils.py +214 -0
- fluidattacks_core/git/upload_repo.py +338 -0
- fluidattacks_core/git/utils.py +143 -0
- fluidattacks_core/git/warp.py +184 -0
- fluidattacks_core_git-12.0.0.dist-info/METADATA +22 -0
- fluidattacks_core_git-12.0.0.dist-info/RECORD +19 -0
- fluidattacks_core_git-12.0.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import math
|
|
3
|
+
import tarfile
|
|
4
|
+
import tempfile
|
|
5
|
+
from collections.abc import AsyncGenerator, Awaitable, Callable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import aiofiles
|
|
10
|
+
import aiohttp
|
|
11
|
+
import anyio
|
|
12
|
+
|
|
13
|
+
from .constants import (
|
|
14
|
+
MAX_RETRIES_PER_PART,
|
|
15
|
+
MAX_S3_PARTS,
|
|
16
|
+
MULTIPART_PART_SIZE,
|
|
17
|
+
MULTIPART_THRESHOLD,
|
|
18
|
+
UPLOAD_CONCURRENCY,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
LOGGER = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _compute_part_size(file_size: int) -> int:
|
|
25
|
+
return max(MULTIPART_PART_SIZE, math.ceil(file_size / MAX_S3_PARTS))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class UploadPartResult:
|
|
30
|
+
part_number: int
|
|
31
|
+
etag: str
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class PartUploadConfig:
|
|
36
|
+
file_path: Path
|
|
37
|
+
part_number: int
|
|
38
|
+
start_byte: int
|
|
39
|
+
end_byte: int
|
|
40
|
+
upload_id: str
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class MultipartCallbacks:
|
|
45
|
+
initiate_upload: Callable[[], Awaitable[str | None]]
|
|
46
|
+
get_part_url: Callable[[str, int], Awaitable[str]]
|
|
47
|
+
complete_upload: Callable[[str, list[dict[str, str | int]]], Awaitable[bool]]
|
|
48
|
+
abort_upload: Callable[[str], Awaitable[None]]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
async def _create_tar_archive(
|
|
52
|
+
repo_path: Path,
|
|
53
|
+
output_path: Path,
|
|
54
|
+
archive_name: str,
|
|
55
|
+
) -> bool:
|
|
56
|
+
git_dir = repo_path / ".git"
|
|
57
|
+
if not await anyio.Path(git_dir).exists():
|
|
58
|
+
LOGGER.error("Git directory not found: %s", git_dir)
|
|
59
|
+
return False
|
|
60
|
+
|
|
61
|
+
def _create() -> None:
|
|
62
|
+
with tarfile.open(output_path, "w:gz") as tar_handler:
|
|
63
|
+
tar_handler.add(str(git_dir), arcname=f"{archive_name}/.git", recursive=True)
|
|
64
|
+
|
|
65
|
+
try:
|
|
66
|
+
await anyio.to_thread.run_sync(_create)
|
|
67
|
+
except (OSError, tarfile.TarError):
|
|
68
|
+
LOGGER.exception("Failed to create tar archive: %s", output_path)
|
|
69
|
+
return False
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
async def _file_chunks(file_path: Path) -> AsyncGenerator[bytes, None]:
|
|
74
|
+
async with aiofiles.open(file_path, "rb") as f:
|
|
75
|
+
while chunk := await f.read(65536):
|
|
76
|
+
yield chunk
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
async def _single_part_upload(
|
|
80
|
+
file_path: Path,
|
|
81
|
+
upload_url: str,
|
|
82
|
+
) -> bool:
|
|
83
|
+
file_stat = await anyio.Path(file_path).stat()
|
|
84
|
+
file_size = file_stat.st_size
|
|
85
|
+
timeout = aiohttp.ClientTimeout(total=3600, sock_connect=30, sock_read=600)
|
|
86
|
+
|
|
87
|
+
async with (
|
|
88
|
+
aiohttp.ClientSession(timeout=timeout) as session,
|
|
89
|
+
session.put(
|
|
90
|
+
upload_url,
|
|
91
|
+
data=_file_chunks(file_path),
|
|
92
|
+
headers={
|
|
93
|
+
"Content-Type": "application/octet-stream",
|
|
94
|
+
"Content-Length": str(file_size),
|
|
95
|
+
},
|
|
96
|
+
) as response,
|
|
97
|
+
):
|
|
98
|
+
if response.status != 200:
|
|
99
|
+
error_text = await response.text()
|
|
100
|
+
LOGGER.error(
|
|
101
|
+
"Single-part upload failed: HTTP %s - %s",
|
|
102
|
+
response.status,
|
|
103
|
+
error_text,
|
|
104
|
+
)
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
return True
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
async def _do_part_upload(
|
|
111
|
+
session: aiohttp.ClientSession,
|
|
112
|
+
part_url: str,
|
|
113
|
+
chunk_data: bytes,
|
|
114
|
+
) -> str | None:
|
|
115
|
+
async with session.put(
|
|
116
|
+
part_url,
|
|
117
|
+
data=chunk_data,
|
|
118
|
+
headers={
|
|
119
|
+
"Content-Type": "application/octet-stream",
|
|
120
|
+
"Content-Length": str(len(chunk_data)),
|
|
121
|
+
},
|
|
122
|
+
) as response:
|
|
123
|
+
if response.status == 200:
|
|
124
|
+
return response.headers.get("ETag", "")
|
|
125
|
+
|
|
126
|
+
error_text = await response.text()
|
|
127
|
+
LOGGER.warning(
|
|
128
|
+
"Part upload failed: HTTP %s - %s",
|
|
129
|
+
response.status,
|
|
130
|
+
error_text,
|
|
131
|
+
)
|
|
132
|
+
return None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
async def _upload_part_with_retry(
|
|
136
|
+
config: PartUploadConfig,
|
|
137
|
+
get_part_url: Callable[[str, int], Awaitable[str]],
|
|
138
|
+
) -> UploadPartResult | None:
|
|
139
|
+
part_size = config.end_byte - config.start_byte
|
|
140
|
+
timeout = aiohttp.ClientTimeout(total=600, sock_connect=30, sock_read=300)
|
|
141
|
+
|
|
142
|
+
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
143
|
+
for attempt in range(1, MAX_RETRIES_PER_PART + 1):
|
|
144
|
+
try:
|
|
145
|
+
part_url = await get_part_url(config.upload_id, config.part_number)
|
|
146
|
+
if not part_url:
|
|
147
|
+
LOGGER.error("Failed to get presigned URL for part %s", config.part_number)
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
async with aiofiles.open(config.file_path, "rb") as f:
|
|
151
|
+
await f.seek(config.start_byte)
|
|
152
|
+
chunk_data = await f.read(part_size)
|
|
153
|
+
|
|
154
|
+
etag = await _do_part_upload(session, part_url, chunk_data)
|
|
155
|
+
if etag:
|
|
156
|
+
LOGGER.info(
|
|
157
|
+
"Part %s uploaded successfully (attempt %s)",
|
|
158
|
+
config.part_number,
|
|
159
|
+
attempt,
|
|
160
|
+
)
|
|
161
|
+
return UploadPartResult(part_number=config.part_number, etag=etag)
|
|
162
|
+
|
|
163
|
+
LOGGER.warning(
|
|
164
|
+
"Part %s upload failed (attempt %s/%s)",
|
|
165
|
+
config.part_number,
|
|
166
|
+
attempt,
|
|
167
|
+
MAX_RETRIES_PER_PART,
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
except (TimeoutError, aiohttp.ClientError, OSError):
|
|
171
|
+
LOGGER.exception(
|
|
172
|
+
"Part %s upload error (attempt %s/%s)",
|
|
173
|
+
config.part_number,
|
|
174
|
+
attempt,
|
|
175
|
+
MAX_RETRIES_PER_PART,
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
if attempt < MAX_RETRIES_PER_PART:
|
|
179
|
+
await anyio.sleep(2**attempt)
|
|
180
|
+
|
|
181
|
+
LOGGER.error(
|
|
182
|
+
"Part %s failed after %s attempts",
|
|
183
|
+
config.part_number,
|
|
184
|
+
MAX_RETRIES_PER_PART,
|
|
185
|
+
)
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
async def _initiate_with_retry(
|
|
190
|
+
initiate_upload: Callable[[], Awaitable[str | None]],
|
|
191
|
+
) -> str | None:
|
|
192
|
+
for attempt in range(1, MAX_RETRIES_PER_PART + 1):
|
|
193
|
+
upload_id = await initiate_upload()
|
|
194
|
+
if upload_id:
|
|
195
|
+
return upload_id
|
|
196
|
+
LOGGER.warning(
|
|
197
|
+
"initiate_upload failed (attempt %s/%s)",
|
|
198
|
+
attempt,
|
|
199
|
+
MAX_RETRIES_PER_PART,
|
|
200
|
+
)
|
|
201
|
+
if attempt < MAX_RETRIES_PER_PART:
|
|
202
|
+
await anyio.sleep(2**attempt)
|
|
203
|
+
LOGGER.error("Failed to initiate multipart upload after %s attempts", MAX_RETRIES_PER_PART)
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
async def _complete_with_retry(
|
|
208
|
+
upload_id: str,
|
|
209
|
+
completed_parts: list[dict[str, str | int]],
|
|
210
|
+
complete_upload: Callable[[str, list[dict[str, str | int]]], Awaitable[bool]],
|
|
211
|
+
) -> bool:
|
|
212
|
+
for attempt in range(1, MAX_RETRIES_PER_PART + 1):
|
|
213
|
+
if await complete_upload(upload_id, completed_parts):
|
|
214
|
+
return True
|
|
215
|
+
LOGGER.warning(
|
|
216
|
+
"complete_upload failed (attempt %s/%s)",
|
|
217
|
+
attempt,
|
|
218
|
+
MAX_RETRIES_PER_PART,
|
|
219
|
+
)
|
|
220
|
+
if attempt < MAX_RETRIES_PER_PART:
|
|
221
|
+
await anyio.sleep(2**attempt)
|
|
222
|
+
LOGGER.error("Failed to complete multipart upload after %s attempts", MAX_RETRIES_PER_PART)
|
|
223
|
+
return False
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
async def _upload_all_parts(
|
|
227
|
+
file_path: Path,
|
|
228
|
+
file_size: int,
|
|
229
|
+
upload_id: str,
|
|
230
|
+
get_part_url: Callable[[str, int], Awaitable[str]],
|
|
231
|
+
part_size: int,
|
|
232
|
+
) -> list[UploadPartResult] | None:
|
|
233
|
+
num_parts = (file_size + part_size - 1) // part_size
|
|
234
|
+
LOGGER.info("Starting multipart upload: %s bytes in %s parts", file_size, num_parts)
|
|
235
|
+
results: list[UploadPartResult | None] = [None] * num_parts
|
|
236
|
+
limiter = anyio.CapacityLimiter(UPLOAD_CONCURRENCY)
|
|
237
|
+
|
|
238
|
+
async def _upload_one(part_number: int) -> None:
|
|
239
|
+
start_byte = (part_number - 1) * part_size
|
|
240
|
+
end_byte = min(part_number * part_size, file_size)
|
|
241
|
+
config = PartUploadConfig(
|
|
242
|
+
file_path=file_path,
|
|
243
|
+
part_number=part_number,
|
|
244
|
+
start_byte=start_byte,
|
|
245
|
+
end_byte=end_byte,
|
|
246
|
+
upload_id=upload_id,
|
|
247
|
+
)
|
|
248
|
+
async with limiter:
|
|
249
|
+
results[part_number - 1] = await _upload_part_with_retry(config, get_part_url)
|
|
250
|
+
|
|
251
|
+
async with anyio.create_task_group() as tg:
|
|
252
|
+
for part_number in range(1, num_parts + 1):
|
|
253
|
+
tg.start_soon(_upload_one, part_number)
|
|
254
|
+
|
|
255
|
+
parts: list[UploadPartResult] = []
|
|
256
|
+
for result in results:
|
|
257
|
+
if result is None:
|
|
258
|
+
LOGGER.error("One or more parts failed to upload")
|
|
259
|
+
return None
|
|
260
|
+
parts.append(result)
|
|
261
|
+
return parts
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
async def _multipart_upload(
|
|
265
|
+
file_path: Path,
|
|
266
|
+
multipart: MultipartCallbacks,
|
|
267
|
+
) -> bool:
|
|
268
|
+
file_stat = await anyio.Path(file_path).stat()
|
|
269
|
+
file_size = file_stat.st_size
|
|
270
|
+
part_size = _compute_part_size(file_size)
|
|
271
|
+
|
|
272
|
+
upload_id = await _initiate_with_retry(multipart.initiate_upload)
|
|
273
|
+
if not upload_id:
|
|
274
|
+
return False
|
|
275
|
+
|
|
276
|
+
completed = False
|
|
277
|
+
try:
|
|
278
|
+
parts = await _upload_all_parts(
|
|
279
|
+
file_path, file_size, upload_id, multipart.get_part_url, part_size
|
|
280
|
+
)
|
|
281
|
+
if parts is None:
|
|
282
|
+
return False
|
|
283
|
+
|
|
284
|
+
completed_parts: list[dict[str, str | int]] = [
|
|
285
|
+
{"ETag": part.etag, "PartNumber": part.part_number} for part in parts
|
|
286
|
+
]
|
|
287
|
+
|
|
288
|
+
if not await _complete_with_retry(upload_id, completed_parts, multipart.complete_upload):
|
|
289
|
+
return False
|
|
290
|
+
|
|
291
|
+
LOGGER.info("Multipart upload completed successfully")
|
|
292
|
+
completed = True
|
|
293
|
+
return True
|
|
294
|
+
finally:
|
|
295
|
+
if not completed:
|
|
296
|
+
await multipart.abort_upload(upload_id)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
async def upload_repo_to_s3(
|
|
300
|
+
repo_path: Path,
|
|
301
|
+
archive_name: str,
|
|
302
|
+
*,
|
|
303
|
+
upload_url: str | None = None,
|
|
304
|
+
multipart: MultipartCallbacks | None = None,
|
|
305
|
+
) -> bool:
|
|
306
|
+
if not await anyio.Path(repo_path).exists():
|
|
307
|
+
LOGGER.error("Repository path does not exist: %s", repo_path)
|
|
308
|
+
return False
|
|
309
|
+
|
|
310
|
+
with tempfile.TemporaryDirectory(
|
|
311
|
+
prefix="fluidattacks_upload_",
|
|
312
|
+
ignore_cleanup_errors=True,
|
|
313
|
+
) as temp_dir:
|
|
314
|
+
tar_path = Path(temp_dir) / f"{archive_name}.tar.gz"
|
|
315
|
+
|
|
316
|
+
if not await _create_tar_archive(repo_path, tar_path, archive_name):
|
|
317
|
+
return False
|
|
318
|
+
|
|
319
|
+
tar_stat = await anyio.Path(tar_path).stat()
|
|
320
|
+
file_size = tar_stat.st_size
|
|
321
|
+
LOGGER.info("Created archive: %s (%s bytes)", tar_path.name, file_size)
|
|
322
|
+
|
|
323
|
+
if file_size <= MULTIPART_THRESHOLD:
|
|
324
|
+
if not upload_url:
|
|
325
|
+
LOGGER.error("upload_url required for single-part upload")
|
|
326
|
+
return False
|
|
327
|
+
LOGGER.info("Using single-part upload (file size <= 5GB)")
|
|
328
|
+
return await _single_part_upload(tar_path, upload_url)
|
|
329
|
+
|
|
330
|
+
if multipart is None:
|
|
331
|
+
LOGGER.error(
|
|
332
|
+
"Multipart callbacks required for files > 5GB (file size: %s bytes)",
|
|
333
|
+
file_size,
|
|
334
|
+
)
|
|
335
|
+
return False
|
|
336
|
+
|
|
337
|
+
LOGGER.info("Using multipart upload (file size > 5GB)")
|
|
338
|
+
return await _multipart_upload(tar_path, multipart)
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import base64
|
|
3
|
+
from contextlib import suppress
|
|
4
|
+
from urllib.parse import (
|
|
5
|
+
ParseResult,
|
|
6
|
+
quote,
|
|
7
|
+
unquote,
|
|
8
|
+
urlparse,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
async def run_git(
|
|
13
|
+
*args: str,
|
|
14
|
+
env: dict[str, str] | None = None,
|
|
15
|
+
cwd: str | None = None,
|
|
16
|
+
timeout: float = 20.0, # noqa: ASYNC109
|
|
17
|
+
) -> tuple[bytes, bytes, int]:
|
|
18
|
+
proc = await asyncio.create_subprocess_exec(
|
|
19
|
+
"git",
|
|
20
|
+
*args,
|
|
21
|
+
stderr=asyncio.subprocess.PIPE,
|
|
22
|
+
stdout=asyncio.subprocess.PIPE,
|
|
23
|
+
stdin=asyncio.subprocess.DEVNULL,
|
|
24
|
+
cwd=cwd,
|
|
25
|
+
env=env,
|
|
26
|
+
)
|
|
27
|
+
try:
|
|
28
|
+
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
|
29
|
+
except asyncio.exceptions.TimeoutError:
|
|
30
|
+
with suppress(ProcessLookupError):
|
|
31
|
+
proc.kill()
|
|
32
|
+
await proc.wait()
|
|
33
|
+
raise
|
|
34
|
+
|
|
35
|
+
return stdout, stderr, proc.returncode if proc.returncode is not None else -1
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _replace_netloc_in_url(parsed_url: ParseResult, netloc: str) -> str:
|
|
39
|
+
return parsed_url._replace(netloc=netloc).geturl()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _format_token_for_provider(provider: str | None, token: str, host: str) -> str:
|
|
43
|
+
if provider == "BITBUCKET":
|
|
44
|
+
return f"x-token-auth:{token}@{host}"
|
|
45
|
+
if provider:
|
|
46
|
+
return f"oauth2:{token}@{host}"
|
|
47
|
+
return f"{token}@{host}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _format_token(
|
|
51
|
+
parsed_url: ParseResult,
|
|
52
|
+
token: str,
|
|
53
|
+
host: str,
|
|
54
|
+
provider: str | None,
|
|
55
|
+
) -> str:
|
|
56
|
+
formatted_token = _format_token_for_provider(provider, token, host)
|
|
57
|
+
return _replace_netloc_in_url(parsed_url, formatted_token)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _quote_if_not_none(value: str | None) -> str | None:
|
|
61
|
+
return quote(value, safe="") if value is not None else value
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _quote_path_in_url(url: str) -> ParseResult:
|
|
65
|
+
parsed_url = urlparse(url)
|
|
66
|
+
return parsed_url._replace(path=quote(unquote(parsed_url.path)))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _get_host_from_url(parsed_url: ParseResult) -> str:
|
|
70
|
+
host = parsed_url.netloc
|
|
71
|
+
if "@" in host:
|
|
72
|
+
host = host.split("@")[-1]
|
|
73
|
+
return host
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _get_url_based_on_credentials( # noqa: PLR0913
|
|
77
|
+
*,
|
|
78
|
+
parsed_url: ParseResult,
|
|
79
|
+
token: str | None,
|
|
80
|
+
host: str,
|
|
81
|
+
provider: str | None,
|
|
82
|
+
user: str | None,
|
|
83
|
+
password: str | None,
|
|
84
|
+
) -> str:
|
|
85
|
+
if token is not None:
|
|
86
|
+
return _format_token(parsed_url, token, host, provider)
|
|
87
|
+
if user is not None and password is not None:
|
|
88
|
+
return _replace_netloc_in_url(parsed_url, f"{user}:{password}@{host}")
|
|
89
|
+
return parsed_url.geturl()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def format_url( # noqa: PLR0913
|
|
93
|
+
*,
|
|
94
|
+
repo_url: str,
|
|
95
|
+
user: str | None = None,
|
|
96
|
+
password: str | None = None,
|
|
97
|
+
token: str | None = None,
|
|
98
|
+
provider: str | None = None,
|
|
99
|
+
is_pat: bool = False,
|
|
100
|
+
) -> str:
|
|
101
|
+
parsed_url = _quote_path_in_url(repo_url)
|
|
102
|
+
if is_pat:
|
|
103
|
+
return parsed_url.geturl()
|
|
104
|
+
|
|
105
|
+
host = _get_host_from_url(parsed_url)
|
|
106
|
+
user = _quote_if_not_none(user)
|
|
107
|
+
password = _quote_if_not_none(password)
|
|
108
|
+
return _get_url_based_on_credentials(
|
|
109
|
+
parsed_url=parsed_url,
|
|
110
|
+
token=token,
|
|
111
|
+
host=host,
|
|
112
|
+
provider=provider,
|
|
113
|
+
user=user,
|
|
114
|
+
password=password,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def get_https_git_config_args(
|
|
119
|
+
*,
|
|
120
|
+
follow_redirects: bool = False,
|
|
121
|
+
is_pat: bool = False,
|
|
122
|
+
token: str | None = None,
|
|
123
|
+
disable_credential_prompt: bool = False,
|
|
124
|
+
) -> list[str]:
|
|
125
|
+
args = [
|
|
126
|
+
"-c",
|
|
127
|
+
"core.symlinks=false",
|
|
128
|
+
"-c",
|
|
129
|
+
"http.sslVerify=false",
|
|
130
|
+
"-c",
|
|
131
|
+
f"http.followRedirects={follow_redirects}",
|
|
132
|
+
]
|
|
133
|
+
if is_pat and token is not None:
|
|
134
|
+
args.extend(
|
|
135
|
+
[
|
|
136
|
+
"-c",
|
|
137
|
+
"http.extraHeader=Authorization: Basic "
|
|
138
|
+
+ base64.b64encode(f":{token}".encode()).decode(),
|
|
139
|
+
]
|
|
140
|
+
)
|
|
141
|
+
if disable_credential_prompt:
|
|
142
|
+
args.extend(["-c", "credential.helper="])
|
|
143
|
+
return args
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
import re
|
|
4
|
+
import socket
|
|
5
|
+
|
|
6
|
+
import aiohttp
|
|
7
|
+
|
|
8
|
+
LOGGER = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
CONFIG_DELAY: int = 5 # For WARP to apply network configurations, in seconds
|
|
11
|
+
DOMAIN_TO_TEST_DNS: str = "api.ipify.org" # Using a domain that used to fail
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class WarpError(Exception):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
async def test_public_ip(expected_ip: str) -> bool:
|
|
19
|
+
ip_service_url = "https://api.ipify.org?format=text"
|
|
20
|
+
try:
|
|
21
|
+
async with aiohttp.ClientSession() as session: # noqa: SIM117
|
|
22
|
+
async with session.get(ip_service_url) as response:
|
|
23
|
+
if response.status == 200:
|
|
24
|
+
public_ip = await response.text()
|
|
25
|
+
LOGGER.info("Current public IP: %s", public_ip)
|
|
26
|
+
return public_ip.strip() == expected_ip
|
|
27
|
+
|
|
28
|
+
LOGGER.error("Failed to fetch public IP. Status code: %s", response.status)
|
|
29
|
+
return False
|
|
30
|
+
|
|
31
|
+
except aiohttp.ClientError:
|
|
32
|
+
LOGGER.exception("Error fetching public IP")
|
|
33
|
+
return False
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
async def public_ip_ready(expected_ip: str, *, attempts: int, seconds_per_attempt: int) -> bool:
|
|
37
|
+
for attempt_number in range(1, attempts + 1):
|
|
38
|
+
LOGGER.info("Checking public IP... Attempt %s/%s", attempt_number, attempts)
|
|
39
|
+
if await test_public_ip(expected_ip):
|
|
40
|
+
LOGGER.info("Public IP test successful after %s attempts", attempt_number)
|
|
41
|
+
return True
|
|
42
|
+
|
|
43
|
+
LOGGER.info("Public IP test failed. Retrying in %s seconds", seconds_per_attempt)
|
|
44
|
+
await asyncio.sleep(seconds_per_attempt)
|
|
45
|
+
|
|
46
|
+
LOGGER.error("Public IP test failed after %s attempts", attempts)
|
|
47
|
+
return False
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
async def _dns_test(host: str) -> bool:
|
|
51
|
+
try:
|
|
52
|
+
socket.gethostbyname(host)
|
|
53
|
+
except socket.gaierror:
|
|
54
|
+
return False
|
|
55
|
+
else:
|
|
56
|
+
return True
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def is_dns_ready(
|
|
60
|
+
*,
|
|
61
|
+
host_to_test_dns: str,
|
|
62
|
+
attempts: int = 40,
|
|
63
|
+
seconds_per_attempt: int = 5,
|
|
64
|
+
) -> bool:
|
|
65
|
+
for attempt_number in range(1, attempts + 1):
|
|
66
|
+
LOGGER.info("Waiting for DNS resolution... Attempt %s/%s", attempt_number, attempts)
|
|
67
|
+
if await _dns_test(host_to_test_dns):
|
|
68
|
+
LOGGER.info("DNS resolution successful after %s attempts", attempt_number)
|
|
69
|
+
return True
|
|
70
|
+
LOGGER.error("DNS resolution failed. Retrying in %s seconds...", seconds_per_attempt)
|
|
71
|
+
await asyncio.sleep(seconds_per_attempt)
|
|
72
|
+
|
|
73
|
+
LOGGER.error("DNS resolution failed after %s attempts on %s", attempts, host_to_test_dns)
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
async def warp_cli(*args: str) -> str:
|
|
78
|
+
proc = await asyncio.create_subprocess_exec(
|
|
79
|
+
"warp-cli",
|
|
80
|
+
"--accept-tos",
|
|
81
|
+
*args,
|
|
82
|
+
stderr=asyncio.subprocess.PIPE,
|
|
83
|
+
stdout=asyncio.subprocess.PIPE,
|
|
84
|
+
stdin=asyncio.subprocess.DEVNULL,
|
|
85
|
+
)
|
|
86
|
+
try:
|
|
87
|
+
stdout, stderr = await asyncio.wait_for(proc.communicate(), 30)
|
|
88
|
+
except (asyncio.exceptions.TimeoutError, OSError) as ex:
|
|
89
|
+
msg = "Failed to run command"
|
|
90
|
+
raise WarpError(msg) from ex
|
|
91
|
+
|
|
92
|
+
if proc.returncode != 0:
|
|
93
|
+
msg = stderr.decode().strip()
|
|
94
|
+
raise WarpError(msg)
|
|
95
|
+
|
|
96
|
+
return stdout.decode().strip()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
async def warp_cli_status() -> str:
|
|
100
|
+
return await warp_cli("status")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
async def warp_cli_connect() -> None:
|
|
104
|
+
response = await warp_cli("connect")
|
|
105
|
+
LOGGER.info("Connect: %s", response)
|
|
106
|
+
await asyncio.sleep(CONFIG_DELAY)
|
|
107
|
+
if not await is_dns_ready(host_to_test_dns=DOMAIN_TO_TEST_DNS):
|
|
108
|
+
msg = "Failed to resolve DNS"
|
|
109
|
+
raise WarpError(msg)
|
|
110
|
+
|
|
111
|
+
LOGGER.info("Connected. Status: %s", await warp_cli_status())
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
async def warp_cli_disconnect() -> None:
|
|
115
|
+
response = await warp_cli("disconnect")
|
|
116
|
+
LOGGER.info("Disconnect: %s", response)
|
|
117
|
+
await asyncio.sleep(CONFIG_DELAY)
|
|
118
|
+
LOGGER.info("Disconnected. Status: %s", await warp_cli_status())
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
async def warp_cli_get_virtual_network_id(vnet_name: str) -> str:
|
|
122
|
+
vnet_id_match = re.search(
|
|
123
|
+
f"ID: (.*)\n Name: {vnet_name}\n",
|
|
124
|
+
await warp_cli("vnet"),
|
|
125
|
+
)
|
|
126
|
+
if not vnet_id_match:
|
|
127
|
+
msg = f"Failed to find virtual network {vnet_name}"
|
|
128
|
+
raise WarpError(msg)
|
|
129
|
+
|
|
130
|
+
return vnet_id_match.groups()[0]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
async def warp_cli_set_virtual_network(vnet_name: str) -> None:
|
|
134
|
+
vnet_id = await warp_cli_get_virtual_network_id(vnet_name)
|
|
135
|
+
await warp_cli("vnet", vnet_id)
|
|
136
|
+
await asyncio.sleep(CONFIG_DELAY)
|
|
137
|
+
LOGGER.info(
|
|
138
|
+
"Setup virtual network. Name: %s, Network ID: %s, Status: %s",
|
|
139
|
+
vnet_name,
|
|
140
|
+
vnet_id,
|
|
141
|
+
await warp_cli_status(),
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _resolve_host(host: str) -> str:
|
|
146
|
+
try:
|
|
147
|
+
return socket.gethostbyname(host)
|
|
148
|
+
except socket.gaierror:
|
|
149
|
+
return host
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
async def _ip_route_get(host: str) -> tuple[bytes, bytes]:
|
|
153
|
+
target = _resolve_host(host)
|
|
154
|
+
proc = await asyncio.create_subprocess_exec(
|
|
155
|
+
"ip",
|
|
156
|
+
"route",
|
|
157
|
+
"get",
|
|
158
|
+
target,
|
|
159
|
+
stdout=asyncio.subprocess.PIPE,
|
|
160
|
+
stderr=asyncio.subprocess.PIPE,
|
|
161
|
+
stdin=asyncio.subprocess.DEVNULL,
|
|
162
|
+
)
|
|
163
|
+
try:
|
|
164
|
+
stdout, stderr = await asyncio.wait_for(proc.communicate(), 5)
|
|
165
|
+
except asyncio.exceptions.TimeoutError as ex:
|
|
166
|
+
msg = "Timeout - Failed to retrieve route"
|
|
167
|
+
raise WarpError(msg) from ex
|
|
168
|
+
|
|
169
|
+
if proc.returncode != 0:
|
|
170
|
+
msg = stderr.decode()
|
|
171
|
+
raise WarpError(msg)
|
|
172
|
+
|
|
173
|
+
return stdout, stderr
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
async def is_using_split_tunnel(host: str) -> bool:
|
|
177
|
+
try:
|
|
178
|
+
stdout, _ = await _ip_route_get(host)
|
|
179
|
+
LOGGER.info("Route command for '%s': %s", host, stdout.decode().replace("\n", " "))
|
|
180
|
+
except WarpError:
|
|
181
|
+
LOGGER.exception("Error getting IP route in split tunnel")
|
|
182
|
+
return False
|
|
183
|
+
else:
|
|
184
|
+
return b"CloudflareWARP" in stdout
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: fluidattacks_core_git
|
|
3
|
+
Version: 12.0.0
|
|
4
|
+
Summary: Fluid Attacks Core Git Library
|
|
5
|
+
Author-email: Development <development@fluidattacks.com>
|
|
6
|
+
License: MPL-2.0
|
|
7
|
+
Classifier: Development Status :: 1 - Planning
|
|
8
|
+
Classifier: Intended Audience :: Developers
|
|
9
|
+
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Requires-Dist: aiofiles>=23.2.1
|
|
15
|
+
Requires-Dist: aiohttp>=3.10.0
|
|
16
|
+
Requires-Dist: anyio>=4.7.0
|
|
17
|
+
Requires-Dist: boto3>=1.34
|
|
18
|
+
Requires-Dist: botocore>=1.40.18
|
|
19
|
+
Requires-Dist: fluidattacks-core-http>=12.0.0
|
|
20
|
+
Requires-Dist: gitpython>=3.1.41
|
|
21
|
+
Requires-Dist: pathspec>=0.12.1
|
|
22
|
+
Requires-Dist: urllib3>=2.6.1
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
fluidattacks_core/git/__init__.py,sha256=Z3ABS9a3nxJG2WJq8IurFenzcwPlOZAGc8GGSs9fZws,10055
|
|
2
|
+
fluidattacks_core/git/classes.py,sha256=vgCVOUF6tqeW0lKtD9giCNFQtzRit44bnu5qOAx7qCI,579
|
|
3
|
+
fluidattacks_core/git/clone.py,sha256=hmCAhqE-hNX7KOSJahmEmnFqrhrD_WMf_7PEUZ6SO-U,6866
|
|
4
|
+
fluidattacks_core/git/codecommit_utils.py,sha256=P9vTk9wj0sdubOSZvfFodLJYPOMJmXQkd9T8jw-jU5k,3212
|
|
5
|
+
fluidattacks_core/git/constants.py,sha256=ntIdf_zQDGZfjahUpjASS-Ry9ZE8iDaEURCPBhEPebk,204
|
|
6
|
+
fluidattacks_core/git/delete_files.py,sha256=_8Cn9izhCc70RKBgFa1TBU9oTVtggg10mexxHn3IkIE,1547
|
|
7
|
+
fluidattacks_core/git/download_file.py,sha256=hPuZhDQaYoG0ROF__CSGVV0UUWOK3771AnU1xXtu-4o,1450
|
|
8
|
+
fluidattacks_core/git/download_repo.py,sha256=US4z1px9zUc33JDBaAxl85Ily6Ut4kMsbrrH0UFfv5k,4424
|
|
9
|
+
fluidattacks_core/git/https_utils.py,sha256=6dRDwaRyDVd3WDm0-XFYJUZMh3lCo1sQjcHcoP0Yekw,6615
|
|
10
|
+
fluidattacks_core/git/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
fluidattacks_core/git/remote.py,sha256=cPuyBMHeGrzRkEjroB6zlRLMA-QH2gIyIkGJNyf8wZc,1255
|
|
12
|
+
fluidattacks_core/git/show_file.py,sha256=zKsFH2JexD1gyeb4fPrPnUebnmc58b8eq2tPuhto7bQ,10902
|
|
13
|
+
fluidattacks_core/git/ssh_utils.py,sha256=3EdkRfWKIbb1-i2zgMogueTSZxAb8Ft7kQb4MbNcfg8,6178
|
|
14
|
+
fluidattacks_core/git/upload_repo.py,sha256=p2FGyDykHhY3Be1youwmfOmsf7M6a9pjGPJGtCWp5mo,10475
|
|
15
|
+
fluidattacks_core/git/utils.py,sha256=E614OPPkGowFjQpvm-pnmRPztnYm6xJQzNkjegkb5P0,3765
|
|
16
|
+
fluidattacks_core/git/warp.py,sha256=SzDtyYG3TM--Nrv2nl1DM23NmSQ1o-Pc26LOvwGIb4k,5651
|
|
17
|
+
fluidattacks_core_git-12.0.0.dist-info/METADATA,sha256=5uiUhLuWS0_lySl7cEgBpX46d2r_Dr7h80UE_aZTbrE,817
|
|
18
|
+
fluidattacks_core_git-12.0.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
19
|
+
fluidattacks_core_git-12.0.0.dist-info/RECORD,,
|