unitlab 2.3.0__py3-none-any.whl → 2.3.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- unitlab/client.py +400 -3
- unitlab/main.py +67 -1
- unitlab/tunnel_config.py +199 -0
- unitlab/tunnel_service_token.py +104 -0
- unitlab/utils.py +3 -0
- {unitlab-2.3.0.dist-info → unitlab-2.3.4.dist-info}/METADATA +6 -10
- unitlab-2.3.4.dist-info/RECORD +14 -0
- {unitlab-2.3.0.dist-info → unitlab-2.3.4.dist-info}/WHEEL +1 -1
- unitlab-2.3.0.dist-info/RECORD +0 -12
- {unitlab-2.3.0.dist-info/licenses → unitlab-2.3.4.dist-info}/LICENSE.md +0 -0
- {unitlab-2.3.0.dist-info → unitlab-2.3.4.dist-info}/entry_points.txt +0 -0
- {unitlab-2.3.0.dist-info → unitlab-2.3.4.dist-info}/top_level.txt +0 -0
unitlab/client.py
CHANGED
@@ -3,16 +3,30 @@ import glob
|
|
3
3
|
import logging
|
4
4
|
import os
|
5
5
|
import urllib.parse
|
6
|
-
|
7
6
|
import aiofiles
|
8
7
|
import aiohttp
|
9
8
|
import requests
|
10
9
|
import tqdm
|
11
|
-
|
10
|
+
import socket
|
11
|
+
import subprocess
|
12
|
+
import signal
|
13
|
+
import re
|
14
|
+
import time
|
15
|
+
import threading
|
16
|
+
import psutil
|
17
|
+
from datetime import datetime, timezone
|
18
|
+
from .tunnel_config import CloudflareTunnel
|
12
19
|
from .utils import get_api_url, handle_exceptions
|
13
20
|
|
14
|
-
logger = logging.getLogger(__name__)
|
15
21
|
|
22
|
+
try:
|
23
|
+
import GPUtil
|
24
|
+
HAS_GPU = True
|
25
|
+
except ImportError:
|
26
|
+
HAS_GPU = False
|
27
|
+
|
28
|
+
|
29
|
+
logger = logging.getLogger(__name__)
|
16
30
|
|
17
31
|
class UnitlabClient:
|
18
32
|
"""A client with a connection to the Unitlab.ai platform.
|
@@ -54,6 +68,20 @@ class UnitlabClient:
|
|
54
68
|
adapter = requests.adapters.HTTPAdapter(max_retries=3)
|
55
69
|
self.api_session.mount("http://", adapter)
|
56
70
|
self.api_session.mount("https://", adapter)
|
71
|
+
|
72
|
+
# Device agent attributes (initialized when needed)
|
73
|
+
self.device_id = None
|
74
|
+
self.base_domain = None
|
75
|
+
self.server_url = None
|
76
|
+
self.hostname = socket.gethostname()
|
77
|
+
self.tunnel_manager = None
|
78
|
+
self.jupyter_url = None
|
79
|
+
self.ssh_url = None
|
80
|
+
self.jupyter_proc = None
|
81
|
+
self.tunnel_proc = None
|
82
|
+
self.jupyter_port = None
|
83
|
+
self.running = True
|
84
|
+
self.metrics_thread = None
|
57
85
|
|
58
86
|
def close(self) -> None:
|
59
87
|
"""Close :class:`UnitlabClient` connections.
|
@@ -234,3 +262,372 @@ class UnitlabClient:
|
|
234
262
|
pbar.update(await f)
|
235
263
|
|
236
264
|
asyncio.run(main())
|
265
|
+
|
266
|
+
def initialize_device_agent(self, server_url: str, device_id: str, base_domain: str):
|
267
|
+
"""Initialize device agent configuration"""
|
268
|
+
self.server_url = server_url.rstrip('/')
|
269
|
+
self.device_id = device_id
|
270
|
+
self.base_domain = base_domain
|
271
|
+
|
272
|
+
# Initialize tunnel manager if available
|
273
|
+
if CloudflareTunnel:
|
274
|
+
self.tunnel_manager = CloudflareTunnel(base_domain, device_id)
|
275
|
+
self.jupyter_url = self.tunnel_manager.jupyter_url
|
276
|
+
self.ssh_url = self.tunnel_manager.ssh_url
|
277
|
+
else:
|
278
|
+
self.tunnel_manager = None
|
279
|
+
self.jupyter_url = f"https://jupyter-{device_id}.{base_domain}"
|
280
|
+
self.ssh_url = f"https://ssh-{device_id}.{base_domain}"
|
281
|
+
|
282
|
+
# Setup signal handlers
|
283
|
+
signal.signal(signal.SIGINT, self._handle_shutdown)
|
284
|
+
signal.signal(signal.SIGTERM, self._handle_shutdown)
|
285
|
+
|
286
|
+
def _handle_shutdown(self, signum, frame):
|
287
|
+
"""Handle shutdown signals"""
|
288
|
+
_ = frame # Unused but required by signal handler signature
|
289
|
+
logger.info(f"Received signal {signum}, shutting down...")
|
290
|
+
self.running = False
|
291
|
+
|
292
|
+
def _get_device_headers(self):
|
293
|
+
"""Get headers for device agent API requests"""
|
294
|
+
headers = {
|
295
|
+
'Content-Type': 'application/json',
|
296
|
+
'User-Agent': f'UnitlabDeviceAgent/{self.device_id}'
|
297
|
+
}
|
298
|
+
|
299
|
+
# Add API key if provided
|
300
|
+
if self.api_key:
|
301
|
+
headers['Authorization'] = f'Api-Key {self.api_key}'
|
302
|
+
|
303
|
+
return headers
|
304
|
+
|
305
|
+
def _post_device(self, endpoint, data=None):
|
306
|
+
"""Make authenticated POST request for device agent"""
|
307
|
+
full_url = urllib.parse.urljoin(self.server_url, endpoint)
|
308
|
+
logger.debug(f"Posting to {full_url} with data: {data}")
|
309
|
+
|
310
|
+
try:
|
311
|
+
response = self.api_session.post(
|
312
|
+
full_url,
|
313
|
+
json=data or {},
|
314
|
+
headers=self._get_device_headers(),
|
315
|
+
)
|
316
|
+
logger.debug(f"Response status: {response.status_code}, Response: {response.text}")
|
317
|
+
response.raise_for_status()
|
318
|
+
return response
|
319
|
+
except Exception as e:
|
320
|
+
logger.error(f"POST request failed to {full_url}: {e}")
|
321
|
+
raise
|
322
|
+
|
323
|
+
def start_jupyter(self) -> bool:
|
324
|
+
"""Start Jupyter notebook server"""
|
325
|
+
try:
|
326
|
+
logger.info("Starting Jupyter notebook...")
|
327
|
+
|
328
|
+
cmd = [
|
329
|
+
"jupyter", "notebook",
|
330
|
+
"--no-browser",
|
331
|
+
"--ServerApp.token=''",
|
332
|
+
"--ServerApp.password=''",
|
333
|
+
"--ServerApp.allow_origin='*'",
|
334
|
+
"--ServerApp.ip='0.0.0.0'"
|
335
|
+
]
|
336
|
+
|
337
|
+
self.jupyter_proc = subprocess.Popen(
|
338
|
+
cmd,
|
339
|
+
stdout=subprocess.PIPE,
|
340
|
+
stderr=subprocess.STDOUT,
|
341
|
+
text=True
|
342
|
+
)
|
343
|
+
|
344
|
+
# Wait for Jupyter to start and get the port
|
345
|
+
start_time = time.time()
|
346
|
+
while time.time() - start_time < 30:
|
347
|
+
line = self.jupyter_proc.stdout.readline()
|
348
|
+
if not line:
|
349
|
+
break
|
350
|
+
|
351
|
+
# Look for the port in the output
|
352
|
+
match = re.search(r'http://.*:(\d+)/', line)
|
353
|
+
if match:
|
354
|
+
self.jupyter_port = match.group(1)
|
355
|
+
logger.info(f"✅ Jupyter started on port {self.jupyter_port}")
|
356
|
+
return True
|
357
|
+
|
358
|
+
raise Exception("Timeout waiting for Jupyter to start")
|
359
|
+
|
360
|
+
except Exception as e:
|
361
|
+
logger.error(f"Failed to start Jupyter: {e}")
|
362
|
+
if self.jupyter_proc:
|
363
|
+
self.jupyter_proc.terminate()
|
364
|
+
self.jupyter_proc = None
|
365
|
+
return False
|
366
|
+
|
367
|
+
def setup_tunnels(self) -> bool:
|
368
|
+
"""Setup Cloudflare tunnels"""
|
369
|
+
try:
|
370
|
+
if not self.jupyter_port:
|
371
|
+
logger.error("Jupyter port not available")
|
372
|
+
return False
|
373
|
+
|
374
|
+
if not self.tunnel_manager:
|
375
|
+
logger.warning("CloudflareTunnel not available, skipping tunnel setup")
|
376
|
+
return True
|
377
|
+
|
378
|
+
logger.info("Setting up Cloudflare tunnels...")
|
379
|
+
self.tunnel_proc = self.tunnel_manager.setup(self.jupyter_port)
|
380
|
+
|
381
|
+
if self.tunnel_proc:
|
382
|
+
logger.info("✅ Tunnels established")
|
383
|
+
self.report_services()
|
384
|
+
return True
|
385
|
+
|
386
|
+
return False
|
387
|
+
|
388
|
+
except Exception as e:
|
389
|
+
logger.error(f"Tunnel setup failed: {e}")
|
390
|
+
return False
|
391
|
+
|
392
|
+
def check_ssh(self) -> bool:
|
393
|
+
"""Check if SSH service is available"""
|
394
|
+
try:
|
395
|
+
# Check if SSH is running
|
396
|
+
result = subprocess.run(
|
397
|
+
["systemctl", "is-active", "ssh"],
|
398
|
+
capture_output=True,
|
399
|
+
text=True,
|
400
|
+
timeout=5
|
401
|
+
)
|
402
|
+
|
403
|
+
if result.stdout.strip() == "active":
|
404
|
+
logger.info("✅ SSH service is active")
|
405
|
+
return True
|
406
|
+
else:
|
407
|
+
logger.warning("SSH service is not active")
|
408
|
+
# Try to start SSH
|
409
|
+
subprocess.run(["sudo", "systemctl", "start", "ssh"], timeout=10)
|
410
|
+
time.sleep(2)
|
411
|
+
return False
|
412
|
+
|
413
|
+
except Exception as e:
|
414
|
+
logger.error(f"SSH check failed: {e}")
|
415
|
+
return False
|
416
|
+
|
417
|
+
def report_services(self):
|
418
|
+
"""Report services to the server"""
|
419
|
+
try:
|
420
|
+
# Report Jupyter service
|
421
|
+
jupyter_data = {
|
422
|
+
'service_type': 'jupyter',
|
423
|
+
'service_name': f'jupyter-{self.device_id}',
|
424
|
+
'local_port': int(self.jupyter_port) if self.jupyter_port else 8888,
|
425
|
+
'tunnel_url': self.jupyter_url,
|
426
|
+
'status': 'online'
|
427
|
+
}
|
428
|
+
|
429
|
+
logger.info(f"Reporting Jupyter service with URL: {self.jupyter_url}")
|
430
|
+
jupyter_response = self._post_device(
|
431
|
+
f"/api/tunnel/agent/jupyter/{self.device_id}/",
|
432
|
+
jupyter_data
|
433
|
+
)
|
434
|
+
logger.info(f"Reported Jupyter service: {jupyter_response.status_code if hasattr(jupyter_response, 'status_code') else jupyter_response}")
|
435
|
+
|
436
|
+
# Report SSH service (always report, even if SSH is not running locally)
|
437
|
+
# Remove https:// prefix for SSH hostname
|
438
|
+
ssh_hostname = self.ssh_url.replace('https://', '')
|
439
|
+
|
440
|
+
# Get current system username
|
441
|
+
import getpass
|
442
|
+
current_user = getpass.getuser()
|
443
|
+
|
444
|
+
# Create SSH connection command
|
445
|
+
ssh_connection_cmd = f"ssh -o ProxyCommand='cloudflared access ssh --hostname {ssh_hostname}' {current_user}@{ssh_hostname}"
|
446
|
+
|
447
|
+
# Check if SSH is available
|
448
|
+
ssh_available = self.check_ssh()
|
449
|
+
|
450
|
+
ssh_data = {
|
451
|
+
'service_type': 'ssh',
|
452
|
+
'service_name': f'ssh-{self.device_id}',
|
453
|
+
'local_port': 22,
|
454
|
+
'tunnel_url': ssh_connection_cmd, # Send the SSH command instead of URL
|
455
|
+
'status': 'online' if ssh_available else 'offline'
|
456
|
+
}
|
457
|
+
|
458
|
+
logger.info(f"Reporting SSH service with command: {ssh_connection_cmd}")
|
459
|
+
ssh_response = self._post_device(
|
460
|
+
f"/api/tunnel/agent/ssh/{self.device_id}/",
|
461
|
+
ssh_data
|
462
|
+
)
|
463
|
+
logger.info(f"Reported SSH service: {ssh_response.status_code if hasattr(ssh_response, 'status_code') else ssh_response}")
|
464
|
+
|
465
|
+
except Exception as e:
|
466
|
+
logger.error(f"Failed to report services: {e}", exc_info=True)
|
467
|
+
|
468
|
+
def collect_metrics(self) -> dict:
|
469
|
+
"""Collect system metrics"""
|
470
|
+
metrics = {}
|
471
|
+
|
472
|
+
# CPU metrics
|
473
|
+
metrics['cpu'] = {
|
474
|
+
'percent': psutil.cpu_percent(interval=1),
|
475
|
+
'count': psutil.cpu_count(),
|
476
|
+
'timestamp': datetime.now(timezone.utc).isoformat()
|
477
|
+
}
|
478
|
+
|
479
|
+
# Memory metrics
|
480
|
+
mem = psutil.virtual_memory()
|
481
|
+
metrics['ram'] = {
|
482
|
+
'total': mem.total,
|
483
|
+
'used': mem.used,
|
484
|
+
'available': mem.available,
|
485
|
+
'percent': mem.percent,
|
486
|
+
'timestamp': datetime.now(timezone.utc).isoformat()
|
487
|
+
}
|
488
|
+
|
489
|
+
# GPU metrics (if available)
|
490
|
+
if HAS_GPU:
|
491
|
+
try:
|
492
|
+
gpus = GPUtil.getGPUs()
|
493
|
+
if gpus:
|
494
|
+
gpu = gpus[0]
|
495
|
+
metrics['gpu'] = {
|
496
|
+
'name': gpu.name,
|
497
|
+
'load': gpu.load * 100,
|
498
|
+
'memory_used': gpu.memoryUsed,
|
499
|
+
'memory_total': gpu.memoryTotal,
|
500
|
+
'temperature': gpu.temperature,
|
501
|
+
'timestamp': datetime.now(timezone.utc).isoformat()
|
502
|
+
}
|
503
|
+
except Exception as e:
|
504
|
+
logger.debug(f"GPU metrics unavailable: {e}")
|
505
|
+
|
506
|
+
return metrics
|
507
|
+
|
508
|
+
def send_metrics(self):
|
509
|
+
"""Send metrics to server"""
|
510
|
+
try:
|
511
|
+
metrics = self.collect_metrics()
|
512
|
+
|
513
|
+
# Send CPU metrics
|
514
|
+
if 'cpu' in metrics:
|
515
|
+
self._post_device(f"/api/tunnel/agent/cpu/{self.device_id}/", metrics['cpu'])
|
516
|
+
|
517
|
+
# Send RAM metrics
|
518
|
+
if 'ram' in metrics:
|
519
|
+
self._post_device(f"/api/tunnel/agent/ram/{self.device_id}/", metrics['ram'])
|
520
|
+
|
521
|
+
# Send GPU metrics if available
|
522
|
+
if 'gpu' in metrics and metrics['gpu']:
|
523
|
+
self._post_device(f"/api/tunnel/agent/gpu/{self.device_id}/", metrics['gpu'])
|
524
|
+
|
525
|
+
logger.debug(f"Metrics sent - CPU: {metrics['cpu']['percent']:.1f}%, RAM: {metrics['ram']['percent']:.1f}%")
|
526
|
+
|
527
|
+
except Exception as e:
|
528
|
+
logger.error(f"Failed to send metrics: {e}")
|
529
|
+
|
530
|
+
def metrics_loop(self):
|
531
|
+
"""Background thread for sending metrics"""
|
532
|
+
logger.info("Starting metrics thread")
|
533
|
+
|
534
|
+
while self.running:
|
535
|
+
try:
|
536
|
+
self.send_metrics()
|
537
|
+
|
538
|
+
# Check if processes are still running
|
539
|
+
if self.jupyter_proc and self.jupyter_proc.poll() is not None:
|
540
|
+
logger.warning("Jupyter process died")
|
541
|
+
self.jupyter_proc = None
|
542
|
+
|
543
|
+
if self.tunnel_proc and self.tunnel_proc.poll() is not None:
|
544
|
+
logger.warning("Tunnel process died")
|
545
|
+
self.tunnel_proc = None
|
546
|
+
|
547
|
+
except Exception as e:
|
548
|
+
logger.error(f"Metrics loop error: {e}")
|
549
|
+
|
550
|
+
# Wait for next interval (default 5 seconds)
|
551
|
+
for _ in range(3):
|
552
|
+
if not self.running:
|
553
|
+
break
|
554
|
+
time.sleep(1)
|
555
|
+
|
556
|
+
logger.info("Metrics thread stopped")
|
557
|
+
|
558
|
+
def run_device_agent(self):
|
559
|
+
"""Main run method for device agent"""
|
560
|
+
logger.info("=" * 50)
|
561
|
+
logger.info("Starting Device Agent")
|
562
|
+
logger.info(f"Device ID: {self.device_id}")
|
563
|
+
logger.info(f"Server: {self.server_url}")
|
564
|
+
logger.info(f"Domain: {self.base_domain}")
|
565
|
+
logger.info("=" * 50)
|
566
|
+
|
567
|
+
# Check SSH
|
568
|
+
self.check_ssh()
|
569
|
+
|
570
|
+
# Start Jupyter
|
571
|
+
if not self.start_jupyter():
|
572
|
+
logger.error("Failed to start Jupyter")
|
573
|
+
return
|
574
|
+
|
575
|
+
# Wait a moment for Jupyter to fully initialize
|
576
|
+
time.sleep(1)
|
577
|
+
|
578
|
+
# Setup tunnels
|
579
|
+
if not self.setup_tunnels():
|
580
|
+
logger.error("Failed to setup tunnels")
|
581
|
+
self.cleanup_device_agent()
|
582
|
+
return
|
583
|
+
|
584
|
+
# Print access information
|
585
|
+
logger.info("=" * 50)
|
586
|
+
logger.info("🎉 All services started successfully!")
|
587
|
+
logger.info(f"📔 Jupyter: {self.jupyter_url}")
|
588
|
+
logger.info(f"🔐 SSH: {self.ssh_url}")
|
589
|
+
# Remove https:// prefix for SSH command display
|
590
|
+
ssh_hostname = self.ssh_url.replace('https://', '')
|
591
|
+
import getpass
|
592
|
+
current_user = getpass.getuser()
|
593
|
+
logger.info(f"🔐 SSH Command: ssh -o ProxyCommand='cloudflared access ssh --hostname {ssh_hostname}' {current_user}@{ssh_hostname}")
|
594
|
+
logger.info("=" * 50)
|
595
|
+
|
596
|
+
# Start metrics thread
|
597
|
+
self.metrics_thread = threading.Thread(target=self.metrics_loop, daemon=True)
|
598
|
+
self.metrics_thread.start()
|
599
|
+
|
600
|
+
# Main loop
|
601
|
+
try:
|
602
|
+
while self.running:
|
603
|
+
time.sleep(1)
|
604
|
+
except KeyboardInterrupt:
|
605
|
+
logger.info("Interrupted by user")
|
606
|
+
|
607
|
+
self.cleanup_device_agent()
|
608
|
+
|
609
|
+
def cleanup_device_agent(self):
|
610
|
+
"""Clean up device agent resources"""
|
611
|
+
logger.info("Cleaning up...")
|
612
|
+
|
613
|
+
self.running = False
|
614
|
+
|
615
|
+
# Stop Jupyter
|
616
|
+
if self.jupyter_proc:
|
617
|
+
logger.info("Stopping Jupyter...")
|
618
|
+
self.jupyter_proc.terminate()
|
619
|
+
try:
|
620
|
+
self.jupyter_proc.wait(timeout=5)
|
621
|
+
except subprocess.TimeoutExpired:
|
622
|
+
self.jupyter_proc.kill()
|
623
|
+
|
624
|
+
# Stop tunnel
|
625
|
+
if self.tunnel_proc:
|
626
|
+
logger.info("Stopping tunnel...")
|
627
|
+
self.tunnel_proc.terminate()
|
628
|
+
try:
|
629
|
+
self.tunnel_proc.wait(timeout=5)
|
630
|
+
except subprocess.TimeoutExpired:
|
631
|
+
self.tunnel_proc.kill()
|
632
|
+
|
633
|
+
logger.info("Cleanup complete")
|
unitlab/main.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
from enum import Enum
|
2
2
|
from pathlib import Path
|
3
3
|
from uuid import UUID
|
4
|
+
import logging
|
5
|
+
import os
|
4
6
|
|
5
7
|
import typer
|
6
8
|
import validators
|
@@ -9,12 +11,16 @@ from typing_extensions import Annotated
|
|
9
11
|
from . import utils
|
10
12
|
from .client import UnitlabClient
|
11
13
|
|
14
|
+
|
12
15
|
app = typer.Typer()
|
13
16
|
project_app = typer.Typer()
|
14
17
|
dataset_app = typer.Typer()
|
18
|
+
agent_app = typer.Typer()
|
19
|
+
|
15
20
|
|
16
21
|
app.add_typer(project_app, name="project", help="Project commands")
|
17
22
|
app.add_typer(dataset_app, name="dataset", help="Dataset commands")
|
23
|
+
app.add_typer(agent_app, name="agent", help="Agent commands")
|
18
24
|
|
19
25
|
|
20
26
|
API_KEY = Annotated[
|
@@ -42,7 +48,7 @@ class AnnotationType(str, Enum):
|
|
42
48
|
@app.command(help="Configure the credentials")
|
43
49
|
def configure(
|
44
50
|
api_key: Annotated[str, typer.Option(help="The api-key obtained from unitlab.ai")],
|
45
|
-
api_url: Annotated[str, typer.Option()] = "https://
|
51
|
+
api_url: Annotated[str, typer.Option()] = "https://localhost/",
|
46
52
|
):
|
47
53
|
if not validators.url(api_url, simple_host=True):
|
48
54
|
raise typer.BadParameter("Invalid api url")
|
@@ -105,5 +111,65 @@ def dataset_download(
|
|
105
111
|
get_client(api_key).dataset_download_files(pk)
|
106
112
|
|
107
113
|
|
114
|
+
def send_metrics_to_server(server_url: str, device_id: str, metrics: dict):
|
115
|
+
"""Standalone function to send metrics to server using client"""
|
116
|
+
client = UnitlabClient(api_key="dummy") # API key not needed for metrics
|
117
|
+
return client.send_metrics_to_server(server_url, device_id, metrics)
|
118
|
+
|
119
|
+
|
120
|
+
def send_metrics_into_server():
|
121
|
+
"""Standalone function to collect system metrics using client"""
|
122
|
+
client = UnitlabClient(api_key="dummy") # API key not needed for metrics
|
123
|
+
return client.collect_system_metrics()
|
124
|
+
|
125
|
+
|
126
|
+
@agent_app.command(name="run", help="Run the device agent with Jupyter, SSH tunnels and metrics")
|
127
|
+
def run_agent(
|
128
|
+
api_key: str,
|
129
|
+
device_id: Annotated[str, typer.Option(help="Device ID")] = None,
|
130
|
+
base_domain: Annotated[str, typer.Option(help="Base domain for tunnels")] = "1scan.uz",
|
131
|
+
|
132
|
+
):
|
133
|
+
"""Run the full device agent with Jupyter, SSH tunnels and metrics reporting"""
|
134
|
+
|
135
|
+
# Setup logging
|
136
|
+
logging.basicConfig(
|
137
|
+
level=logging.INFO,
|
138
|
+
format='%(asctime)s - %(levelname)s - %(message)s',
|
139
|
+
handlers=[logging.StreamHandler()]
|
140
|
+
)
|
141
|
+
|
142
|
+
# Get server URL from environment or use default
|
143
|
+
server_url = 'https://api-dev.unitlab.ai/'
|
144
|
+
|
145
|
+
# Generate unique device ID if not provided
|
146
|
+
if not device_id:
|
147
|
+
import uuid
|
148
|
+
import platform
|
149
|
+
# Try environment variable first
|
150
|
+
device_id = os.getenv('DEVICE_ID')
|
151
|
+
if not device_id:
|
152
|
+
# Generate a unique ID based on hostname and random UUID
|
153
|
+
hostname = platform.node().replace('.', '-').replace(' ', '-')[:20]
|
154
|
+
random_suffix = str(uuid.uuid4())[:8]
|
155
|
+
device_id = f"{hostname}-{random_suffix}"
|
156
|
+
|
157
|
+
|
158
|
+
# Create client and initialize device agent
|
159
|
+
client = UnitlabClient(api_key=api_key)
|
160
|
+
client.initialize_device_agent(
|
161
|
+
server_url=server_url,
|
162
|
+
device_id=device_id,
|
163
|
+
base_domain=base_domain
|
164
|
+
)
|
165
|
+
|
166
|
+
try:
|
167
|
+
client.run_device_agent()
|
168
|
+
except Exception as e:
|
169
|
+
logging.error(f"Fatal error: {e}")
|
170
|
+
client.cleanup_device_agent()
|
171
|
+
raise typer.Exit(1)
|
172
|
+
|
173
|
+
|
108
174
|
if __name__ == "__main__":
|
109
175
|
app()
|
unitlab/tunnel_config.py
ADDED
@@ -0,0 +1,199 @@
|
|
1
|
+
"""
|
2
|
+
Cloudflare Tunnel Configuration using Service Token
|
3
|
+
No user login required - uses pre-generated service token
|
4
|
+
"""
|
5
|
+
|
6
|
+
import os
|
7
|
+
import subprocess
|
8
|
+
import socket
|
9
|
+
import time
|
10
|
+
import logging
|
11
|
+
|
12
|
+
logger = logging.getLogger(__name__)
|
13
|
+
|
14
|
+
|
15
|
+
class CloudflareTunnel:
|
16
|
+
def __init__(self, base_domain, device_id): # base_domain kept for compatibility
|
17
|
+
"""
|
18
|
+
Initialize tunnel with service token
|
19
|
+
No login or credential files needed
|
20
|
+
"""
|
21
|
+
# Configuration
|
22
|
+
self.base_domain = "1scan.uz" # Hardcoded domain
|
23
|
+
self.device_id = device_id
|
24
|
+
self.hostname = socket.gethostname()
|
25
|
+
|
26
|
+
# Service token - replace with your actual token from cloudflared tunnel token command
|
27
|
+
# This token can ONLY run the tunnel, cannot modify or delete it
|
28
|
+
# To generate: cloudflared tunnel token unitlab-shared
|
29
|
+
self.service_token = os.getenv(
|
30
|
+
"CLOUDFLARE_TUNNEL_TOKEN",
|
31
|
+
"eyJhIjoiYzkxMTkyYWUyMGE1ZDQzZjY1ZTA4NzU1MGQ4ZGM4OWIiLCJzIjoiZmdnSHowbFJFRnBHa05TZzIzV3JKMVBiaDVROGVUd0oyYWtJWThXdjhtTT0iLCJ0IjoiYjMzZGFhOGYtMmNjMy00Y2FkLWEyMjgtOTdlMDYwNzBlNjAwIn0=" # Replace this with your actual token from step 1
|
32
|
+
)
|
33
|
+
|
34
|
+
if self.service_token == "YOUR_SERVICE_TOKEN_HERE":
|
35
|
+
logger.warning(
|
36
|
+
"⚠️ No service token configured. "
|
37
|
+
"Set CLOUDFLARE_TUNNEL_TOKEN env var or update the token in tunnel_config.py"
|
38
|
+
)
|
39
|
+
|
40
|
+
# Subdomain names
|
41
|
+
self.jupyter_subdomain = f"jupyter-{device_id}"
|
42
|
+
self.ssh_subdomain = f"ssh-{device_id}"
|
43
|
+
|
44
|
+
# Full URLs
|
45
|
+
self.jupyter_url = f"https://{self.jupyter_subdomain}.{self.base_domain}"
|
46
|
+
self.ssh_url = f"https://{self.ssh_subdomain}.{self.base_domain}"
|
47
|
+
|
48
|
+
self.tunnel_process = None
|
49
|
+
|
50
|
+
def check_cloudflared_installed(self):
|
51
|
+
"""Check if cloudflared is installed"""
|
52
|
+
try:
|
53
|
+
result = subprocess.run(
|
54
|
+
["cloudflared", "--version"],
|
55
|
+
capture_output=True,
|
56
|
+
text=True
|
57
|
+
)
|
58
|
+
return result.returncode == 0
|
59
|
+
except FileNotFoundError:
|
60
|
+
return False
|
61
|
+
|
62
|
+
def install_cloudflared(self):
|
63
|
+
"""Auto-install cloudflared if not present"""
|
64
|
+
import platform
|
65
|
+
|
66
|
+
system = platform.system().lower()
|
67
|
+
machine = platform.machine().lower()
|
68
|
+
|
69
|
+
print("📦 Installing cloudflared...")
|
70
|
+
|
71
|
+
try:
|
72
|
+
if system == "linux":
|
73
|
+
# Determine architecture
|
74
|
+
if machine in ["x86_64", "amd64"]:
|
75
|
+
arch = "amd64"
|
76
|
+
elif machine in ["aarch64", "arm64"]:
|
77
|
+
arch = "arm64"
|
78
|
+
else:
|
79
|
+
arch = "386"
|
80
|
+
|
81
|
+
# Download and install
|
82
|
+
url = f"https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-{arch}"
|
83
|
+
|
84
|
+
commands = [
|
85
|
+
f"curl -L {url} -o /tmp/cloudflared",
|
86
|
+
"chmod +x /tmp/cloudflared",
|
87
|
+
"sudo mv /tmp/cloudflared /usr/local/bin/cloudflared 2>/dev/null || "
|
88
|
+
"mkdir -p ~/.local/bin && mv /tmp/cloudflared ~/.local/bin/cloudflared"
|
89
|
+
]
|
90
|
+
|
91
|
+
for cmd in commands:
|
92
|
+
subprocess.run(cmd, shell=True, check=False)
|
93
|
+
|
94
|
+
# Add ~/.local/bin to PATH if needed
|
95
|
+
local_bin = os.path.expanduser("~/.local/bin")
|
96
|
+
if local_bin not in os.environ.get("PATH", ""):
|
97
|
+
os.environ["PATH"] = f"{local_bin}:{os.environ['PATH']}"
|
98
|
+
|
99
|
+
return self.check_cloudflared_installed()
|
100
|
+
|
101
|
+
elif system == "darwin":
|
102
|
+
# Try homebrew first
|
103
|
+
result = subprocess.run(["brew", "install", "cloudflared"], capture_output=True)
|
104
|
+
if result.returncode != 0:
|
105
|
+
# Fallback to direct download
|
106
|
+
url = "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-darwin-amd64.tgz"
|
107
|
+
subprocess.run(f"curl -L {url} | tar xz", shell=True)
|
108
|
+
subprocess.run("sudo mv cloudflared /usr/local/bin/", shell=True)
|
109
|
+
|
110
|
+
return self.check_cloudflared_installed()
|
111
|
+
|
112
|
+
elif system == "windows":
|
113
|
+
print("⚠️ Please install cloudflared manually on Windows")
|
114
|
+
print(" Download from: https://github.com/cloudflare/cloudflared/releases")
|
115
|
+
return False
|
116
|
+
|
117
|
+
except Exception as e:
|
118
|
+
logger.error(f"Failed to install cloudflared: {e}")
|
119
|
+
return False
|
120
|
+
|
121
|
+
def setup(self, jupyter_port): # jupyter_port kept for compatibility
|
122
|
+
"""
|
123
|
+
Setup and start tunnel with service token
|
124
|
+
No login required!
|
125
|
+
"""
|
126
|
+
print("🚀 Setting up Cloudflare tunnel with service token...")
|
127
|
+
|
128
|
+
# Check if cloudflared is installed
|
129
|
+
if not self.check_cloudflared_installed():
|
130
|
+
print("cloudflared not found, attempting to install...")
|
131
|
+
if not self.install_cloudflared():
|
132
|
+
print("❌ Failed to install cloudflared. Please install it manually:")
|
133
|
+
print(" https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation/")
|
134
|
+
return None
|
135
|
+
|
136
|
+
# Start tunnel with service token
|
137
|
+
return self.start_tunnel_with_token()
|
138
|
+
|
139
|
+
def start_tunnel_with_token(self):
|
140
|
+
"""
|
141
|
+
Start the tunnel using service token
|
142
|
+
This is much simpler than the credential file approach
|
143
|
+
"""
|
144
|
+
try:
|
145
|
+
print("🚀 Starting Cloudflare tunnel...")
|
146
|
+
|
147
|
+
# Simple command with service token
|
148
|
+
cmd = [
|
149
|
+
"cloudflared",
|
150
|
+
"tunnel",
|
151
|
+
"--no-autoupdate", # Prevent auto-updates during run
|
152
|
+
"run",
|
153
|
+
"--token",
|
154
|
+
self.service_token
|
155
|
+
]
|
156
|
+
|
157
|
+
# Start the tunnel process
|
158
|
+
self.tunnel_process = subprocess.Popen(
|
159
|
+
cmd,
|
160
|
+
stdout=subprocess.PIPE,
|
161
|
+
stderr=subprocess.STDOUT,
|
162
|
+
text=True,
|
163
|
+
bufsize=1
|
164
|
+
)
|
165
|
+
|
166
|
+
# Wait for tunnel to establish
|
167
|
+
print("⏳ Waiting for tunnel to connect...")
|
168
|
+
time.sleep(5)
|
169
|
+
|
170
|
+
# Check if process is still running
|
171
|
+
if self.tunnel_process.poll() is None:
|
172
|
+
print("✅ Tunnel is running!")
|
173
|
+
print(f"📌 Jupyter URL: {self.jupyter_url}")
|
174
|
+
print(f"📌 SSH URL: {self.ssh_url}")
|
175
|
+
return self.tunnel_process
|
176
|
+
else:
|
177
|
+
# Read any error output
|
178
|
+
output = self.tunnel_process.stdout.read()
|
179
|
+
print("❌ Tunnel failed to start")
|
180
|
+
print(f"Error output: {output}")
|
181
|
+
return None
|
182
|
+
|
183
|
+
except Exception as e:
|
184
|
+
print(f"❌ Error starting tunnel: {e}")
|
185
|
+
return None
|
186
|
+
|
187
|
+
def stop(self):
|
188
|
+
"""Stop the tunnel if running"""
|
189
|
+
if self.tunnel_process and self.tunnel_process.poll() is None:
|
190
|
+
print("Stopping tunnel...")
|
191
|
+
self.tunnel_process.terminate()
|
192
|
+
self.tunnel_process.wait(timeout=5)
|
193
|
+
print("Tunnel stopped")
|
194
|
+
|
195
|
+
# Removed all the old methods that are no longer needed:
|
196
|
+
# - login() - not needed with service token
|
197
|
+
# - create_tunnel() - tunnel already exists
|
198
|
+
# - configure_dns() - already configured
|
199
|
+
# - create_config_file() - not needed with service token
|
@@ -0,0 +1,104 @@
|
|
1
|
+
"""
|
2
|
+
Service Token implementation for Cloudflare Tunnel
|
3
|
+
More secure than embedding full credentials
|
4
|
+
"""
|
5
|
+
|
6
|
+
import os
|
7
|
+
import subprocess
|
8
|
+
import time
|
9
|
+
from pathlib import Path
|
10
|
+
|
11
|
+
|
12
|
+
class ServiceTokenTunnel:
|
13
|
+
"""
|
14
|
+
Use Cloudflare Service Token instead of credentials file
|
15
|
+
This is more secure and doesn't require login
|
16
|
+
"""
|
17
|
+
|
18
|
+
# Embed the service token (generated once by admin)
|
19
|
+
# This token can ONLY run the tunnel, cannot modify it
|
20
|
+
DEFAULT_SERVICE_TOKEN = "eyJhIjoiYzkxMTkyYWUyMGE1ZDQzZjY1ZTA4NzU1MGQ4ZGM4OWIiLCJzIjoiZmdnSHowbFJFRnBHa05TZzIzV3JKMVBiaDVROGVUd0oyYWtJWThXdjhtTT0iLCJ0IjoiYjMzZGFhOGYtMmNjMy00Y2FkLWEyMjgtOTdlMDYwNzBlNjAwIn0="
|
21
|
+
|
22
|
+
def __init__(self, device_id, service_token=None):
|
23
|
+
self.device_id = device_id
|
24
|
+
# Allow override via environment variable or parameter
|
25
|
+
self.service_token = (
|
26
|
+
service_token or
|
27
|
+
os.getenv("CLOUDFLARE_TUNNEL_TOKEN") or
|
28
|
+
self.DEFAULT_SERVICE_TOKEN
|
29
|
+
)
|
30
|
+
|
31
|
+
# With service token, we don't need credential files
|
32
|
+
# The token contains all necessary information
|
33
|
+
|
34
|
+
def start_tunnel_with_token(self):
|
35
|
+
"""
|
36
|
+
Start tunnel using service token
|
37
|
+
No login, no credential files needed
|
38
|
+
"""
|
39
|
+
print("🚀 Starting tunnel with service token...")
|
40
|
+
|
41
|
+
# Service token method - super simple!
|
42
|
+
cmd = [
|
43
|
+
"cloudflared",
|
44
|
+
"tunnel",
|
45
|
+
"run",
|
46
|
+
"--token",
|
47
|
+
self.service_token
|
48
|
+
]
|
49
|
+
|
50
|
+
try:
|
51
|
+
# Start the tunnel
|
52
|
+
process = subprocess.Popen(
|
53
|
+
cmd,
|
54
|
+
stdout=subprocess.PIPE,
|
55
|
+
stderr=subprocess.PIPE,
|
56
|
+
text=True
|
57
|
+
)
|
58
|
+
|
59
|
+
# Wait for tunnel to establish
|
60
|
+
time.sleep(3)
|
61
|
+
|
62
|
+
if process.poll() is None:
|
63
|
+
print("✅ Tunnel running with service token")
|
64
|
+
return process
|
65
|
+
else:
|
66
|
+
print("❌ Failed to start tunnel")
|
67
|
+
return None
|
68
|
+
|
69
|
+
except Exception as e:
|
70
|
+
print(f"❌ Error: {e}")
|
71
|
+
return None
|
72
|
+
|
73
|
+
def get_tunnel_info(self):
|
74
|
+
"""
|
75
|
+
Service tokens use predetermined URLs
|
76
|
+
The admin sets these up when creating the tunnel
|
77
|
+
"""
|
78
|
+
# These URLs are configured when creating the tunnel
|
79
|
+
base_domain = "1scan.uz"
|
80
|
+
return {
|
81
|
+
"jupyter_url": f"https://jupyter-{self.device_id}.{base_domain}",
|
82
|
+
"ssh_url": f"https://ssh-{self.device_id}.{base_domain}"
|
83
|
+
}
|
84
|
+
|
85
|
+
|
86
|
+
# Usage example
|
87
|
+
def run_with_service_token(device_id):
|
88
|
+
"""
|
89
|
+
Example of how simple it is with service token
|
90
|
+
"""
|
91
|
+
tunnel = ServiceTokenTunnel(device_id)
|
92
|
+
|
93
|
+
# No login needed!
|
94
|
+
# No credential files needed!
|
95
|
+
# Just run with the token
|
96
|
+
process = tunnel.start_tunnel_with_token()
|
97
|
+
|
98
|
+
if process:
|
99
|
+
urls = tunnel.get_tunnel_info()
|
100
|
+
print(f"Jupyter: {urls['jupyter_url']}")
|
101
|
+
print(f"SSH: {urls['ssh_url']}")
|
102
|
+
return process
|
103
|
+
|
104
|
+
return None
|
unitlab/utils.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
import logging
|
2
2
|
from configparser import ConfigParser
|
3
3
|
from pathlib import Path
|
4
|
+
import logging
|
5
|
+
import requests
|
4
6
|
|
5
7
|
import requests
|
6
8
|
|
@@ -63,3 +65,4 @@ def get_api_url() -> str:
|
|
63
65
|
return config.get("default", "api_url")
|
64
66
|
except Exception:
|
65
67
|
return "https://api.unitlab.ai"
|
68
|
+
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.1
|
2
2
|
Name: unitlab
|
3
|
-
Version: 2.3.
|
3
|
+
Version: 2.3.4
|
4
4
|
Home-page: https://github.com/teamunitlab/unitlab-sdk
|
5
5
|
Author: Unitlab Inc.
|
6
6
|
Author-email: team@unitlab.ai
|
@@ -21,11 +21,7 @@ Requires-Dist: requests
|
|
21
21
|
Requires-Dist: tqdm
|
22
22
|
Requires-Dist: typer
|
23
23
|
Requires-Dist: validators
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
Dynamic: keywords
|
29
|
-
Dynamic: license
|
30
|
-
Dynamic: license-file
|
31
|
-
Dynamic: requires-dist
|
24
|
+
Requires-Dist: psutil
|
25
|
+
Requires-Dist: pyyaml
|
26
|
+
Requires-Dist: jupyter
|
27
|
+
|
@@ -0,0 +1,14 @@
|
|
1
|
+
unitlab/__init__.py,sha256=Wtk5kQ_MTlxtd3mxJIn2qHVK5URrVcasMMPjD3BtrVM,214
|
2
|
+
unitlab/__main__.py,sha256=6Hs2PV7EYc5Tid4g4OtcLXhqVHiNYTGzSBdoOnW2HXA,29
|
3
|
+
unitlab/client.py,sha256=LGbjto3jisF5LojlYjJyLKRc55wxfZTO_tkoipZBKd4,22910
|
4
|
+
unitlab/exceptions.py,sha256=68Tr6LreEzjQ3Vns8HAaWdtewtkNUJOvPazbf6NSnXU,950
|
5
|
+
unitlab/main.py,sha256=XD8PF_EJ1gAKqHcRT_bWy62LWrODGWMx2roKoaQaoY4,5253
|
6
|
+
unitlab/tunnel_config.py,sha256=WhxjuOjvOXz16KbMAgpvbCpRD0Y5y53y9jVizQ7CxR8,7841
|
7
|
+
unitlab/tunnel_service_token.py,sha256=ji96a4s4W2cFJrHZle0zBD85Ac_T862-gCKzBUomrxM,3125
|
8
|
+
unitlab/utils.py,sha256=83ekAxxfXecFTg76Z62BGDybC_skKJHYoLyawCD9wGM,1920
|
9
|
+
unitlab-2.3.4.dist-info/LICENSE.md,sha256=Gn7RRvByorAcAaM-WbyUpsgi5ED1-bKFFshbWfYYz2Y,1069
|
10
|
+
unitlab-2.3.4.dist-info/METADATA,sha256=4GHYHBRrtcbbHw06gcZ5mfucnjJvOGFC4Y8DvRqw-28,814
|
11
|
+
unitlab-2.3.4.dist-info/WHEEL,sha256=iAkIy5fosb7FzIOwONchHf19Qu7_1wCWyFNR5gu9nU0,91
|
12
|
+
unitlab-2.3.4.dist-info/entry_points.txt,sha256=ig-PjKEqSCj3UTdyANgEi4tsAU84DyXdaOJ02NHX4bY,45
|
13
|
+
unitlab-2.3.4.dist-info/top_level.txt,sha256=Al4ZlTYE3fTJK2o6YLCDMH5_DjuQkffRBMxgmWbKaqQ,8
|
14
|
+
unitlab-2.3.4.dist-info/RECORD,,
|
unitlab-2.3.0.dist-info/RECORD
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
unitlab/__init__.py,sha256=Wtk5kQ_MTlxtd3mxJIn2qHVK5URrVcasMMPjD3BtrVM,214
|
2
|
-
unitlab/__main__.py,sha256=6Hs2PV7EYc5Tid4g4OtcLXhqVHiNYTGzSBdoOnW2HXA,29
|
3
|
-
unitlab/client.py,sha256=j8m0An4Bt3ARA1ps7P2o-NLcfWFPoaXG_PxIdBDAEpQ,8374
|
4
|
-
unitlab/exceptions.py,sha256=68Tr6LreEzjQ3Vns8HAaWdtewtkNUJOvPazbf6NSnXU,950
|
5
|
-
unitlab/main.py,sha256=3_awGLQ39dOA4WpOLkngzTK1X9ttOs9avh4PvJFlH34,3039
|
6
|
-
unitlab/utils.py,sha256=U95JqeUvKP0p5Ka0KIar0e8Uqe_4TZmZ7N68UdEGfpk,1888
|
7
|
-
unitlab-2.3.0.dist-info/licenses/LICENSE.md,sha256=Gn7RRvByorAcAaM-WbyUpsgi5ED1-bKFFshbWfYYz2Y,1069
|
8
|
-
unitlab-2.3.0.dist-info/METADATA,sha256=9gk8MoNLgV1jPR9TdT_NSvM8wqAqxU10isgBNjQzgNY,903
|
9
|
-
unitlab-2.3.0.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
|
10
|
-
unitlab-2.3.0.dist-info/entry_points.txt,sha256=ig-PjKEqSCj3UTdyANgEi4tsAU84DyXdaOJ02NHX4bY,45
|
11
|
-
unitlab-2.3.0.dist-info/top_level.txt,sha256=Al4ZlTYE3fTJK2o6YLCDMH5_DjuQkffRBMxgmWbKaqQ,8
|
12
|
-
unitlab-2.3.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|