dstack 0.18.43__py3-none-any.whl → 0.19.0__py3-none-any.whl

dstack/_internal/server/services/metrics.py

@@ -1,113 +1,172 @@
 import json
+from collections import defaultdict
+from collections.abc import Sequence
 from datetime import datetime, timezone
+from typing import Optional
 
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from dstack._internal.core.errors import ResourceNotExistsError
+from dstack._internal.core.models.instances import Resources
 from dstack._internal.core.models.metrics import JobMetrics, Metric
-from dstack._internal.server.models import JobMetricsPoint, JobModel, ProjectModel
-from dstack._internal.server.services.jobs import get_run_job_model
+from dstack._internal.server.models import JobMetricsPoint, JobModel
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.utils.common import get_or_error
+from dstack._internal.utils.logging import get_logger
 
-
-async def get_job_metrics(
-    session: AsyncSession,
-    project: ProjectModel,
-    run_name: str,
-    replica_num: int,
-    job_num: int,
-) -> JobMetrics:
-    job_model = await get_run_job_model(
-        session=session,
-        project=project,
-        run_name=run_name,
-        replica_num=replica_num,
-        job_num=job_num,
-    )
-    if job_model is None:
-        raise ResourceNotExistsError("Found no job with given parameters")
-    job_metrics = await _get_job_metrics(
-        session=session,
-        job_model=job_model,
-    )
-    return job_metrics
+logger = get_logger(__name__)
 
 
-async def _get_job_metrics(
+async def get_job_metrics(
     session: AsyncSession,
     job_model: JobModel,
+    after: Optional[datetime] = None,
+    before: Optional[datetime] = None,
+    limit: Optional[int] = None,
 ) -> JobMetrics:
-    res = await session.execute(
+    """
+    Returns metrics ordered from the latest to the earliest.
+
+    Expected usage:
+        * limit=100 — get the latest 100 points
+        * after=<now - 1 hour> — get points for the last one hour
+        * before=<earliest timestamp from the last batch>, limit=100 ­— paginate back in history
+    """
+    stmt = (
         select(JobMetricsPoint)
         .where(JobMetricsPoint.job_id == job_model.id)
         .order_by(JobMetricsPoint.timestamp_micro.desc())
-        .limit(2)
     )
+    if after is not None:
+        # we need +1 point for cpu_usage_percent, thus >=
+        stmt = stmt.where(JobMetricsPoint.timestamp_micro >= _datetime_to_unix_time_micro(after))
+    if before is not None:
+        stmt = stmt.where(JobMetricsPoint.timestamp_micro < _datetime_to_unix_time_micro(before))
+    if limit is not None:
+        # +1 for cpu_usage_percent
+        stmt = stmt.limit(limit + 1)
+    res = await session.execute(stmt)
     points = res.scalars().all()
+    # we need at least 2 points to calculate cpu_usage_percent
     if len(points) < 2:
         return JobMetrics(metrics=[])
-    last_point = points[0]
-    prev_point = points[1]
-    return _calculate_job_metrics(last_point, prev_point)
+    return _calculate_job_metrics(job_model, points)
 
 
-def _calculate_job_metrics(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> JobMetrics:
-    metrics = []
-    timestamp = _unix_time_micro_to_datetime(last_point.timestamp_micro)
-    metrics.append(
+def _calculate_job_metrics(job_model: JobModel, points: Sequence[JobMetricsPoint]) -> JobMetrics:
+    timestamps: list[datetime] = []
+    cpu_usage_points: list[int] = []
+    memory_usage_points: list[int] = []
+    memory_working_set_points: list[int] = []
+    gpus_memory_usage_points: defaultdict[int, list[int]] = defaultdict(list)
+    gpus_util_points: defaultdict[int, list[int]] = defaultdict(list)
+
+    cpus_detected_num: Optional[int] = None
+    memory_total: Optional[int] = None
+    gpu_memory_total: Optional[int] = None
+    resources: Optional[Resources] = None
+    jrd = get_job_runtime_data(job_model)
+    if jrd is not None and jrd.offer is not None:
+        resources = jrd.offer.instance.resources
+    else:
+        jpd = get_job_provisioning_data(job_model)
+        if jpd is not None:
+            resources = jpd.instance_type.resources
+    if resources is not None:
+        cpus_detected_num = resources.cpus
+        memory_total = resources.memory_mib * 1024 * 1024
+        if len(resources.gpus) > 0:
+            gpu_memory_total = resources.gpus[0].memory_mib * 1024 * 1024
+
+    gpus_detected_num: Optional[int] = None
+    gpus_detected_num_mismatch: bool = False
+    for point, prev_point in zip(points, points[1:]):
+        timestamps.append(_unix_time_micro_to_datetime(point.timestamp_micro))
+        cpu_usage_points.append(_get_cpu_usage(point, prev_point))
+        memory_usage_points.append(point.memory_usage_bytes)
+        memory_working_set_points.append(point.memory_working_set_bytes)
+        gpus_memory_usage = json.loads(point.gpus_memory_usage_bytes)
+        gpus_util = json.loads(point.gpus_util_percent)
+        if gpus_detected_num is None:
+            gpus_detected_num = len(gpus_memory_usage)
+        if len(gpus_memory_usage) != gpus_detected_num or len(gpus_util) != gpus_detected_num:
+            gpus_detected_num_mismatch = True
+        if not gpus_detected_num_mismatch:
+            for i in range(gpus_detected_num):
+                gpus_memory_usage_points[i].append(gpus_memory_usage[i])
+                gpus_util_points[i].append(gpus_util[i])
+
+    metrics: list[Metric] = [
         Metric(
             name="cpu_usage_percent",
-            timestamps=[timestamp],
-            values=[_get_cpu_usage(last_point, prev_point)],
-        )
-    )
-    metrics.append(
+            timestamps=timestamps,
+            values=cpu_usage_points,
+        ),
         Metric(
             name="memory_usage_bytes",
-            timestamps=[timestamp],
-            values=[last_point.memory_usage_bytes],
-        )
-    )
-    metrics.append(
+            timestamps=timestamps,
+            values=memory_usage_points,
+        ),
         Metric(
             name="memory_working_set_bytes",
-            timestamps=[timestamp],
-            values=[last_point.memory_working_set_bytes],
-        )
-    )
-
-    gpus_memory_usage_bytes = json.loads(last_point.gpus_memory_usage_bytes)
-    gpus_util_percent = json.loads(last_point.gpus_util_percent)
-    gpus_detected_num = len(gpus_memory_usage_bytes)
-    metrics.append(
-        Metric(
-            name="gpus_detected_num",
-            timestamps=[timestamp],
-            values=[gpus_detected_num],
-        )
-    )
-    for i in range(gpus_detected_num):
+            timestamps=timestamps,
+            values=memory_working_set_points,
+        ),
+    ]
+    if cpus_detected_num is not None:
+        metrics.append(_make_constant_metric("cpus_detected_num", timestamps, cpus_detected_num))
+    if memory_total is not None:
+        metrics.append(_make_constant_metric("memory_total_bytes", timestamps, memory_total))
+    if gpus_detected_num_mismatch:
+        # If number of GPUs changed in the time window, skip GPU metrics altogether, otherwise
+        # results can be unpredictable (e.g, one GPU takes place of another, as they are
+        # identified by an array index only).
+        logger.warning("gpus_detected_num mismatch, skipping GPU metrics")
+    else:
         metrics.append(
-            Metric(
-                name=f"gpu_memory_usage_bytes_gpu{i}",
-                timestamps=[timestamp],
-                values=[gpus_memory_usage_bytes[i]],
-            )
+            _make_constant_metric("gpus_detected_num", timestamps, get_or_error(gpus_detected_num))
         )
-        metrics.append(
-            Metric(
-                name=f"gpu_util_percent_gpu{i}",
-                timestamps=[timestamp],
-                values=[gpus_util_percent[i]],
+        if gpu_memory_total is not None:
+            metrics.append(
+                _make_constant_metric("gpu_memory_total_bytes", timestamps, gpu_memory_total)
+            )
+        for index, gpu_memory_usage_points in gpus_memory_usage_points.items():
+            metrics.append(
+                Metric(
+                    name=f"gpu_memory_usage_bytes_gpu{index}",
+                    timestamps=timestamps,
+                    values=gpu_memory_usage_points,
+                )
+            )
+        for index, gpu_util_points in gpus_util_points.items():
+            metrics.append(
+                Metric(
+                    name=f"gpu_util_percent_gpu{index}",
+                    timestamps=timestamps,
+                    values=gpu_util_points,
+                )
             )
-        )
     return JobMetrics(metrics=metrics)
 
 
+def _make_constant_metric(name: str, timestamps: list[datetime], value: float) -> Metric:
+    return Metric(
+        name=name,
+        timestamps=timestamps,
+        values=[value] * len(timestamps),
+    )
+
+
 def _get_cpu_usage(last_point: JobMetricsPoint, prev_point: JobMetricsPoint) -> int:
     window = last_point.timestamp_micro - prev_point.timestamp_micro
+    if window == 0:
+        return 0
     return round((last_point.cpu_usage_micro - prev_point.cpu_usage_micro) / window * 100)
 
 
 def _unix_time_micro_to_datetime(unix_time_ms: int) -> datetime:
     return datetime.fromtimestamp(unix_time_ms / 1_000_000, tz=timezone.utc)
+
+
+def _datetime_to_unix_time_micro(dt: datetime) -> int:
+    return int(dt.timestamp() * 1_000_000)

dstack/_internal/server/services/offers.py

@@ -7,7 +7,7 @@ from dstack._internal.core.backends import (
     BACKENDS_WITH_MULTINODE_SUPPORT,
     BACKENDS_WITH_RESERVATION_SUPPORT,
 )
-from dstack._internal.core.backends.base import Backend
+from dstack._internal.core.backends.base.backend import Backend
 from dstack._internal.core.models.backends.base import BackendType
 from dstack._internal.core.models.instances import (
     InstanceOfferWithAvailability,

dstack/_internal/server/services/projects.py

@@ -7,19 +7,22 @@ from sqlalchemy import func as safunc
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
-from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError
-from dstack._internal.core.models.backends import BackendInfo
-from dstack._internal.core.models.backends.dstack import (
-    DstackBaseBackendConfigInfo,
-    DstackConfigInfo,
+from dstack._internal.core.backends.configurators import get_configurator
+from dstack._internal.core.backends.dstack.models import (
+    DstackBackendConfig,
+    DstackBaseBackendConfig,
 )
+from dstack._internal.core.backends.models import BackendInfo
+from dstack._internal.core.errors import ForbiddenError, ResourceExistsError, ServerClientError
 from dstack._internal.core.models.common import is_core_model_instance
 from dstack._internal.core.models.projects import Member, MemberPermissions, Project
 from dstack._internal.core.models.users import GlobalRole, ProjectRole
 from dstack._internal.server.models import MemberModel, ProjectModel, UserModel
 from dstack._internal.server.schemas.projects import MemberSetting
 from dstack._internal.server.services import users
-from dstack._internal.server.services.backends import get_configurator
+from dstack._internal.server.services.backends import (
+    get_backend_config_from_backend_model,
+)
 from dstack._internal.server.services.permissions import get_default_permissions
 from dstack._internal.server.settings import DEFAULT_PROJECT_NAME
 from dstack._internal.utils.common import get_current_datetime, run_async
@@ -176,12 +179,16 @@ async def set_project_members(
     # FIXME: potentially long write transaction
     # clear_project_members() issues DELETE without commit
     await clear_project_members(session=session, project=project)
-    usernames = [m.username for m in members]
-    res = await session.execute(select(UserModel).where(UserModel.name.in_(usernames)))
+    names = [m.username for m in members]
+    res = await session.execute(
+        select(UserModel).where((UserModel.name.in_(names)) | (UserModel.email.in_(names)))
+    )
     users = res.scalars().all()
+    # Create lookup maps for both username and email
     username_to_user = {user.name: user for user in users}
+    email_to_user = {user.email: user for user in users if user.email}
     for i, member in enumerate(members):
-        user_to_add = username_to_user.get(member.username)
+        user_to_add = username_to_user.get(member.username) or email_to_user.get(member.username)
         if user_to_add is None:
             continue
         await add_project_member(
@@ -376,20 +383,22 @@ def project_model_to_project(
                     b.type.value,
                 )
                 continue
-            config_info = configurator.get_config_info(model=b, include_creds=False)
-            if is_core_model_instance(config_info, DstackConfigInfo):
-                for backend_type in config_info.base_backends:
+            backend_config = get_backend_config_from_backend_model(
+                configurator, b, include_creds=False
+            )
+            if is_core_model_instance(backend_config, DstackBackendConfig):
+                for backend_type in backend_config.base_backends:
                     backends.append(
                         BackendInfo(
                             name=backend_type,
-                            config=DstackBaseBackendConfigInfo(type=backend_type),
+                            config=DstackBaseBackendConfig(type=backend_type),
                         )
                     )
             else:
                 backends.append(
                     BackendInfo(
                         name=b.type,
-                        config=config_info,
+                        config=backend_config,
                     )
                 )
     return Project(

dstack/_internal/server/services/prometheus.py

@@ -0,0 +1,245 @@
+import itertools
+from collections.abc import Generator, Iterable
+from datetime import timezone
+
+from prometheus_client import Metric
+from prometheus_client.parser import text_string_to_metric_families
+from prometheus_client.samples import Sample
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import joinedload
+
+from dstack._internal.core.models.instances import InstanceStatus
+from dstack._internal.core.models.runs import JobStatus, RunSpec
+from dstack._internal.server.models import (
+    InstanceModel,
+    JobModel,
+    JobPrometheusMetrics,
+    ProjectModel,
+    RunModel,
+)
+from dstack._internal.server.services.instances import get_instance_offer
+from dstack._internal.server.services.jobs import get_job_provisioning_data, get_job_runtime_data
+from dstack._internal.utils.common import get_current_datetime
+
+_INSTANCE_DURATION = "dstack_instance_duration_seconds_total"
+_INSTANCE_PRICE = "dstack_instance_price_dollars_per_hour"
+_INSTANCE_GPU_COUNT = "dstack_instance_gpu_count"
+_JOB_DURATION = "dstack_job_duration_seconds_total"
+_JOB_PRICE = "dstack_job_price_dollars_per_hour"
+_JOB_GPU_COUNT = "dstack_job_gpu_count"
+
+
+async def get_metrics(session: AsyncSession) -> str:
+    metrics_iter = itertools.chain(
+        await get_instance_metrics(session),
+        await get_job_metrics(session),
+        await get_job_gpu_metrics(session),
+    )
+    return "\n".join(_render_metrics(metrics_iter)) + "\n"
+
+
+async def get_instance_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(InstanceModel)
+        .join(ProjectModel)
+        .where(
+            InstanceModel.deleted == False,
+            InstanceModel.status.in_(
+                [
+                    InstanceStatus.PROVISIONING,
+                    InstanceStatus.IDLE,
+                    InstanceStatus.BUSY,
+                    InstanceStatus.TERMINATING,
+                ]
+            ),
+        )
+        .order_by(ProjectModel.name, InstanceModel.name)
+        .options(
+            joinedload(InstanceModel.project),
+            joinedload(InstanceModel.fleet),
+        )
+    )
+    instances = res.unique().scalars().all()
+    metrics: dict[str, Metric] = {
+        _INSTANCE_DURATION: Metric(
+            name=_INSTANCE_DURATION,
+            documentation="Total seconds the instance is running",
+            typ="counter",
+        ),
+        _INSTANCE_PRICE: Metric(
+            name=_INSTANCE_PRICE, documentation="Instance price, USD/hour", typ="gauge"
+        ),
+        _INSTANCE_GPU_COUNT: Metric(
+            name=_INSTANCE_GPU_COUNT, documentation="Instance GPU count", typ="gauge"
+        ),
+    }
+    now = get_current_datetime()
+    for instance in instances:
+        fleet = instance.fleet
+        offer = get_instance_offer(instance)
+        gpu = ""
+        gpu_count = 0
+        if offer is not None and len(offer.instance.resources.gpus) > 0:
+            gpu = offer.instance.resources.gpus[0].name
+            gpu_count = len(offer.instance.resources.gpus)
+        labels: dict[str, str] = {
+            "dstack_project_name": instance.project.name,
+            "dstack_fleet_name": fleet.name if fleet is not None else "",
+            "dstack_fleet_id": str(fleet.id) if fleet is not None else "",
+            "dstack_instance_name": str(instance.name),
+            "dstack_instance_id": str(instance.id),
+            "dstack_instance_type": offer.instance.name if offer is not None else "",
+            "dstack_backend": instance.backend.value if instance.backend is not None else "",
+            "dstack_gpu": gpu,
+        }
+        duration = (now - instance.created_at.replace(tzinfo=timezone.utc)).total_seconds()
+        metrics[_INSTANCE_DURATION].add_sample(
+            name=_INSTANCE_DURATION, labels=labels, value=duration
+        )
+        metrics[_INSTANCE_PRICE].add_sample(
+            name=_INSTANCE_PRICE, labels=labels, value=instance.price or 0.0
+        )
+        metrics[_INSTANCE_GPU_COUNT].add_sample(
+            name=_INSTANCE_GPU_COUNT, labels=labels, value=gpu_count
+        )
+    return metrics.values()
+
+
+async def get_job_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(JobModel)
+        .join(ProjectModel)
+        .where(
+            JobModel.status.in_(
+                [
+                    JobStatus.PROVISIONING,
+                    JobStatus.PULLING,
+                    JobStatus.RUNNING,
+                    JobStatus.TERMINATING,
+                ]
+            )
+        )
+        .order_by(ProjectModel.name, JobModel.job_name)
+        .options(
+            joinedload(JobModel.project),
+            joinedload(JobModel.run).joinedload(RunModel.user),
+        )
+    )
+    jobs = res.scalars().all()
+    metrics: dict[str, Metric] = {
+        _JOB_DURATION: Metric(
+            name=_JOB_DURATION, documentation="Total seconds the job is running", typ="counter"
+        ),
+        _JOB_PRICE: Metric(
+            name=_JOB_PRICE, documentation="Job instance price, USD/hour", typ="gauge"
+        ),
+        _JOB_GPU_COUNT: Metric(name=_JOB_GPU_COUNT, documentation="Job GPU count", typ="gauge"),
+    }
+    now = get_current_datetime()
+    for job in jobs:
+        jpd = get_job_provisioning_data(job)
+        if jpd is None:
+            continue
+        jrd = get_job_runtime_data(job)
+        gpus = jpd.instance_type.resources.gpus
+        price = jpd.price
+        if jrd is not None and jrd.offer is not None:
+            gpus = jrd.offer.instance.resources.gpus
+            price = jrd.offer.price
+        run_spec = RunSpec.__response__.parse_raw(job.run.run_spec)
+        labels = _get_job_labels(job)
+        labels["dstack_run_type"] = run_spec.configuration.type
+        labels["dstack_backend"] = jpd.get_base_backend().value
+        labels["dstack_gpu"] = gpus[0].name if gpus else ""
+        duration = (now - job.submitted_at.replace(tzinfo=timezone.utc)).total_seconds()
+        metrics[_JOB_DURATION].add_sample(name=_JOB_DURATION, labels=labels, value=duration)
+        metrics[_JOB_PRICE].add_sample(name=_JOB_PRICE, labels=labels, value=price)
+        metrics[_JOB_GPU_COUNT].add_sample(name=_JOB_GPU_COUNT, labels=labels, value=len(gpus))
+    return metrics.values()
+
+
+async def get_job_gpu_metrics(session: AsyncSession) -> Iterable[Metric]:
+    res = await session.execute(
+        select(JobPrometheusMetrics)
+        .join(JobModel)
+        .join(ProjectModel)
+        .where(JobModel.status.in_([JobStatus.RUNNING]))
+        .order_by(ProjectModel.name, JobModel.job_name)
+        .options(
+            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
+            joinedload(JobPrometheusMetrics.job)
+            .joinedload(JobModel.run)
+            .joinedload(RunModel.user),
+        )
+    )
+    metrics_models = res.scalars().all()
+    return _parse_and_enrich_job_gpu_metrics(metrics_models)
+
+
+async def get_project_metrics(session: AsyncSession, project: ProjectModel) -> str:
+    res = await session.execute(
+        select(JobPrometheusMetrics)
+        .join(JobModel)
+        .where(
+            JobModel.project_id == project.id,
+            JobModel.status.in_([JobStatus.RUNNING]),
+        )
+        .order_by(JobModel.job_name)
+        .options(
+            joinedload(JobPrometheusMetrics.job).joinedload(JobModel.project),
+            joinedload(JobPrometheusMetrics.job)
+            .joinedload(JobModel.run)
+            .joinedload(RunModel.user),
+        )
+    )
+    metrics_models = res.scalars().all()
+    return "\n".join(_render_metrics(_parse_and_enrich_job_gpu_metrics(metrics_models))) + "\n"
+
+
+def _parse_and_enrich_job_gpu_metrics(
+    metrics_models: Iterable[JobPrometheusMetrics],
+) -> Iterable[Metric]:
+    metrics: dict[str, Metric] = {}
+    for metrics_model in metrics_models:
+        for metric in text_string_to_metric_families(metrics_model.text):
+            samples = metric.samples
+            metric.samples = []
+            name = metric.name
+            metric = metrics.setdefault(name, metric)
+            for sample in samples:
+                labels = sample.labels
+                labels.update(_get_job_labels(metrics_model.job))
+                # text_string_to_metric_families "fixes" counter names appending _total,
+                # we rebuild Sample to revert this
+                metric.samples.append(Sample(name, labels, *sample[2:]))
+    return metrics.values()
+
+
+def _get_job_labels(job: JobModel) -> dict[str, str]:
+    return {
+        "dstack_project_name": job.project.name,
+        "dstack_user_name": job.run.user.name,
+        "dstack_run_name": job.run_name,
+        "dstack_run_id": str(job.run_id),
+        "dstack_job_name": job.job_name,
+        "dstack_job_id": str(job.id),
+        "dstack_job_num": str(job.job_num),
+        "dstack_replica_num": str(job.replica_num),
+    }
+
+
+def _render_metrics(metrics: Iterable[Metric]) -> Generator[str, None, None]:
+    for metric in metrics:
+        if not metric.samples:
+            continue
+        yield f"# HELP {metric.name} {metric.documentation}"
+        yield f"# TYPE {metric.name} {metric.type}"
+        for sample in metric.samples:
+            parts: list[str] = [f"{sample.name}{{"]
+            parts.extend(",".join(f'{name}="{value}"' for name, value in sample.labels.items()))
+            parts.append(f"}} {float(sample.value)}")
+            # text_string_to_metric_families converts milliseconds to float seconds
+            if isinstance(sample.timestamp, float):
+                parts.append(f" {int(sample.timestamp * 1000)}")
+            yield "".join(parts)

dstack/_internal/server/services/runner/client.py

@@ -178,9 +178,6 @@ class ShimClient:
     # API v1 (a.k.a. Legacy API) — `/api/{submit,pull,stop}`
     _API_V2_MIN_SHIM_VERSION = (0, 18, 34)
 
-    # A surrogate task ID for API-v1-over-v2 emulation (`_v2_compat_*` methods)
-    _LEGACY_TASK_ID = "00000000-0000-0000-0000-000000000000"
-
     _shim_version: Optional["_Version"]
     _api_version: int
     _negotiated: bool = False
@@ -339,6 +336,20 @@ class ShimClient:
         resp = self._request("GET", "/api/pull", raise_for_status=True)
         return self._response(LegacyPullResponse, resp)
 
+    # Metrics
+
+    def get_task_metrics(self, task_id: "_TaskID") -> Optional[str]:
+        resp = self._request("GET", f"/metrics/tasks/{task_id}")
+        if resp.status_code == HTTPStatus.NOT_FOUND:
+            # Metrics exporter is not installed or old shim version
+            return None
+        if resp.status_code == HTTPStatus.BAD_GATEWAY:
+            # Metrics exporter is not available or returned an error
+            logger.info("failed to collect metrics for task %s: %s", task_id, resp.text)
+            return None
+        self._raise_for_status(resp)
+        return resp.text
+
     # Private methods used for public methods implementations
 
     def _request(