label-studio-sdk 2.0.12__py3-none-any.whl → 2.0.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of label-studio-sdk might be problematic. Click here for more details.
- label_studio_sdk/__init__.py +2 -8
- label_studio_sdk/actions/__init__.py +2 -0
- label_studio_sdk/actions/types/__init__.py +2 -0
- label_studio_sdk/actions/types/actions_list_response_item.py +2 -1
- label_studio_sdk/actions/types/actions_list_response_item_permission.py +5 -0
- label_studio_sdk/core/client_wrapper.py +6 -1
- label_studio_sdk/export_storage/azure/client.py +14 -10
- label_studio_sdk/export_storage/azure_spi/client.py +14 -10
- label_studio_sdk/export_storage/databricks/client.py +14 -10
- label_studio_sdk/export_storage/gcs/client.py +14 -10
- label_studio_sdk/export_storage/gcswif/client.py +14 -10
- label_studio_sdk/export_storage/local/client.py +14 -10
- label_studio_sdk/export_storage/redis/client.py +14 -10
- label_studio_sdk/export_storage/s3/client.py +14 -10
- label_studio_sdk/export_storage/s3s/client.py +14 -10
- label_studio_sdk/import_storage/azure/client.py +14 -10
- label_studio_sdk/import_storage/azure_spi/client.py +14 -10
- label_studio_sdk/import_storage/databricks/client.py +14 -10
- label_studio_sdk/import_storage/gcs/client.py +14 -10
- label_studio_sdk/import_storage/gcswif/client.py +14 -10
- label_studio_sdk/import_storage/local/client.py +14 -10
- label_studio_sdk/import_storage/redis/client.py +14 -10
- label_studio_sdk/import_storage/s3/client.py +14 -10
- label_studio_sdk/import_storage/s3s/client.py +14 -10
- label_studio_sdk/label_interface/control_tags.py +2 -1
- label_studio_sdk/organizations/__init__.py +1 -2
- label_studio_sdk/organizations/client.py +0 -36
- label_studio_sdk/organizations/permissions/client.py +32 -0
- label_studio_sdk/projects/__init__.py +10 -0
- label_studio_sdk/projects/metrics/custom/client.py +48 -0
- label_studio_sdk/projects/stats/__init__.py +10 -0
- label_studio_sdk/projects/stats/client.py +346 -0
- label_studio_sdk/projects/stats/types/__init__.py +14 -0
- label_studio_sdk/projects/stats/types/stats_users_ground_truth_agreement_response.py +27 -0
- label_studio_sdk/projects/stats/types/stats_users_ground_truth_agreement_response_agreement_value.py +5 -0
- label_studio_sdk/projects/stats/types/stats_users_review_score_response.py +35 -0
- label_studio_sdk/projects/stats/types/stats_users_review_score_response_performance_score_value.py +5 -0
- label_studio_sdk/projects/stats/types/stats_users_review_score_response_review_score_value.py +5 -0
- label_studio_sdk/prompts/versions/client.py +24 -0
- label_studio_sdk/tasks/client.py +10 -0
- label_studio_sdk/types/__init__.py +0 -6
- label_studio_sdk/types/activity_log.py +1 -0
- label_studio_sdk/types/all_roles_project_list.py +1 -1
- label_studio_sdk/types/billing_flags.py +1 -1
- label_studio_sdk/types/default_role.py +0 -9
- label_studio_sdk/types/import_api_request.py +5 -0
- label_studio_sdk/types/lse_organization.py +1 -10
- label_studio_sdk/types/lse_project.py +1 -5
- label_studio_sdk/types/lse_project_counts.py +1 -1
- label_studio_sdk/types/lse_task.py +5 -0
- label_studio_sdk/types/lse_task_serializer_for_reviewers.py +5 -0
- label_studio_sdk/versions/client.py +0 -12
- {label_studio_sdk-2.0.12.dist-info → label_studio_sdk-2.0.13.dist-info}/METADATA +1 -1
- {label_studio_sdk-2.0.12.dist-info → label_studio_sdk-2.0.13.dist-info}/RECORD +56 -55
- label_studio_sdk/organizations/types/__init__.py +0 -5
- label_studio_sdk/organizations/types/patched_default_role_request_custom_scripts_editable_by.py +0 -7
- label_studio_sdk/types/custom_scripts_editable_by_enum.py +0 -5
- label_studio_sdk/types/default_role_custom_scripts_editable_by.py +0 -7
- label_studio_sdk/types/lse_organization_custom_scripts_editable_by.py +0 -7
- {label_studio_sdk-2.0.12.dist-info → label_studio_sdk-2.0.13.dist-info}/LICENSE +0 -0
- {label_studio_sdk-2.0.12.dist-info → label_studio_sdk-2.0.13.dist-info}/WHEEL +0 -0
|
@@ -11,6 +11,7 @@ from ...core.api_error import ApiError
|
|
|
11
11
|
from .types.stats_model_version_ground_truth_agreement_response import StatsModelVersionGroundTruthAgreementResponse
|
|
12
12
|
from .types.stats_model_version_prediction_agreement_response import StatsModelVersionPredictionAgreementResponse
|
|
13
13
|
from .types.stats_iaa_response import StatsIaaResponse
|
|
14
|
+
from .types.stats_users_ground_truth_agreement_response import StatsUsersGroundTruthAgreementResponse
|
|
14
15
|
from .types.stats_agreement_annotator_response import StatsAgreementAnnotatorResponse
|
|
15
16
|
from .types.stats_agreement_annotators_response import StatsAgreementAnnotatorsResponse
|
|
16
17
|
from .types.stats_data_filters_response import StatsDataFiltersResponse
|
|
@@ -18,6 +19,7 @@ from .types.stats_finished_tasks_response import StatsFinishedTasksResponse
|
|
|
18
19
|
from .types.stats_lead_time_response import StatsLeadTimeResponse
|
|
19
20
|
from .types.stats_total_agreement_response import StatsTotalAgreementResponse
|
|
20
21
|
from .types.stats_users_prediction_agreement_response import StatsUsersPredictionAgreementResponse
|
|
22
|
+
from .types.stats_users_review_score_response import StatsUsersReviewScoreResponse
|
|
21
23
|
from .types.stats_user_prediction_agreement_response import StatsUserPredictionAgreementResponse
|
|
22
24
|
from .types.stats_user_review_score_response import StatsUserReviewScoreResponse
|
|
23
25
|
from .types.stats_user_ground_truth_agreement_response import StatsUserGroundTruthAgreementResponse
|
|
@@ -300,6 +302,76 @@ class StatsClient:
|
|
|
300
302
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
301
303
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
302
304
|
|
|
305
|
+
def users_ground_truth_agreement(
|
|
306
|
+
self,
|
|
307
|
+
id: int,
|
|
308
|
+
*,
|
|
309
|
+
ids: str,
|
|
310
|
+
per_label: typing.Optional[bool] = None,
|
|
311
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
312
|
+
) -> StatsUsersGroundTruthAgreementResponse:
|
|
313
|
+
"""
|
|
314
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
315
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
316
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
317
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
318
|
+
</p>
|
|
319
|
+
</Card>
|
|
320
|
+
Get ground truth agreement statistics for multiple users within a project.
|
|
321
|
+
|
|
322
|
+
Parameters
|
|
323
|
+
----------
|
|
324
|
+
id : int
|
|
325
|
+
|
|
326
|
+
ids : str
|
|
327
|
+
Comma separated list of user IDs to get ground truth agreement for
|
|
328
|
+
|
|
329
|
+
per_label : typing.Optional[bool]
|
|
330
|
+
Per label
|
|
331
|
+
|
|
332
|
+
request_options : typing.Optional[RequestOptions]
|
|
333
|
+
Request-specific configuration.
|
|
334
|
+
|
|
335
|
+
Returns
|
|
336
|
+
-------
|
|
337
|
+
StatsUsersGroundTruthAgreementResponse
|
|
338
|
+
Ground truth agreement statistics for multiple users
|
|
339
|
+
|
|
340
|
+
Examples
|
|
341
|
+
--------
|
|
342
|
+
from label_studio_sdk import LabelStudio
|
|
343
|
+
|
|
344
|
+
client = LabelStudio(
|
|
345
|
+
api_key="YOUR_API_KEY",
|
|
346
|
+
)
|
|
347
|
+
client.projects.stats.users_ground_truth_agreement(
|
|
348
|
+
id=1,
|
|
349
|
+
ids="ids",
|
|
350
|
+
)
|
|
351
|
+
"""
|
|
352
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
353
|
+
f"api/projects/{jsonable_encoder(id)}/stats/agreement-groundtruth",
|
|
354
|
+
method="GET",
|
|
355
|
+
params={
|
|
356
|
+
"ids": ids,
|
|
357
|
+
"per_label": per_label,
|
|
358
|
+
},
|
|
359
|
+
request_options=request_options,
|
|
360
|
+
)
|
|
361
|
+
try:
|
|
362
|
+
if 200 <= _response.status_code < 300:
|
|
363
|
+
return typing.cast(
|
|
364
|
+
StatsUsersGroundTruthAgreementResponse,
|
|
365
|
+
construct_type(
|
|
366
|
+
type_=StatsUsersGroundTruthAgreementResponse, # type: ignore
|
|
367
|
+
object_=_response.json(),
|
|
368
|
+
),
|
|
369
|
+
)
|
|
370
|
+
_response_json = _response.json()
|
|
371
|
+
except JSONDecodeError:
|
|
372
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
373
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
374
|
+
|
|
303
375
|
def agreement_annotator(
|
|
304
376
|
self, id: int, user_id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
305
377
|
) -> StatsAgreementAnnotatorResponse:
|
|
@@ -422,6 +494,12 @@ class StatsClient:
|
|
|
422
494
|
self, id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
423
495
|
) -> StatsDataFiltersResponse:
|
|
424
496
|
"""
|
|
497
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
498
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
499
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
500
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
501
|
+
</p>
|
|
502
|
+
</Card>
|
|
425
503
|
Get statistics about user data filters and their usage within a project.
|
|
426
504
|
|
|
427
505
|
Parameters
|
|
@@ -470,6 +548,12 @@ class StatsClient:
|
|
|
470
548
|
self, id: int, *, user_pk: typing.Optional[int] = None, request_options: typing.Optional[RequestOptions] = None
|
|
471
549
|
) -> StatsFinishedTasksResponse:
|
|
472
550
|
"""
|
|
551
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
552
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
553
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
554
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
555
|
+
</p>
|
|
556
|
+
</Card>
|
|
473
557
|
Get statistics about finished tasks for a project.
|
|
474
558
|
|
|
475
559
|
Parameters
|
|
@@ -522,6 +606,12 @@ class StatsClient:
|
|
|
522
606
|
|
|
523
607
|
def lead_time(self, id: int, *, request_options: typing.Optional[RequestOptions] = None) -> StatsLeadTimeResponse:
|
|
524
608
|
"""
|
|
609
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
610
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
611
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
612
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
613
|
+
</p>
|
|
614
|
+
</Card>
|
|
525
615
|
Get lead time statistics across the project, including average annotation time.
|
|
526
616
|
|
|
527
617
|
Parameters
|
|
@@ -640,6 +730,12 @@ class StatsClient:
|
|
|
640
730
|
request_options: typing.Optional[RequestOptions] = None,
|
|
641
731
|
) -> typing.Dict[str, typing.Optional[typing.Any]]:
|
|
642
732
|
"""
|
|
733
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
734
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
735
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
736
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
737
|
+
</p>
|
|
738
|
+
</Card>
|
|
643
739
|
Start stats recalculation for given project
|
|
644
740
|
|
|
645
741
|
Parameters
|
|
@@ -760,6 +856,76 @@ class StatsClient:
|
|
|
760
856
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
761
857
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
762
858
|
|
|
859
|
+
def users_review_score(
|
|
860
|
+
self,
|
|
861
|
+
id: int,
|
|
862
|
+
*,
|
|
863
|
+
ids: str,
|
|
864
|
+
per_label: typing.Optional[bool] = None,
|
|
865
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
866
|
+
) -> StatsUsersReviewScoreResponse:
|
|
867
|
+
"""
|
|
868
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
869
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
870
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
871
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
872
|
+
</p>
|
|
873
|
+
</Card>
|
|
874
|
+
Get review score and performance score statistics for multiple annotators within a project.
|
|
875
|
+
|
|
876
|
+
Parameters
|
|
877
|
+
----------
|
|
878
|
+
id : int
|
|
879
|
+
|
|
880
|
+
ids : str
|
|
881
|
+
Comma separated list of annotator user IDs to get review scores for
|
|
882
|
+
|
|
883
|
+
per_label : typing.Optional[bool]
|
|
884
|
+
Per label
|
|
885
|
+
|
|
886
|
+
request_options : typing.Optional[RequestOptions]
|
|
887
|
+
Request-specific configuration.
|
|
888
|
+
|
|
889
|
+
Returns
|
|
890
|
+
-------
|
|
891
|
+
StatsUsersReviewScoreResponse
|
|
892
|
+
Review scores and performance scores for multiple annotators
|
|
893
|
+
|
|
894
|
+
Examples
|
|
895
|
+
--------
|
|
896
|
+
from label_studio_sdk import LabelStudio
|
|
897
|
+
|
|
898
|
+
client = LabelStudio(
|
|
899
|
+
api_key="YOUR_API_KEY",
|
|
900
|
+
)
|
|
901
|
+
client.projects.stats.users_review_score(
|
|
902
|
+
id=1,
|
|
903
|
+
ids="ids",
|
|
904
|
+
)
|
|
905
|
+
"""
|
|
906
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
907
|
+
f"api/projects/{jsonable_encoder(id)}/user-stats/review_score",
|
|
908
|
+
method="GET",
|
|
909
|
+
params={
|
|
910
|
+
"ids": ids,
|
|
911
|
+
"per_label": per_label,
|
|
912
|
+
},
|
|
913
|
+
request_options=request_options,
|
|
914
|
+
)
|
|
915
|
+
try:
|
|
916
|
+
if 200 <= _response.status_code < 300:
|
|
917
|
+
return typing.cast(
|
|
918
|
+
StatsUsersReviewScoreResponse,
|
|
919
|
+
construct_type(
|
|
920
|
+
type_=StatsUsersReviewScoreResponse, # type: ignore
|
|
921
|
+
object_=_response.json(),
|
|
922
|
+
),
|
|
923
|
+
)
|
|
924
|
+
_response_json = _response.json()
|
|
925
|
+
except JSONDecodeError:
|
|
926
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
927
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
928
|
+
|
|
763
929
|
def user_prediction_agreement(
|
|
764
930
|
self,
|
|
765
931
|
id: int,
|
|
@@ -1273,6 +1439,84 @@ class AsyncStatsClient:
|
|
|
1273
1439
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1274
1440
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1275
1441
|
|
|
1442
|
+
async def users_ground_truth_agreement(
|
|
1443
|
+
self,
|
|
1444
|
+
id: int,
|
|
1445
|
+
*,
|
|
1446
|
+
ids: str,
|
|
1447
|
+
per_label: typing.Optional[bool] = None,
|
|
1448
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
1449
|
+
) -> StatsUsersGroundTruthAgreementResponse:
|
|
1450
|
+
"""
|
|
1451
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1452
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1453
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1454
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1455
|
+
</p>
|
|
1456
|
+
</Card>
|
|
1457
|
+
Get ground truth agreement statistics for multiple users within a project.
|
|
1458
|
+
|
|
1459
|
+
Parameters
|
|
1460
|
+
----------
|
|
1461
|
+
id : int
|
|
1462
|
+
|
|
1463
|
+
ids : str
|
|
1464
|
+
Comma separated list of user IDs to get ground truth agreement for
|
|
1465
|
+
|
|
1466
|
+
per_label : typing.Optional[bool]
|
|
1467
|
+
Per label
|
|
1468
|
+
|
|
1469
|
+
request_options : typing.Optional[RequestOptions]
|
|
1470
|
+
Request-specific configuration.
|
|
1471
|
+
|
|
1472
|
+
Returns
|
|
1473
|
+
-------
|
|
1474
|
+
StatsUsersGroundTruthAgreementResponse
|
|
1475
|
+
Ground truth agreement statistics for multiple users
|
|
1476
|
+
|
|
1477
|
+
Examples
|
|
1478
|
+
--------
|
|
1479
|
+
import asyncio
|
|
1480
|
+
|
|
1481
|
+
from label_studio_sdk import AsyncLabelStudio
|
|
1482
|
+
|
|
1483
|
+
client = AsyncLabelStudio(
|
|
1484
|
+
api_key="YOUR_API_KEY",
|
|
1485
|
+
)
|
|
1486
|
+
|
|
1487
|
+
|
|
1488
|
+
async def main() -> None:
|
|
1489
|
+
await client.projects.stats.users_ground_truth_agreement(
|
|
1490
|
+
id=1,
|
|
1491
|
+
ids="ids",
|
|
1492
|
+
)
|
|
1493
|
+
|
|
1494
|
+
|
|
1495
|
+
asyncio.run(main())
|
|
1496
|
+
"""
|
|
1497
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
1498
|
+
f"api/projects/{jsonable_encoder(id)}/stats/agreement-groundtruth",
|
|
1499
|
+
method="GET",
|
|
1500
|
+
params={
|
|
1501
|
+
"ids": ids,
|
|
1502
|
+
"per_label": per_label,
|
|
1503
|
+
},
|
|
1504
|
+
request_options=request_options,
|
|
1505
|
+
)
|
|
1506
|
+
try:
|
|
1507
|
+
if 200 <= _response.status_code < 300:
|
|
1508
|
+
return typing.cast(
|
|
1509
|
+
StatsUsersGroundTruthAgreementResponse,
|
|
1510
|
+
construct_type(
|
|
1511
|
+
type_=StatsUsersGroundTruthAgreementResponse, # type: ignore
|
|
1512
|
+
object_=_response.json(),
|
|
1513
|
+
),
|
|
1514
|
+
)
|
|
1515
|
+
_response_json = _response.json()
|
|
1516
|
+
except JSONDecodeError:
|
|
1517
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1518
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1519
|
+
|
|
1276
1520
|
async def agreement_annotator(
|
|
1277
1521
|
self, id: int, user_id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
1278
1522
|
) -> StatsAgreementAnnotatorResponse:
|
|
@@ -1411,6 +1655,12 @@ class AsyncStatsClient:
|
|
|
1411
1655
|
self, id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
1412
1656
|
) -> StatsDataFiltersResponse:
|
|
1413
1657
|
"""
|
|
1658
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1659
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1660
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1661
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1662
|
+
</p>
|
|
1663
|
+
</Card>
|
|
1414
1664
|
Get statistics about user data filters and their usage within a project.
|
|
1415
1665
|
|
|
1416
1666
|
Parameters
|
|
@@ -1467,6 +1717,12 @@ class AsyncStatsClient:
|
|
|
1467
1717
|
self, id: int, *, user_pk: typing.Optional[int] = None, request_options: typing.Optional[RequestOptions] = None
|
|
1468
1718
|
) -> StatsFinishedTasksResponse:
|
|
1469
1719
|
"""
|
|
1720
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1721
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1722
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1723
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1724
|
+
</p>
|
|
1725
|
+
</Card>
|
|
1470
1726
|
Get statistics about finished tasks for a project.
|
|
1471
1727
|
|
|
1472
1728
|
Parameters
|
|
@@ -1529,6 +1785,12 @@ class AsyncStatsClient:
|
|
|
1529
1785
|
self, id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
1530
1786
|
) -> StatsLeadTimeResponse:
|
|
1531
1787
|
"""
|
|
1788
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1789
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1790
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1791
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1792
|
+
</p>
|
|
1793
|
+
</Card>
|
|
1532
1794
|
Get lead time statistics across the project, including average annotation time.
|
|
1533
1795
|
|
|
1534
1796
|
Parameters
|
|
@@ -1663,6 +1925,12 @@ class AsyncStatsClient:
|
|
|
1663
1925
|
request_options: typing.Optional[RequestOptions] = None,
|
|
1664
1926
|
) -> typing.Dict[str, typing.Optional[typing.Any]]:
|
|
1665
1927
|
"""
|
|
1928
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1929
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1930
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1931
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1932
|
+
</p>
|
|
1933
|
+
</Card>
|
|
1666
1934
|
Start stats recalculation for given project
|
|
1667
1935
|
|
|
1668
1936
|
Parameters
|
|
@@ -1799,6 +2067,84 @@ class AsyncStatsClient:
|
|
|
1799
2067
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1800
2068
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1801
2069
|
|
|
2070
|
+
async def users_review_score(
|
|
2071
|
+
self,
|
|
2072
|
+
id: int,
|
|
2073
|
+
*,
|
|
2074
|
+
ids: str,
|
|
2075
|
+
per_label: typing.Optional[bool] = None,
|
|
2076
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
2077
|
+
) -> StatsUsersReviewScoreResponse:
|
|
2078
|
+
"""
|
|
2079
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
2080
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
2081
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
2082
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
2083
|
+
</p>
|
|
2084
|
+
</Card>
|
|
2085
|
+
Get review score and performance score statistics for multiple annotators within a project.
|
|
2086
|
+
|
|
2087
|
+
Parameters
|
|
2088
|
+
----------
|
|
2089
|
+
id : int
|
|
2090
|
+
|
|
2091
|
+
ids : str
|
|
2092
|
+
Comma separated list of annotator user IDs to get review scores for
|
|
2093
|
+
|
|
2094
|
+
per_label : typing.Optional[bool]
|
|
2095
|
+
Per label
|
|
2096
|
+
|
|
2097
|
+
request_options : typing.Optional[RequestOptions]
|
|
2098
|
+
Request-specific configuration.
|
|
2099
|
+
|
|
2100
|
+
Returns
|
|
2101
|
+
-------
|
|
2102
|
+
StatsUsersReviewScoreResponse
|
|
2103
|
+
Review scores and performance scores for multiple annotators
|
|
2104
|
+
|
|
2105
|
+
Examples
|
|
2106
|
+
--------
|
|
2107
|
+
import asyncio
|
|
2108
|
+
|
|
2109
|
+
from label_studio_sdk import AsyncLabelStudio
|
|
2110
|
+
|
|
2111
|
+
client = AsyncLabelStudio(
|
|
2112
|
+
api_key="YOUR_API_KEY",
|
|
2113
|
+
)
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
async def main() -> None:
|
|
2117
|
+
await client.projects.stats.users_review_score(
|
|
2118
|
+
id=1,
|
|
2119
|
+
ids="ids",
|
|
2120
|
+
)
|
|
2121
|
+
|
|
2122
|
+
|
|
2123
|
+
asyncio.run(main())
|
|
2124
|
+
"""
|
|
2125
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
2126
|
+
f"api/projects/{jsonable_encoder(id)}/user-stats/review_score",
|
|
2127
|
+
method="GET",
|
|
2128
|
+
params={
|
|
2129
|
+
"ids": ids,
|
|
2130
|
+
"per_label": per_label,
|
|
2131
|
+
},
|
|
2132
|
+
request_options=request_options,
|
|
2133
|
+
)
|
|
2134
|
+
try:
|
|
2135
|
+
if 200 <= _response.status_code < 300:
|
|
2136
|
+
return typing.cast(
|
|
2137
|
+
StatsUsersReviewScoreResponse,
|
|
2138
|
+
construct_type(
|
|
2139
|
+
type_=StatsUsersReviewScoreResponse, # type: ignore
|
|
2140
|
+
object_=_response.json(),
|
|
2141
|
+
),
|
|
2142
|
+
)
|
|
2143
|
+
_response_json = _response.json()
|
|
2144
|
+
except JSONDecodeError:
|
|
2145
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
2146
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
2147
|
+
|
|
1802
2148
|
async def user_prediction_agreement(
|
|
1803
2149
|
self,
|
|
1804
2150
|
id: int,
|
|
@@ -27,10 +27,19 @@ from .stats_user_prediction_agreement_response_average_prediction_agreement_per_
|
|
|
27
27
|
from .stats_user_review_score_response import StatsUserReviewScoreResponse
|
|
28
28
|
from .stats_user_review_score_response_performance_score import StatsUserReviewScoreResponsePerformanceScore
|
|
29
29
|
from .stats_user_review_score_response_review_score import StatsUserReviewScoreResponseReviewScore
|
|
30
|
+
from .stats_users_ground_truth_agreement_response import StatsUsersGroundTruthAgreementResponse
|
|
31
|
+
from .stats_users_ground_truth_agreement_response_agreement_value import (
|
|
32
|
+
StatsUsersGroundTruthAgreementResponseAgreementValue,
|
|
33
|
+
)
|
|
30
34
|
from .stats_users_prediction_agreement_response import StatsUsersPredictionAgreementResponse
|
|
31
35
|
from .stats_users_prediction_agreement_response_agreement_value import (
|
|
32
36
|
StatsUsersPredictionAgreementResponseAgreementValue,
|
|
33
37
|
)
|
|
38
|
+
from .stats_users_review_score_response import StatsUsersReviewScoreResponse
|
|
39
|
+
from .stats_users_review_score_response_performance_score_value import (
|
|
40
|
+
StatsUsersReviewScoreResponsePerformanceScoreValue,
|
|
41
|
+
)
|
|
42
|
+
from .stats_users_review_score_response_review_score_value import StatsUsersReviewScoreResponseReviewScoreValue
|
|
34
43
|
|
|
35
44
|
__all__ = [
|
|
36
45
|
"StatsAgreementAnnotatorResponse",
|
|
@@ -58,6 +67,11 @@ __all__ = [
|
|
|
58
67
|
"StatsUserReviewScoreResponse",
|
|
59
68
|
"StatsUserReviewScoreResponsePerformanceScore",
|
|
60
69
|
"StatsUserReviewScoreResponseReviewScore",
|
|
70
|
+
"StatsUsersGroundTruthAgreementResponse",
|
|
71
|
+
"StatsUsersGroundTruthAgreementResponseAgreementValue",
|
|
61
72
|
"StatsUsersPredictionAgreementResponse",
|
|
62
73
|
"StatsUsersPredictionAgreementResponseAgreementValue",
|
|
74
|
+
"StatsUsersReviewScoreResponse",
|
|
75
|
+
"StatsUsersReviewScoreResponsePerformanceScoreValue",
|
|
76
|
+
"StatsUsersReviewScoreResponseReviewScoreValue",
|
|
63
77
|
]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# This file was auto-generated by Fern from our API Definition.
|
|
2
|
+
|
|
3
|
+
from ....core.unchecked_base_model import UncheckedBaseModel
|
|
4
|
+
import typing
|
|
5
|
+
from .stats_users_ground_truth_agreement_response_agreement_value import (
|
|
6
|
+
StatsUsersGroundTruthAgreementResponseAgreementValue,
|
|
7
|
+
)
|
|
8
|
+
import pydantic
|
|
9
|
+
from ....core.pydantic_utilities import IS_PYDANTIC_V2
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class StatsUsersGroundTruthAgreementResponse(UncheckedBaseModel):
|
|
13
|
+
agreement: typing.Optional[typing.Dict[str, StatsUsersGroundTruthAgreementResponseAgreementValue]] = pydantic.Field(
|
|
14
|
+
default=None
|
|
15
|
+
)
|
|
16
|
+
"""
|
|
17
|
+
Dictionary mapping user IDs to their ground truth agreement scores
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
if IS_PYDANTIC_V2:
|
|
21
|
+
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
|
|
22
|
+
else:
|
|
23
|
+
|
|
24
|
+
class Config:
|
|
25
|
+
frozen = True
|
|
26
|
+
smart_union = True
|
|
27
|
+
extra = pydantic.Extra.allow
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# This file was auto-generated by Fern from our API Definition.
|
|
2
|
+
|
|
3
|
+
from ....core.unchecked_base_model import UncheckedBaseModel
|
|
4
|
+
import typing
|
|
5
|
+
from .stats_users_review_score_response_performance_score_value import (
|
|
6
|
+
StatsUsersReviewScoreResponsePerformanceScoreValue,
|
|
7
|
+
)
|
|
8
|
+
import pydantic
|
|
9
|
+
from .stats_users_review_score_response_review_score_value import StatsUsersReviewScoreResponseReviewScoreValue
|
|
10
|
+
from ....core.pydantic_utilities import IS_PYDANTIC_V2
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class StatsUsersReviewScoreResponse(UncheckedBaseModel):
|
|
14
|
+
performance_score: typing.Optional[typing.Dict[str, StatsUsersReviewScoreResponsePerformanceScoreValue]] = (
|
|
15
|
+
pydantic.Field(default=None)
|
|
16
|
+
)
|
|
17
|
+
"""
|
|
18
|
+
Performance scores mapped by annotator ID
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
review_score: typing.Optional[typing.Dict[str, StatsUsersReviewScoreResponseReviewScoreValue]] = pydantic.Field(
|
|
22
|
+
default=None
|
|
23
|
+
)
|
|
24
|
+
"""
|
|
25
|
+
Review scores mapped by annotator ID
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
if IS_PYDANTIC_V2:
|
|
29
|
+
model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
|
|
30
|
+
else:
|
|
31
|
+
|
|
32
|
+
class Config:
|
|
33
|
+
frozen = True
|
|
34
|
+
smart_union = True
|
|
35
|
+
extra = pydantic.Extra.allow
|
|
@@ -515,6 +515,12 @@ class VersionsClient:
|
|
|
515
515
|
request_options: typing.Optional[RequestOptions] = None,
|
|
516
516
|
) -> RefinedPromptResponse:
|
|
517
517
|
"""
|
|
518
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
519
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
520
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
521
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
522
|
+
</p>
|
|
523
|
+
</Card>
|
|
518
524
|
Get the refined prompt based on the `refinement_job_id`.
|
|
519
525
|
|
|
520
526
|
Parameters
|
|
@@ -580,6 +586,12 @@ class VersionsClient:
|
|
|
580
586
|
request_options: typing.Optional[RequestOptions] = None,
|
|
581
587
|
) -> RefinedPromptResponse:
|
|
582
588
|
"""
|
|
589
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
590
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
591
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
592
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
593
|
+
</p>
|
|
594
|
+
</Card>
|
|
583
595
|
Refine a prompt version using a teacher model and save the refined prompt as a new version.
|
|
584
596
|
|
|
585
597
|
Parameters
|
|
@@ -1211,6 +1223,12 @@ class AsyncVersionsClient:
|
|
|
1211
1223
|
request_options: typing.Optional[RequestOptions] = None,
|
|
1212
1224
|
) -> RefinedPromptResponse:
|
|
1213
1225
|
"""
|
|
1226
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1227
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1228
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1229
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1230
|
+
</p>
|
|
1231
|
+
</Card>
|
|
1214
1232
|
Get the refined prompt based on the `refinement_job_id`.
|
|
1215
1233
|
|
|
1216
1234
|
Parameters
|
|
@@ -1284,6 +1302,12 @@ class AsyncVersionsClient:
|
|
|
1284
1302
|
request_options: typing.Optional[RequestOptions] = None,
|
|
1285
1303
|
) -> RefinedPromptResponse:
|
|
1286
1304
|
"""
|
|
1305
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1306
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1307
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1308
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1309
|
+
</p>
|
|
1310
|
+
</Card>
|
|
1287
1311
|
Refine a prompt version using a teacher model and save the refined prompt as a new version.
|
|
1288
1312
|
|
|
1289
1313
|
Parameters
|
label_studio_sdk/tasks/client.py
CHANGED
|
@@ -519,6 +519,7 @@ class TasksClient:
|
|
|
519
519
|
last_comment_updated_at: typing.Optional[dt.datetime] = OMIT,
|
|
520
520
|
meta: typing.Optional[typing.Optional[typing.Any]] = OMIT,
|
|
521
521
|
overlap: typing.Optional[int] = OMIT,
|
|
522
|
+
precomputed_agreement: typing.Optional[float] = OMIT,
|
|
522
523
|
predictions_score: typing.Optional[float] = OMIT,
|
|
523
524
|
project: typing.Optional[int] = OMIT,
|
|
524
525
|
reviewed: typing.Optional[bool] = OMIT,
|
|
@@ -565,6 +566,9 @@ class TasksClient:
|
|
|
565
566
|
overlap : typing.Optional[int]
|
|
566
567
|
Number of distinct annotators that processed the current task
|
|
567
568
|
|
|
569
|
+
precomputed_agreement : typing.Optional[float]
|
|
570
|
+
Average agreement score for the task
|
|
571
|
+
|
|
568
572
|
predictions_score : typing.Optional[float]
|
|
569
573
|
|
|
570
574
|
project : typing.Optional[int]
|
|
@@ -618,6 +622,7 @@ class TasksClient:
|
|
|
618
622
|
"last_comment_updated_at": last_comment_updated_at,
|
|
619
623
|
"meta": meta,
|
|
620
624
|
"overlap": overlap,
|
|
625
|
+
"precomputed_agreement": precomputed_agreement,
|
|
621
626
|
"predictions_score": predictions_score,
|
|
622
627
|
"project": project,
|
|
623
628
|
"reviewed": reviewed,
|
|
@@ -1373,6 +1378,7 @@ class AsyncTasksClient:
|
|
|
1373
1378
|
last_comment_updated_at: typing.Optional[dt.datetime] = OMIT,
|
|
1374
1379
|
meta: typing.Optional[typing.Optional[typing.Any]] = OMIT,
|
|
1375
1380
|
overlap: typing.Optional[int] = OMIT,
|
|
1381
|
+
precomputed_agreement: typing.Optional[float] = OMIT,
|
|
1376
1382
|
predictions_score: typing.Optional[float] = OMIT,
|
|
1377
1383
|
project: typing.Optional[int] = OMIT,
|
|
1378
1384
|
reviewed: typing.Optional[bool] = OMIT,
|
|
@@ -1419,6 +1425,9 @@ class AsyncTasksClient:
|
|
|
1419
1425
|
overlap : typing.Optional[int]
|
|
1420
1426
|
Number of distinct annotators that processed the current task
|
|
1421
1427
|
|
|
1428
|
+
precomputed_agreement : typing.Optional[float]
|
|
1429
|
+
Average agreement score for the task
|
|
1430
|
+
|
|
1422
1431
|
predictions_score : typing.Optional[float]
|
|
1423
1432
|
|
|
1424
1433
|
project : typing.Optional[int]
|
|
@@ -1480,6 +1489,7 @@ class AsyncTasksClient:
|
|
|
1480
1489
|
"last_comment_updated_at": last_comment_updated_at,
|
|
1481
1490
|
"meta": meta,
|
|
1482
1491
|
"overlap": overlap,
|
|
1492
|
+
"precomputed_agreement": precomputed_agreement,
|
|
1483
1493
|
"predictions_score": predictions_score,
|
|
1484
1494
|
"project": project,
|
|
1485
1495
|
"reviewed": reviewed,
|