label-studio-sdk 2.0.10__py3-none-any.whl → 2.0.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of label-studio-sdk might be problematic. Click here for more details.
- label_studio_sdk/__init__.py +10 -0
- label_studio_sdk/activity_logs/client.py +12 -0
- label_studio_sdk/annotation_history/client.py +36 -0
- label_studio_sdk/annotation_reviews/client.py +60 -0
- label_studio_sdk/billing/client.py +12 -0
- label_studio_sdk/comments/client.py +72 -0
- label_studio_sdk/export_storage/azure_spi/client.py +84 -0
- label_studio_sdk/export_storage/databricks/client.py +84 -0
- label_studio_sdk/export_storage/gcswif/client.py +84 -0
- label_studio_sdk/export_storage/s3s/client.py +84 -0
- label_studio_sdk/import_storage/azure_spi/client.py +84 -0
- label_studio_sdk/import_storage/databricks/client.py +84 -0
- label_studio_sdk/import_storage/gcswif/client.py +84 -0
- label_studio_sdk/import_storage/s3s/client.py +84 -0
- label_studio_sdk/label_interface/control_tags.py +12 -7
- label_studio_sdk/ml/client.py +14 -2
- label_studio_sdk/model_providers/client.py +72 -0
- label_studio_sdk/organizations/client.py +30 -6
- label_studio_sdk/organizations/invites/client.py +24 -0
- label_studio_sdk/organizations/members/client.py +24 -0
- label_studio_sdk/organizations/permissions/client.py +60 -0
- label_studio_sdk/project_templates/client.py +76 -4
- label_studio_sdk/projects/__init__.py +14 -1
- label_studio_sdk/projects/assignments/client.py +60 -0
- label_studio_sdk/projects/client.py +20 -0
- label_studio_sdk/projects/members/bulk/client.py +24 -0
- label_studio_sdk/projects/members/client.py +149 -30
- label_studio_sdk/projects/members/paginated/client.py +12 -0
- label_studio_sdk/projects/metrics/client.py +24 -0
- label_studio_sdk/projects/pauses/client.py +60 -0
- label_studio_sdk/projects/roles/__init__.py +2 -0
- label_studio_sdk/projects/roles/client.py +555 -0
- label_studio_sdk/projects/stats/__init__.py +12 -0
- label_studio_sdk/projects/stats/client.py +766 -1
- label_studio_sdk/projects/stats/types/__init__.py +14 -0
- label_studio_sdk/projects/stats/types/stats_agreement_annotators_response.py +22 -0
- label_studio_sdk/projects/stats/types/stats_model_version_annotator_agreement_response.py +19 -0
- label_studio_sdk/projects/stats/types/stats_model_version_ground_truth_agreement_response.py +19 -0
- label_studio_sdk/projects/stats/types/stats_model_version_prediction_agreement_response.py +19 -0
- label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response.py +27 -0
- label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response_agreement_value.py +5 -0
- label_studio_sdk/prompts/client.py +96 -0
- label_studio_sdk/prompts/indicators/client.py +24 -0
- label_studio_sdk/prompts/runs/client.py +36 -0
- label_studio_sdk/prompts/versions/client.py +84 -0
- label_studio_sdk/sso/saml/client.py +24 -0
- label_studio_sdk/sso/scim/client.py +24 -0
- label_studio_sdk/types/__init__.py +10 -0
- label_studio_sdk/types/all_roles_project_list.py +5 -1
- label_studio_sdk/types/default_role.py +2 -2
- label_studio_sdk/types/lse_organization.py +1 -1
- label_studio_sdk/types/lse_project.py +5 -1
- label_studio_sdk/types/lse_project_create.py +5 -1
- label_studio_sdk/types/lse_project_response.py +5 -1
- label_studio_sdk/types/lse_project_update.py +5 -1
- label_studio_sdk/types/project.py +198 -0
- label_studio_sdk/types/project_member.py +19 -0
- label_studio_sdk/types/project_role.py +35 -0
- label_studio_sdk/types/project_sampling.py +7 -0
- label_studio_sdk/types/project_skip_queue.py +7 -0
- label_studio_sdk/types/project_template.py +1 -1
- label_studio_sdk/types/project_template_request.py +1 -1
- label_studio_sdk/users/client.py +24 -0
- label_studio_sdk/versions/client.py +12 -0
- label_studio_sdk/workspaces/__init__.py +2 -2
- label_studio_sdk/workspaces/client.py +4 -0
- label_studio_sdk/workspaces/projects/__init__.py +2 -0
- label_studio_sdk/workspaces/projects/client.py +352 -0
- {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/METADATA +1 -1
- {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/RECORD +72 -57
- {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/LICENSE +0 -0
- {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/WHEEL +0 -0
|
@@ -3,16 +3,21 @@
|
|
|
3
3
|
from ...core.client_wrapper import SyncClientWrapper
|
|
4
4
|
import typing
|
|
5
5
|
from ...core.request_options import RequestOptions
|
|
6
|
-
from .types.
|
|
6
|
+
from .types.stats_model_version_annotator_agreement_response import StatsModelVersionAnnotatorAgreementResponse
|
|
7
7
|
from ...core.jsonable_encoder import jsonable_encoder
|
|
8
8
|
from ...core.unchecked_base_model import construct_type
|
|
9
9
|
from json.decoder import JSONDecodeError
|
|
10
10
|
from ...core.api_error import ApiError
|
|
11
|
+
from .types.stats_model_version_ground_truth_agreement_response import StatsModelVersionGroundTruthAgreementResponse
|
|
12
|
+
from .types.stats_model_version_prediction_agreement_response import StatsModelVersionPredictionAgreementResponse
|
|
13
|
+
from .types.stats_iaa_response import StatsIaaResponse
|
|
11
14
|
from .types.stats_agreement_annotator_response import StatsAgreementAnnotatorResponse
|
|
15
|
+
from .types.stats_agreement_annotators_response import StatsAgreementAnnotatorsResponse
|
|
12
16
|
from .types.stats_data_filters_response import StatsDataFiltersResponse
|
|
13
17
|
from .types.stats_finished_tasks_response import StatsFinishedTasksResponse
|
|
14
18
|
from .types.stats_lead_time_response import StatsLeadTimeResponse
|
|
15
19
|
from .types.stats_total_agreement_response import StatsTotalAgreementResponse
|
|
20
|
+
from .types.stats_users_prediction_agreement_response import StatsUsersPredictionAgreementResponse
|
|
16
21
|
from .types.stats_user_prediction_agreement_response import StatsUserPredictionAgreementResponse
|
|
17
22
|
from .types.stats_user_review_score_response import StatsUserReviewScoreResponse
|
|
18
23
|
from .types.stats_user_ground_truth_agreement_response import StatsUserGroundTruthAgreementResponse
|
|
@@ -23,6 +28,199 @@ class StatsClient:
|
|
|
23
28
|
def __init__(self, *, client_wrapper: SyncClientWrapper):
|
|
24
29
|
self._client_wrapper = client_wrapper
|
|
25
30
|
|
|
31
|
+
def model_version_annotator_agreement(
|
|
32
|
+
self, id: int, model_version: str, *, request_options: typing.Optional[RequestOptions] = None
|
|
33
|
+
) -> StatsModelVersionAnnotatorAgreementResponse:
|
|
34
|
+
"""
|
|
35
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
36
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
37
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
38
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
39
|
+
</p>
|
|
40
|
+
</Card>
|
|
41
|
+
Get agreement between a given model version and all annotators in the project for overlapping tasks.
|
|
42
|
+
|
|
43
|
+
Parameters
|
|
44
|
+
----------
|
|
45
|
+
id : int
|
|
46
|
+
|
|
47
|
+
model_version : str
|
|
48
|
+
|
|
49
|
+
request_options : typing.Optional[RequestOptions]
|
|
50
|
+
Request-specific configuration.
|
|
51
|
+
|
|
52
|
+
Returns
|
|
53
|
+
-------
|
|
54
|
+
StatsModelVersionAnnotatorAgreementResponse
|
|
55
|
+
Model-version overall agreement vs annotators
|
|
56
|
+
|
|
57
|
+
Examples
|
|
58
|
+
--------
|
|
59
|
+
from label_studio_sdk import LabelStudio
|
|
60
|
+
|
|
61
|
+
client = LabelStudio(
|
|
62
|
+
api_key="YOUR_API_KEY",
|
|
63
|
+
)
|
|
64
|
+
client.projects.stats.model_version_annotator_agreement(
|
|
65
|
+
id=1,
|
|
66
|
+
model_version="model_version",
|
|
67
|
+
)
|
|
68
|
+
"""
|
|
69
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
70
|
+
f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement",
|
|
71
|
+
method="GET",
|
|
72
|
+
request_options=request_options,
|
|
73
|
+
)
|
|
74
|
+
try:
|
|
75
|
+
if 200 <= _response.status_code < 300:
|
|
76
|
+
return typing.cast(
|
|
77
|
+
StatsModelVersionAnnotatorAgreementResponse,
|
|
78
|
+
construct_type(
|
|
79
|
+
type_=StatsModelVersionAnnotatorAgreementResponse, # type: ignore
|
|
80
|
+
object_=_response.json(),
|
|
81
|
+
),
|
|
82
|
+
)
|
|
83
|
+
_response_json = _response.json()
|
|
84
|
+
except JSONDecodeError:
|
|
85
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
86
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
87
|
+
|
|
88
|
+
def model_version_ground_truth_agreement(
|
|
89
|
+
self,
|
|
90
|
+
id: int,
|
|
91
|
+
model_version: str,
|
|
92
|
+
*,
|
|
93
|
+
per_label: typing.Optional[bool] = None,
|
|
94
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
95
|
+
) -> StatsModelVersionGroundTruthAgreementResponse:
|
|
96
|
+
"""
|
|
97
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
98
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
99
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
100
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
101
|
+
</p>
|
|
102
|
+
</Card>
|
|
103
|
+
Get agreement between a given model version and ground truth annotations in the project for overlapping tasks.
|
|
104
|
+
|
|
105
|
+
Parameters
|
|
106
|
+
----------
|
|
107
|
+
id : int
|
|
108
|
+
|
|
109
|
+
model_version : str
|
|
110
|
+
|
|
111
|
+
per_label : typing.Optional[bool]
|
|
112
|
+
Calculate agreement per label
|
|
113
|
+
|
|
114
|
+
request_options : typing.Optional[RequestOptions]
|
|
115
|
+
Request-specific configuration.
|
|
116
|
+
|
|
117
|
+
Returns
|
|
118
|
+
-------
|
|
119
|
+
StatsModelVersionGroundTruthAgreementResponse
|
|
120
|
+
Model-version ground truth agreement
|
|
121
|
+
|
|
122
|
+
Examples
|
|
123
|
+
--------
|
|
124
|
+
from label_studio_sdk import LabelStudio
|
|
125
|
+
|
|
126
|
+
client = LabelStudio(
|
|
127
|
+
api_key="YOUR_API_KEY",
|
|
128
|
+
)
|
|
129
|
+
client.projects.stats.model_version_ground_truth_agreement(
|
|
130
|
+
id=1,
|
|
131
|
+
model_version="model_version",
|
|
132
|
+
)
|
|
133
|
+
"""
|
|
134
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
135
|
+
f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement-groundtruth",
|
|
136
|
+
method="GET",
|
|
137
|
+
params={
|
|
138
|
+
"per_label": per_label,
|
|
139
|
+
},
|
|
140
|
+
request_options=request_options,
|
|
141
|
+
)
|
|
142
|
+
try:
|
|
143
|
+
if 200 <= _response.status_code < 300:
|
|
144
|
+
return typing.cast(
|
|
145
|
+
StatsModelVersionGroundTruthAgreementResponse,
|
|
146
|
+
construct_type(
|
|
147
|
+
type_=StatsModelVersionGroundTruthAgreementResponse, # type: ignore
|
|
148
|
+
object_=_response.json(),
|
|
149
|
+
),
|
|
150
|
+
)
|
|
151
|
+
_response_json = _response.json()
|
|
152
|
+
except JSONDecodeError:
|
|
153
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
154
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
155
|
+
|
|
156
|
+
def model_version_prediction_agreement(
|
|
157
|
+
self,
|
|
158
|
+
id: int,
|
|
159
|
+
model_version: str,
|
|
160
|
+
*,
|
|
161
|
+
per_label: typing.Optional[bool] = None,
|
|
162
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
163
|
+
) -> StatsModelVersionPredictionAgreementResponse:
|
|
164
|
+
"""
|
|
165
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
166
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
167
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
168
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
169
|
+
</p>
|
|
170
|
+
</Card>
|
|
171
|
+
Get agreement between a given model version and all other model versions in the project for overlapping tasks.
|
|
172
|
+
|
|
173
|
+
Parameters
|
|
174
|
+
----------
|
|
175
|
+
id : int
|
|
176
|
+
|
|
177
|
+
model_version : str
|
|
178
|
+
|
|
179
|
+
per_label : typing.Optional[bool]
|
|
180
|
+
Calculate agreement per label
|
|
181
|
+
|
|
182
|
+
request_options : typing.Optional[RequestOptions]
|
|
183
|
+
Request-specific configuration.
|
|
184
|
+
|
|
185
|
+
Returns
|
|
186
|
+
-------
|
|
187
|
+
StatsModelVersionPredictionAgreementResponse
|
|
188
|
+
Model-version prediction agreement
|
|
189
|
+
|
|
190
|
+
Examples
|
|
191
|
+
--------
|
|
192
|
+
from label_studio_sdk import LabelStudio
|
|
193
|
+
|
|
194
|
+
client = LabelStudio(
|
|
195
|
+
api_key="YOUR_API_KEY",
|
|
196
|
+
)
|
|
197
|
+
client.projects.stats.model_version_prediction_agreement(
|
|
198
|
+
id=1,
|
|
199
|
+
model_version="model_version",
|
|
200
|
+
)
|
|
201
|
+
"""
|
|
202
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
203
|
+
f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/prediction",
|
|
204
|
+
method="GET",
|
|
205
|
+
params={
|
|
206
|
+
"per_label": per_label,
|
|
207
|
+
},
|
|
208
|
+
request_options=request_options,
|
|
209
|
+
)
|
|
210
|
+
try:
|
|
211
|
+
if 200 <= _response.status_code < 300:
|
|
212
|
+
return typing.cast(
|
|
213
|
+
StatsModelVersionPredictionAgreementResponse,
|
|
214
|
+
construct_type(
|
|
215
|
+
type_=StatsModelVersionPredictionAgreementResponse, # type: ignore
|
|
216
|
+
object_=_response.json(),
|
|
217
|
+
),
|
|
218
|
+
)
|
|
219
|
+
_response_json = _response.json()
|
|
220
|
+
except JSONDecodeError:
|
|
221
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
222
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
223
|
+
|
|
26
224
|
def iaa(
|
|
27
225
|
self,
|
|
28
226
|
id: int,
|
|
@@ -34,6 +232,12 @@ class StatsClient:
|
|
|
34
232
|
request_options: typing.Optional[RequestOptions] = None,
|
|
35
233
|
) -> StatsIaaResponse:
|
|
36
234
|
"""
|
|
235
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
236
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
237
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
238
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
239
|
+
</p>
|
|
240
|
+
</Card>
|
|
37
241
|
Get Inter-Annotator Agreement (IAA) matrix for a project, showing agreement between all annotators.
|
|
38
242
|
|
|
39
243
|
Parameters
|
|
@@ -100,6 +304,12 @@ class StatsClient:
|
|
|
100
304
|
self, id: int, user_id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
101
305
|
) -> StatsAgreementAnnotatorResponse:
|
|
102
306
|
"""
|
|
307
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
308
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
309
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
310
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
311
|
+
</p>
|
|
312
|
+
</Card>
|
|
103
313
|
Get agreement statistics for a specific annotator within a project.
|
|
104
314
|
|
|
105
315
|
Parameters
|
|
@@ -147,6 +357,67 @@ class StatsClient:
|
|
|
147
357
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
148
358
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
149
359
|
|
|
360
|
+
def agreement_annotators(
|
|
361
|
+
self, id: int, *, ids: str, request_options: typing.Optional[RequestOptions] = None
|
|
362
|
+
) -> StatsAgreementAnnotatorsResponse:
|
|
363
|
+
"""
|
|
364
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
365
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
366
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
367
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
368
|
+
</p>
|
|
369
|
+
</Card>
|
|
370
|
+
Get agreement statistics for multiple annotators within a project.
|
|
371
|
+
|
|
372
|
+
Parameters
|
|
373
|
+
----------
|
|
374
|
+
id : int
|
|
375
|
+
|
|
376
|
+
ids : str
|
|
377
|
+
Comma separated list of annotator user IDs to get agreement scores for
|
|
378
|
+
|
|
379
|
+
request_options : typing.Optional[RequestOptions]
|
|
380
|
+
Request-specific configuration.
|
|
381
|
+
|
|
382
|
+
Returns
|
|
383
|
+
-------
|
|
384
|
+
StatsAgreementAnnotatorsResponse
|
|
385
|
+
Multiple annotator agreement statistics
|
|
386
|
+
|
|
387
|
+
Examples
|
|
388
|
+
--------
|
|
389
|
+
from label_studio_sdk import LabelStudio
|
|
390
|
+
|
|
391
|
+
client = LabelStudio(
|
|
392
|
+
api_key="YOUR_API_KEY",
|
|
393
|
+
)
|
|
394
|
+
client.projects.stats.agreement_annotators(
|
|
395
|
+
id=1,
|
|
396
|
+
ids="ids",
|
|
397
|
+
)
|
|
398
|
+
"""
|
|
399
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
400
|
+
f"api/projects/{jsonable_encoder(id)}/stats/agreement_annotators",
|
|
401
|
+
method="GET",
|
|
402
|
+
params={
|
|
403
|
+
"ids": ids,
|
|
404
|
+
},
|
|
405
|
+
request_options=request_options,
|
|
406
|
+
)
|
|
407
|
+
try:
|
|
408
|
+
if 200 <= _response.status_code < 300:
|
|
409
|
+
return typing.cast(
|
|
410
|
+
StatsAgreementAnnotatorsResponse,
|
|
411
|
+
construct_type(
|
|
412
|
+
type_=StatsAgreementAnnotatorsResponse, # type: ignore
|
|
413
|
+
object_=_response.json(),
|
|
414
|
+
),
|
|
415
|
+
)
|
|
416
|
+
_response_json = _response.json()
|
|
417
|
+
except JSONDecodeError:
|
|
418
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
419
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
420
|
+
|
|
150
421
|
def data_filters(
|
|
151
422
|
self, id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
152
423
|
) -> StatsDataFiltersResponse:
|
|
@@ -303,6 +574,12 @@ class StatsClient:
|
|
|
303
574
|
request_options: typing.Optional[RequestOptions] = None,
|
|
304
575
|
) -> StatsTotalAgreementResponse:
|
|
305
576
|
"""
|
|
577
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
578
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
579
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
580
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
581
|
+
</p>
|
|
582
|
+
</Card>
|
|
306
583
|
Overall or per-label total agreement across the project.
|
|
307
584
|
|
|
308
585
|
NOTE: due to an open issue in Fern, SDK clients will raise ApiError upon handling a 204 response. As a workaround, wrap call to this function in a try-except block.
|
|
@@ -413,6 +690,76 @@ class StatsClient:
|
|
|
413
690
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
414
691
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
415
692
|
|
|
693
|
+
def users_prediction_agreement(
|
|
694
|
+
self,
|
|
695
|
+
id: int,
|
|
696
|
+
*,
|
|
697
|
+
ids: str,
|
|
698
|
+
per_label: typing.Optional[bool] = None,
|
|
699
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
700
|
+
) -> StatsUsersPredictionAgreementResponse:
|
|
701
|
+
"""
|
|
702
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
703
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
704
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
705
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
706
|
+
</p>
|
|
707
|
+
</Card>
|
|
708
|
+
Get prediction agreement statistics for multiple annotators within a project.
|
|
709
|
+
|
|
710
|
+
Parameters
|
|
711
|
+
----------
|
|
712
|
+
id : int
|
|
713
|
+
|
|
714
|
+
ids : str
|
|
715
|
+
Comma separated list of annotator user IDs to get agreement scores for
|
|
716
|
+
|
|
717
|
+
per_label : typing.Optional[bool]
|
|
718
|
+
Per label
|
|
719
|
+
|
|
720
|
+
request_options : typing.Optional[RequestOptions]
|
|
721
|
+
Request-specific configuration.
|
|
722
|
+
|
|
723
|
+
Returns
|
|
724
|
+
-------
|
|
725
|
+
StatsUsersPredictionAgreementResponse
|
|
726
|
+
Prediction agreement statistics for multiple annotators
|
|
727
|
+
|
|
728
|
+
Examples
|
|
729
|
+
--------
|
|
730
|
+
from label_studio_sdk import LabelStudio
|
|
731
|
+
|
|
732
|
+
client = LabelStudio(
|
|
733
|
+
api_key="YOUR_API_KEY",
|
|
734
|
+
)
|
|
735
|
+
client.projects.stats.users_prediction_agreement(
|
|
736
|
+
id=1,
|
|
737
|
+
ids="ids",
|
|
738
|
+
)
|
|
739
|
+
"""
|
|
740
|
+
_response = self._client_wrapper.httpx_client.request(
|
|
741
|
+
f"api/projects/{jsonable_encoder(id)}/user-stats/prediction",
|
|
742
|
+
method="GET",
|
|
743
|
+
params={
|
|
744
|
+
"ids": ids,
|
|
745
|
+
"per_label": per_label,
|
|
746
|
+
},
|
|
747
|
+
request_options=request_options,
|
|
748
|
+
)
|
|
749
|
+
try:
|
|
750
|
+
if 200 <= _response.status_code < 300:
|
|
751
|
+
return typing.cast(
|
|
752
|
+
StatsUsersPredictionAgreementResponse,
|
|
753
|
+
construct_type(
|
|
754
|
+
type_=StatsUsersPredictionAgreementResponse, # type: ignore
|
|
755
|
+
object_=_response.json(),
|
|
756
|
+
),
|
|
757
|
+
)
|
|
758
|
+
_response_json = _response.json()
|
|
759
|
+
except JSONDecodeError:
|
|
760
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
761
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
762
|
+
|
|
416
763
|
def user_prediction_agreement(
|
|
417
764
|
self,
|
|
418
765
|
id: int,
|
|
@@ -422,6 +769,12 @@ class StatsClient:
|
|
|
422
769
|
request_options: typing.Optional[RequestOptions] = None,
|
|
423
770
|
) -> StatsUserPredictionAgreementResponse:
|
|
424
771
|
"""
|
|
772
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
773
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
774
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
775
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
776
|
+
</p>
|
|
777
|
+
</Card>
|
|
425
778
|
Get prediction agreement statistics for a specific user within a project.
|
|
426
779
|
|
|
427
780
|
Parameters
|
|
@@ -484,6 +837,12 @@ class StatsClient:
|
|
|
484
837
|
request_options: typing.Optional[RequestOptions] = None,
|
|
485
838
|
) -> StatsUserReviewScoreResponse:
|
|
486
839
|
"""
|
|
840
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
841
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
842
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
843
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
844
|
+
</p>
|
|
845
|
+
</Card>
|
|
487
846
|
Get review score statistics for a specific user within a project.
|
|
488
847
|
|
|
489
848
|
Parameters
|
|
@@ -546,6 +905,12 @@ class StatsClient:
|
|
|
546
905
|
request_options: typing.Optional[RequestOptions] = None,
|
|
547
906
|
) -> StatsUserGroundTruthAgreementResponse:
|
|
548
907
|
"""
|
|
908
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
909
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
910
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
911
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
912
|
+
</p>
|
|
913
|
+
</Card>
|
|
549
914
|
Get ground truth agreement statistics for a specific user within a project.
|
|
550
915
|
|
|
551
916
|
Parameters
|
|
@@ -604,6 +969,223 @@ class AsyncStatsClient:
|
|
|
604
969
|
def __init__(self, *, client_wrapper: AsyncClientWrapper):
|
|
605
970
|
self._client_wrapper = client_wrapper
|
|
606
971
|
|
|
972
|
+
async def model_version_annotator_agreement(
|
|
973
|
+
self, id: int, model_version: str, *, request_options: typing.Optional[RequestOptions] = None
|
|
974
|
+
) -> StatsModelVersionAnnotatorAgreementResponse:
|
|
975
|
+
"""
|
|
976
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
977
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
978
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
979
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
980
|
+
</p>
|
|
981
|
+
</Card>
|
|
982
|
+
Get agreement between a given model version and all annotators in the project for overlapping tasks.
|
|
983
|
+
|
|
984
|
+
Parameters
|
|
985
|
+
----------
|
|
986
|
+
id : int
|
|
987
|
+
|
|
988
|
+
model_version : str
|
|
989
|
+
|
|
990
|
+
request_options : typing.Optional[RequestOptions]
|
|
991
|
+
Request-specific configuration.
|
|
992
|
+
|
|
993
|
+
Returns
|
|
994
|
+
-------
|
|
995
|
+
StatsModelVersionAnnotatorAgreementResponse
|
|
996
|
+
Model-version overall agreement vs annotators
|
|
997
|
+
|
|
998
|
+
Examples
|
|
999
|
+
--------
|
|
1000
|
+
import asyncio
|
|
1001
|
+
|
|
1002
|
+
from label_studio_sdk import AsyncLabelStudio
|
|
1003
|
+
|
|
1004
|
+
client = AsyncLabelStudio(
|
|
1005
|
+
api_key="YOUR_API_KEY",
|
|
1006
|
+
)
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
async def main() -> None:
|
|
1010
|
+
await client.projects.stats.model_version_annotator_agreement(
|
|
1011
|
+
id=1,
|
|
1012
|
+
model_version="model_version",
|
|
1013
|
+
)
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
asyncio.run(main())
|
|
1017
|
+
"""
|
|
1018
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
1019
|
+
f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement",
|
|
1020
|
+
method="GET",
|
|
1021
|
+
request_options=request_options,
|
|
1022
|
+
)
|
|
1023
|
+
try:
|
|
1024
|
+
if 200 <= _response.status_code < 300:
|
|
1025
|
+
return typing.cast(
|
|
1026
|
+
StatsModelVersionAnnotatorAgreementResponse,
|
|
1027
|
+
construct_type(
|
|
1028
|
+
type_=StatsModelVersionAnnotatorAgreementResponse, # type: ignore
|
|
1029
|
+
object_=_response.json(),
|
|
1030
|
+
),
|
|
1031
|
+
)
|
|
1032
|
+
_response_json = _response.json()
|
|
1033
|
+
except JSONDecodeError:
|
|
1034
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1035
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1036
|
+
|
|
1037
|
+
async def model_version_ground_truth_agreement(
|
|
1038
|
+
self,
|
|
1039
|
+
id: int,
|
|
1040
|
+
model_version: str,
|
|
1041
|
+
*,
|
|
1042
|
+
per_label: typing.Optional[bool] = None,
|
|
1043
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
1044
|
+
) -> StatsModelVersionGroundTruthAgreementResponse:
|
|
1045
|
+
"""
|
|
1046
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1047
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1048
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1049
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1050
|
+
</p>
|
|
1051
|
+
</Card>
|
|
1052
|
+
Get agreement between a given model version and ground truth annotations in the project for overlapping tasks.
|
|
1053
|
+
|
|
1054
|
+
Parameters
|
|
1055
|
+
----------
|
|
1056
|
+
id : int
|
|
1057
|
+
|
|
1058
|
+
model_version : str
|
|
1059
|
+
|
|
1060
|
+
per_label : typing.Optional[bool]
|
|
1061
|
+
Calculate agreement per label
|
|
1062
|
+
|
|
1063
|
+
request_options : typing.Optional[RequestOptions]
|
|
1064
|
+
Request-specific configuration.
|
|
1065
|
+
|
|
1066
|
+
Returns
|
|
1067
|
+
-------
|
|
1068
|
+
StatsModelVersionGroundTruthAgreementResponse
|
|
1069
|
+
Model-version ground truth agreement
|
|
1070
|
+
|
|
1071
|
+
Examples
|
|
1072
|
+
--------
|
|
1073
|
+
import asyncio
|
|
1074
|
+
|
|
1075
|
+
from label_studio_sdk import AsyncLabelStudio
|
|
1076
|
+
|
|
1077
|
+
client = AsyncLabelStudio(
|
|
1078
|
+
api_key="YOUR_API_KEY",
|
|
1079
|
+
)
|
|
1080
|
+
|
|
1081
|
+
|
|
1082
|
+
async def main() -> None:
|
|
1083
|
+
await client.projects.stats.model_version_ground_truth_agreement(
|
|
1084
|
+
id=1,
|
|
1085
|
+
model_version="model_version",
|
|
1086
|
+
)
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
asyncio.run(main())
|
|
1090
|
+
"""
|
|
1091
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
1092
|
+
f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement-groundtruth",
|
|
1093
|
+
method="GET",
|
|
1094
|
+
params={
|
|
1095
|
+
"per_label": per_label,
|
|
1096
|
+
},
|
|
1097
|
+
request_options=request_options,
|
|
1098
|
+
)
|
|
1099
|
+
try:
|
|
1100
|
+
if 200 <= _response.status_code < 300:
|
|
1101
|
+
return typing.cast(
|
|
1102
|
+
StatsModelVersionGroundTruthAgreementResponse,
|
|
1103
|
+
construct_type(
|
|
1104
|
+
type_=StatsModelVersionGroundTruthAgreementResponse, # type: ignore
|
|
1105
|
+
object_=_response.json(),
|
|
1106
|
+
),
|
|
1107
|
+
)
|
|
1108
|
+
_response_json = _response.json()
|
|
1109
|
+
except JSONDecodeError:
|
|
1110
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1111
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1112
|
+
|
|
1113
|
+
async def model_version_prediction_agreement(
|
|
1114
|
+
self,
|
|
1115
|
+
id: int,
|
|
1116
|
+
model_version: str,
|
|
1117
|
+
*,
|
|
1118
|
+
per_label: typing.Optional[bool] = None,
|
|
1119
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
1120
|
+
) -> StatsModelVersionPredictionAgreementResponse:
|
|
1121
|
+
"""
|
|
1122
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1123
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1124
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1125
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1126
|
+
</p>
|
|
1127
|
+
</Card>
|
|
1128
|
+
Get agreement between a given model version and all other model versions in the project for overlapping tasks.
|
|
1129
|
+
|
|
1130
|
+
Parameters
|
|
1131
|
+
----------
|
|
1132
|
+
id : int
|
|
1133
|
+
|
|
1134
|
+
model_version : str
|
|
1135
|
+
|
|
1136
|
+
per_label : typing.Optional[bool]
|
|
1137
|
+
Calculate agreement per label
|
|
1138
|
+
|
|
1139
|
+
request_options : typing.Optional[RequestOptions]
|
|
1140
|
+
Request-specific configuration.
|
|
1141
|
+
|
|
1142
|
+
Returns
|
|
1143
|
+
-------
|
|
1144
|
+
StatsModelVersionPredictionAgreementResponse
|
|
1145
|
+
Model-version prediction agreement
|
|
1146
|
+
|
|
1147
|
+
Examples
|
|
1148
|
+
--------
|
|
1149
|
+
import asyncio
|
|
1150
|
+
|
|
1151
|
+
from label_studio_sdk import AsyncLabelStudio
|
|
1152
|
+
|
|
1153
|
+
client = AsyncLabelStudio(
|
|
1154
|
+
api_key="YOUR_API_KEY",
|
|
1155
|
+
)
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
async def main() -> None:
|
|
1159
|
+
await client.projects.stats.model_version_prediction_agreement(
|
|
1160
|
+
id=1,
|
|
1161
|
+
model_version="model_version",
|
|
1162
|
+
)
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
asyncio.run(main())
|
|
1166
|
+
"""
|
|
1167
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
1168
|
+
f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/prediction",
|
|
1169
|
+
method="GET",
|
|
1170
|
+
params={
|
|
1171
|
+
"per_label": per_label,
|
|
1172
|
+
},
|
|
1173
|
+
request_options=request_options,
|
|
1174
|
+
)
|
|
1175
|
+
try:
|
|
1176
|
+
if 200 <= _response.status_code < 300:
|
|
1177
|
+
return typing.cast(
|
|
1178
|
+
StatsModelVersionPredictionAgreementResponse,
|
|
1179
|
+
construct_type(
|
|
1180
|
+
type_=StatsModelVersionPredictionAgreementResponse, # type: ignore
|
|
1181
|
+
object_=_response.json(),
|
|
1182
|
+
),
|
|
1183
|
+
)
|
|
1184
|
+
_response_json = _response.json()
|
|
1185
|
+
except JSONDecodeError:
|
|
1186
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1187
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1188
|
+
|
|
607
1189
|
async def iaa(
|
|
608
1190
|
self,
|
|
609
1191
|
id: int,
|
|
@@ -615,6 +1197,12 @@ class AsyncStatsClient:
|
|
|
615
1197
|
request_options: typing.Optional[RequestOptions] = None,
|
|
616
1198
|
) -> StatsIaaResponse:
|
|
617
1199
|
"""
|
|
1200
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1201
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1202
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1203
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1204
|
+
</p>
|
|
1205
|
+
</Card>
|
|
618
1206
|
Get Inter-Annotator Agreement (IAA) matrix for a project, showing agreement between all annotators.
|
|
619
1207
|
|
|
620
1208
|
Parameters
|
|
@@ -689,6 +1277,12 @@ class AsyncStatsClient:
|
|
|
689
1277
|
self, id: int, user_id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
690
1278
|
) -> StatsAgreementAnnotatorResponse:
|
|
691
1279
|
"""
|
|
1280
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1281
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1282
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1283
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1284
|
+
</p>
|
|
1285
|
+
</Card>
|
|
692
1286
|
Get agreement statistics for a specific annotator within a project.
|
|
693
1287
|
|
|
694
1288
|
Parameters
|
|
@@ -744,6 +1338,75 @@ class AsyncStatsClient:
|
|
|
744
1338
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
745
1339
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
746
1340
|
|
|
1341
|
+
async def agreement_annotators(
|
|
1342
|
+
self, id: int, *, ids: str, request_options: typing.Optional[RequestOptions] = None
|
|
1343
|
+
) -> StatsAgreementAnnotatorsResponse:
|
|
1344
|
+
"""
|
|
1345
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1346
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1347
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1348
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1349
|
+
</p>
|
|
1350
|
+
</Card>
|
|
1351
|
+
Get agreement statistics for multiple annotators within a project.
|
|
1352
|
+
|
|
1353
|
+
Parameters
|
|
1354
|
+
----------
|
|
1355
|
+
id : int
|
|
1356
|
+
|
|
1357
|
+
ids : str
|
|
1358
|
+
Comma separated list of annotator user IDs to get agreement scores for
|
|
1359
|
+
|
|
1360
|
+
request_options : typing.Optional[RequestOptions]
|
|
1361
|
+
Request-specific configuration.
|
|
1362
|
+
|
|
1363
|
+
Returns
|
|
1364
|
+
-------
|
|
1365
|
+
StatsAgreementAnnotatorsResponse
|
|
1366
|
+
Multiple annotator agreement statistics
|
|
1367
|
+
|
|
1368
|
+
Examples
|
|
1369
|
+
--------
|
|
1370
|
+
import asyncio
|
|
1371
|
+
|
|
1372
|
+
from label_studio_sdk import AsyncLabelStudio
|
|
1373
|
+
|
|
1374
|
+
client = AsyncLabelStudio(
|
|
1375
|
+
api_key="YOUR_API_KEY",
|
|
1376
|
+
)
|
|
1377
|
+
|
|
1378
|
+
|
|
1379
|
+
async def main() -> None:
|
|
1380
|
+
await client.projects.stats.agreement_annotators(
|
|
1381
|
+
id=1,
|
|
1382
|
+
ids="ids",
|
|
1383
|
+
)
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
asyncio.run(main())
|
|
1387
|
+
"""
|
|
1388
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
1389
|
+
f"api/projects/{jsonable_encoder(id)}/stats/agreement_annotators",
|
|
1390
|
+
method="GET",
|
|
1391
|
+
params={
|
|
1392
|
+
"ids": ids,
|
|
1393
|
+
},
|
|
1394
|
+
request_options=request_options,
|
|
1395
|
+
)
|
|
1396
|
+
try:
|
|
1397
|
+
if 200 <= _response.status_code < 300:
|
|
1398
|
+
return typing.cast(
|
|
1399
|
+
StatsAgreementAnnotatorsResponse,
|
|
1400
|
+
construct_type(
|
|
1401
|
+
type_=StatsAgreementAnnotatorsResponse, # type: ignore
|
|
1402
|
+
object_=_response.json(),
|
|
1403
|
+
),
|
|
1404
|
+
)
|
|
1405
|
+
_response_json = _response.json()
|
|
1406
|
+
except JSONDecodeError:
|
|
1407
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1408
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1409
|
+
|
|
747
1410
|
async def data_filters(
|
|
748
1411
|
self, id: int, *, request_options: typing.Optional[RequestOptions] = None
|
|
749
1412
|
) -> StatsDataFiltersResponse:
|
|
@@ -926,6 +1589,12 @@ class AsyncStatsClient:
|
|
|
926
1589
|
request_options: typing.Optional[RequestOptions] = None,
|
|
927
1590
|
) -> StatsTotalAgreementResponse:
|
|
928
1591
|
"""
|
|
1592
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1593
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1594
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1595
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1596
|
+
</p>
|
|
1597
|
+
</Card>
|
|
929
1598
|
Overall or per-label total agreement across the project.
|
|
930
1599
|
|
|
931
1600
|
NOTE: due to an open issue in Fern, SDK clients will raise ApiError upon handling a 204 response. As a workaround, wrap call to this function in a try-except block.
|
|
@@ -1052,6 +1721,84 @@ class AsyncStatsClient:
|
|
|
1052
1721
|
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1053
1722
|
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1054
1723
|
|
|
1724
|
+
async def users_prediction_agreement(
|
|
1725
|
+
self,
|
|
1726
|
+
id: int,
|
|
1727
|
+
*,
|
|
1728
|
+
ids: str,
|
|
1729
|
+
per_label: typing.Optional[bool] = None,
|
|
1730
|
+
request_options: typing.Optional[RequestOptions] = None,
|
|
1731
|
+
) -> StatsUsersPredictionAgreementResponse:
|
|
1732
|
+
"""
|
|
1733
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1734
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1735
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1736
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1737
|
+
</p>
|
|
1738
|
+
</Card>
|
|
1739
|
+
Get prediction agreement statistics for multiple annotators within a project.
|
|
1740
|
+
|
|
1741
|
+
Parameters
|
|
1742
|
+
----------
|
|
1743
|
+
id : int
|
|
1744
|
+
|
|
1745
|
+
ids : str
|
|
1746
|
+
Comma separated list of annotator user IDs to get agreement scores for
|
|
1747
|
+
|
|
1748
|
+
per_label : typing.Optional[bool]
|
|
1749
|
+
Per label
|
|
1750
|
+
|
|
1751
|
+
request_options : typing.Optional[RequestOptions]
|
|
1752
|
+
Request-specific configuration.
|
|
1753
|
+
|
|
1754
|
+
Returns
|
|
1755
|
+
-------
|
|
1756
|
+
StatsUsersPredictionAgreementResponse
|
|
1757
|
+
Prediction agreement statistics for multiple annotators
|
|
1758
|
+
|
|
1759
|
+
Examples
|
|
1760
|
+
--------
|
|
1761
|
+
import asyncio
|
|
1762
|
+
|
|
1763
|
+
from label_studio_sdk import AsyncLabelStudio
|
|
1764
|
+
|
|
1765
|
+
client = AsyncLabelStudio(
|
|
1766
|
+
api_key="YOUR_API_KEY",
|
|
1767
|
+
)
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
async def main() -> None:
|
|
1771
|
+
await client.projects.stats.users_prediction_agreement(
|
|
1772
|
+
id=1,
|
|
1773
|
+
ids="ids",
|
|
1774
|
+
)
|
|
1775
|
+
|
|
1776
|
+
|
|
1777
|
+
asyncio.run(main())
|
|
1778
|
+
"""
|
|
1779
|
+
_response = await self._client_wrapper.httpx_client.request(
|
|
1780
|
+
f"api/projects/{jsonable_encoder(id)}/user-stats/prediction",
|
|
1781
|
+
method="GET",
|
|
1782
|
+
params={
|
|
1783
|
+
"ids": ids,
|
|
1784
|
+
"per_label": per_label,
|
|
1785
|
+
},
|
|
1786
|
+
request_options=request_options,
|
|
1787
|
+
)
|
|
1788
|
+
try:
|
|
1789
|
+
if 200 <= _response.status_code < 300:
|
|
1790
|
+
return typing.cast(
|
|
1791
|
+
StatsUsersPredictionAgreementResponse,
|
|
1792
|
+
construct_type(
|
|
1793
|
+
type_=StatsUsersPredictionAgreementResponse, # type: ignore
|
|
1794
|
+
object_=_response.json(),
|
|
1795
|
+
),
|
|
1796
|
+
)
|
|
1797
|
+
_response_json = _response.json()
|
|
1798
|
+
except JSONDecodeError:
|
|
1799
|
+
raise ApiError(status_code=_response.status_code, body=_response.text)
|
|
1800
|
+
raise ApiError(status_code=_response.status_code, body=_response_json)
|
|
1801
|
+
|
|
1055
1802
|
async def user_prediction_agreement(
|
|
1056
1803
|
self,
|
|
1057
1804
|
id: int,
|
|
@@ -1061,6 +1808,12 @@ class AsyncStatsClient:
|
|
|
1061
1808
|
request_options: typing.Optional[RequestOptions] = None,
|
|
1062
1809
|
) -> StatsUserPredictionAgreementResponse:
|
|
1063
1810
|
"""
|
|
1811
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1812
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1813
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1814
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1815
|
+
</p>
|
|
1816
|
+
</Card>
|
|
1064
1817
|
Get prediction agreement statistics for a specific user within a project.
|
|
1065
1818
|
|
|
1066
1819
|
Parameters
|
|
@@ -1131,6 +1884,12 @@ class AsyncStatsClient:
|
|
|
1131
1884
|
request_options: typing.Optional[RequestOptions] = None,
|
|
1132
1885
|
) -> StatsUserReviewScoreResponse:
|
|
1133
1886
|
"""
|
|
1887
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1888
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1889
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1890
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1891
|
+
</p>
|
|
1892
|
+
</Card>
|
|
1134
1893
|
Get review score statistics for a specific user within a project.
|
|
1135
1894
|
|
|
1136
1895
|
Parameters
|
|
@@ -1201,6 +1960,12 @@ class AsyncStatsClient:
|
|
|
1201
1960
|
request_options: typing.Optional[RequestOptions] = None,
|
|
1202
1961
|
) -> StatsUserGroundTruthAgreementResponse:
|
|
1203
1962
|
"""
|
|
1963
|
+
<Card href="https://humansignal.com/goenterprise">
|
|
1964
|
+
<img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
|
|
1965
|
+
<p style="margin-top: 10px; font-size: 14px;">
|
|
1966
|
+
This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
|
|
1967
|
+
</p>
|
|
1968
|
+
</Card>
|
|
1204
1969
|
Get ground truth agreement statistics for a specific user within a project.
|
|
1205
1970
|
|
|
1206
1971
|
Parameters
|