label-studio-sdk 2.0.10__py3-none-any.whl → 2.0.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of label-studio-sdk might be problematic. Click here for more details.

Files changed (72) hide show
  1. label_studio_sdk/__init__.py +10 -0
  2. label_studio_sdk/activity_logs/client.py +12 -0
  3. label_studio_sdk/annotation_history/client.py +36 -0
  4. label_studio_sdk/annotation_reviews/client.py +60 -0
  5. label_studio_sdk/billing/client.py +12 -0
  6. label_studio_sdk/comments/client.py +72 -0
  7. label_studio_sdk/export_storage/azure_spi/client.py +84 -0
  8. label_studio_sdk/export_storage/databricks/client.py +84 -0
  9. label_studio_sdk/export_storage/gcswif/client.py +84 -0
  10. label_studio_sdk/export_storage/s3s/client.py +84 -0
  11. label_studio_sdk/import_storage/azure_spi/client.py +84 -0
  12. label_studio_sdk/import_storage/databricks/client.py +84 -0
  13. label_studio_sdk/import_storage/gcswif/client.py +84 -0
  14. label_studio_sdk/import_storage/s3s/client.py +84 -0
  15. label_studio_sdk/label_interface/control_tags.py +12 -7
  16. label_studio_sdk/ml/client.py +14 -2
  17. label_studio_sdk/model_providers/client.py +72 -0
  18. label_studio_sdk/organizations/client.py +30 -6
  19. label_studio_sdk/organizations/invites/client.py +24 -0
  20. label_studio_sdk/organizations/members/client.py +24 -0
  21. label_studio_sdk/organizations/permissions/client.py +60 -0
  22. label_studio_sdk/project_templates/client.py +76 -4
  23. label_studio_sdk/projects/__init__.py +14 -1
  24. label_studio_sdk/projects/assignments/client.py +60 -0
  25. label_studio_sdk/projects/client.py +20 -0
  26. label_studio_sdk/projects/members/bulk/client.py +24 -0
  27. label_studio_sdk/projects/members/client.py +149 -30
  28. label_studio_sdk/projects/members/paginated/client.py +12 -0
  29. label_studio_sdk/projects/metrics/client.py +24 -0
  30. label_studio_sdk/projects/pauses/client.py +60 -0
  31. label_studio_sdk/projects/roles/__init__.py +2 -0
  32. label_studio_sdk/projects/roles/client.py +555 -0
  33. label_studio_sdk/projects/stats/__init__.py +12 -0
  34. label_studio_sdk/projects/stats/client.py +766 -1
  35. label_studio_sdk/projects/stats/types/__init__.py +14 -0
  36. label_studio_sdk/projects/stats/types/stats_agreement_annotators_response.py +22 -0
  37. label_studio_sdk/projects/stats/types/stats_model_version_annotator_agreement_response.py +19 -0
  38. label_studio_sdk/projects/stats/types/stats_model_version_ground_truth_agreement_response.py +19 -0
  39. label_studio_sdk/projects/stats/types/stats_model_version_prediction_agreement_response.py +19 -0
  40. label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response.py +27 -0
  41. label_studio_sdk/projects/stats/types/stats_users_prediction_agreement_response_agreement_value.py +5 -0
  42. label_studio_sdk/prompts/client.py +96 -0
  43. label_studio_sdk/prompts/indicators/client.py +24 -0
  44. label_studio_sdk/prompts/runs/client.py +36 -0
  45. label_studio_sdk/prompts/versions/client.py +84 -0
  46. label_studio_sdk/sso/saml/client.py +24 -0
  47. label_studio_sdk/sso/scim/client.py +24 -0
  48. label_studio_sdk/types/__init__.py +10 -0
  49. label_studio_sdk/types/all_roles_project_list.py +5 -1
  50. label_studio_sdk/types/default_role.py +2 -2
  51. label_studio_sdk/types/lse_organization.py +1 -1
  52. label_studio_sdk/types/lse_project.py +5 -1
  53. label_studio_sdk/types/lse_project_create.py +5 -1
  54. label_studio_sdk/types/lse_project_response.py +5 -1
  55. label_studio_sdk/types/lse_project_update.py +5 -1
  56. label_studio_sdk/types/project.py +198 -0
  57. label_studio_sdk/types/project_member.py +19 -0
  58. label_studio_sdk/types/project_role.py +35 -0
  59. label_studio_sdk/types/project_sampling.py +7 -0
  60. label_studio_sdk/types/project_skip_queue.py +7 -0
  61. label_studio_sdk/types/project_template.py +1 -1
  62. label_studio_sdk/types/project_template_request.py +1 -1
  63. label_studio_sdk/users/client.py +24 -0
  64. label_studio_sdk/versions/client.py +12 -0
  65. label_studio_sdk/workspaces/__init__.py +2 -2
  66. label_studio_sdk/workspaces/client.py +4 -0
  67. label_studio_sdk/workspaces/projects/__init__.py +2 -0
  68. label_studio_sdk/workspaces/projects/client.py +352 -0
  69. {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/METADATA +1 -1
  70. {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/RECORD +72 -57
  71. {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/LICENSE +0 -0
  72. {label_studio_sdk-2.0.10.dist-info → label_studio_sdk-2.0.12.dist-info}/WHEEL +0 -0
@@ -3,16 +3,21 @@
3
3
  from ...core.client_wrapper import SyncClientWrapper
4
4
  import typing
5
5
  from ...core.request_options import RequestOptions
6
- from .types.stats_iaa_response import StatsIaaResponse
6
+ from .types.stats_model_version_annotator_agreement_response import StatsModelVersionAnnotatorAgreementResponse
7
7
  from ...core.jsonable_encoder import jsonable_encoder
8
8
  from ...core.unchecked_base_model import construct_type
9
9
  from json.decoder import JSONDecodeError
10
10
  from ...core.api_error import ApiError
11
+ from .types.stats_model_version_ground_truth_agreement_response import StatsModelVersionGroundTruthAgreementResponse
12
+ from .types.stats_model_version_prediction_agreement_response import StatsModelVersionPredictionAgreementResponse
13
+ from .types.stats_iaa_response import StatsIaaResponse
11
14
  from .types.stats_agreement_annotator_response import StatsAgreementAnnotatorResponse
15
+ from .types.stats_agreement_annotators_response import StatsAgreementAnnotatorsResponse
12
16
  from .types.stats_data_filters_response import StatsDataFiltersResponse
13
17
  from .types.stats_finished_tasks_response import StatsFinishedTasksResponse
14
18
  from .types.stats_lead_time_response import StatsLeadTimeResponse
15
19
  from .types.stats_total_agreement_response import StatsTotalAgreementResponse
20
+ from .types.stats_users_prediction_agreement_response import StatsUsersPredictionAgreementResponse
16
21
  from .types.stats_user_prediction_agreement_response import StatsUserPredictionAgreementResponse
17
22
  from .types.stats_user_review_score_response import StatsUserReviewScoreResponse
18
23
  from .types.stats_user_ground_truth_agreement_response import StatsUserGroundTruthAgreementResponse
@@ -23,6 +28,199 @@ class StatsClient:
23
28
  def __init__(self, *, client_wrapper: SyncClientWrapper):
24
29
  self._client_wrapper = client_wrapper
25
30
 
31
+ def model_version_annotator_agreement(
32
+ self, id: int, model_version: str, *, request_options: typing.Optional[RequestOptions] = None
33
+ ) -> StatsModelVersionAnnotatorAgreementResponse:
34
+ """
35
+ <Card href="https://humansignal.com/goenterprise">
36
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
37
+ <p style="margin-top: 10px; font-size: 14px;">
38
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
39
+ </p>
40
+ </Card>
41
+ Get agreement between a given model version and all annotators in the project for overlapping tasks.
42
+
43
+ Parameters
44
+ ----------
45
+ id : int
46
+
47
+ model_version : str
48
+
49
+ request_options : typing.Optional[RequestOptions]
50
+ Request-specific configuration.
51
+
52
+ Returns
53
+ -------
54
+ StatsModelVersionAnnotatorAgreementResponse
55
+ Model-version overall agreement vs annotators
56
+
57
+ Examples
58
+ --------
59
+ from label_studio_sdk import LabelStudio
60
+
61
+ client = LabelStudio(
62
+ api_key="YOUR_API_KEY",
63
+ )
64
+ client.projects.stats.model_version_annotator_agreement(
65
+ id=1,
66
+ model_version="model_version",
67
+ )
68
+ """
69
+ _response = self._client_wrapper.httpx_client.request(
70
+ f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement",
71
+ method="GET",
72
+ request_options=request_options,
73
+ )
74
+ try:
75
+ if 200 <= _response.status_code < 300:
76
+ return typing.cast(
77
+ StatsModelVersionAnnotatorAgreementResponse,
78
+ construct_type(
79
+ type_=StatsModelVersionAnnotatorAgreementResponse, # type: ignore
80
+ object_=_response.json(),
81
+ ),
82
+ )
83
+ _response_json = _response.json()
84
+ except JSONDecodeError:
85
+ raise ApiError(status_code=_response.status_code, body=_response.text)
86
+ raise ApiError(status_code=_response.status_code, body=_response_json)
87
+
88
+ def model_version_ground_truth_agreement(
89
+ self,
90
+ id: int,
91
+ model_version: str,
92
+ *,
93
+ per_label: typing.Optional[bool] = None,
94
+ request_options: typing.Optional[RequestOptions] = None,
95
+ ) -> StatsModelVersionGroundTruthAgreementResponse:
96
+ """
97
+ <Card href="https://humansignal.com/goenterprise">
98
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
99
+ <p style="margin-top: 10px; font-size: 14px;">
100
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
101
+ </p>
102
+ </Card>
103
+ Get agreement between a given model version and ground truth annotations in the project for overlapping tasks.
104
+
105
+ Parameters
106
+ ----------
107
+ id : int
108
+
109
+ model_version : str
110
+
111
+ per_label : typing.Optional[bool]
112
+ Calculate agreement per label
113
+
114
+ request_options : typing.Optional[RequestOptions]
115
+ Request-specific configuration.
116
+
117
+ Returns
118
+ -------
119
+ StatsModelVersionGroundTruthAgreementResponse
120
+ Model-version ground truth agreement
121
+
122
+ Examples
123
+ --------
124
+ from label_studio_sdk import LabelStudio
125
+
126
+ client = LabelStudio(
127
+ api_key="YOUR_API_KEY",
128
+ )
129
+ client.projects.stats.model_version_ground_truth_agreement(
130
+ id=1,
131
+ model_version="model_version",
132
+ )
133
+ """
134
+ _response = self._client_wrapper.httpx_client.request(
135
+ f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement-groundtruth",
136
+ method="GET",
137
+ params={
138
+ "per_label": per_label,
139
+ },
140
+ request_options=request_options,
141
+ )
142
+ try:
143
+ if 200 <= _response.status_code < 300:
144
+ return typing.cast(
145
+ StatsModelVersionGroundTruthAgreementResponse,
146
+ construct_type(
147
+ type_=StatsModelVersionGroundTruthAgreementResponse, # type: ignore
148
+ object_=_response.json(),
149
+ ),
150
+ )
151
+ _response_json = _response.json()
152
+ except JSONDecodeError:
153
+ raise ApiError(status_code=_response.status_code, body=_response.text)
154
+ raise ApiError(status_code=_response.status_code, body=_response_json)
155
+
156
+ def model_version_prediction_agreement(
157
+ self,
158
+ id: int,
159
+ model_version: str,
160
+ *,
161
+ per_label: typing.Optional[bool] = None,
162
+ request_options: typing.Optional[RequestOptions] = None,
163
+ ) -> StatsModelVersionPredictionAgreementResponse:
164
+ """
165
+ <Card href="https://humansignal.com/goenterprise">
166
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
167
+ <p style="margin-top: 10px; font-size: 14px;">
168
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
169
+ </p>
170
+ </Card>
171
+ Get agreement between a given model version and all other model versions in the project for overlapping tasks.
172
+
173
+ Parameters
174
+ ----------
175
+ id : int
176
+
177
+ model_version : str
178
+
179
+ per_label : typing.Optional[bool]
180
+ Calculate agreement per label
181
+
182
+ request_options : typing.Optional[RequestOptions]
183
+ Request-specific configuration.
184
+
185
+ Returns
186
+ -------
187
+ StatsModelVersionPredictionAgreementResponse
188
+ Model-version prediction agreement
189
+
190
+ Examples
191
+ --------
192
+ from label_studio_sdk import LabelStudio
193
+
194
+ client = LabelStudio(
195
+ api_key="YOUR_API_KEY",
196
+ )
197
+ client.projects.stats.model_version_prediction_agreement(
198
+ id=1,
199
+ model_version="model_version",
200
+ )
201
+ """
202
+ _response = self._client_wrapper.httpx_client.request(
203
+ f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/prediction",
204
+ method="GET",
205
+ params={
206
+ "per_label": per_label,
207
+ },
208
+ request_options=request_options,
209
+ )
210
+ try:
211
+ if 200 <= _response.status_code < 300:
212
+ return typing.cast(
213
+ StatsModelVersionPredictionAgreementResponse,
214
+ construct_type(
215
+ type_=StatsModelVersionPredictionAgreementResponse, # type: ignore
216
+ object_=_response.json(),
217
+ ),
218
+ )
219
+ _response_json = _response.json()
220
+ except JSONDecodeError:
221
+ raise ApiError(status_code=_response.status_code, body=_response.text)
222
+ raise ApiError(status_code=_response.status_code, body=_response_json)
223
+
26
224
  def iaa(
27
225
  self,
28
226
  id: int,
@@ -34,6 +232,12 @@ class StatsClient:
34
232
  request_options: typing.Optional[RequestOptions] = None,
35
233
  ) -> StatsIaaResponse:
36
234
  """
235
+ <Card href="https://humansignal.com/goenterprise">
236
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
237
+ <p style="margin-top: 10px; font-size: 14px;">
238
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
239
+ </p>
240
+ </Card>
37
241
  Get Inter-Annotator Agreement (IAA) matrix for a project, showing agreement between all annotators.
38
242
 
39
243
  Parameters
@@ -100,6 +304,12 @@ class StatsClient:
100
304
  self, id: int, user_id: int, *, request_options: typing.Optional[RequestOptions] = None
101
305
  ) -> StatsAgreementAnnotatorResponse:
102
306
  """
307
+ <Card href="https://humansignal.com/goenterprise">
308
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
309
+ <p style="margin-top: 10px; font-size: 14px;">
310
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
311
+ </p>
312
+ </Card>
103
313
  Get agreement statistics for a specific annotator within a project.
104
314
 
105
315
  Parameters
@@ -147,6 +357,67 @@ class StatsClient:
147
357
  raise ApiError(status_code=_response.status_code, body=_response.text)
148
358
  raise ApiError(status_code=_response.status_code, body=_response_json)
149
359
 
360
+ def agreement_annotators(
361
+ self, id: int, *, ids: str, request_options: typing.Optional[RequestOptions] = None
362
+ ) -> StatsAgreementAnnotatorsResponse:
363
+ """
364
+ <Card href="https://humansignal.com/goenterprise">
365
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
366
+ <p style="margin-top: 10px; font-size: 14px;">
367
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
368
+ </p>
369
+ </Card>
370
+ Get agreement statistics for multiple annotators within a project.
371
+
372
+ Parameters
373
+ ----------
374
+ id : int
375
+
376
+ ids : str
377
+ Comma separated list of annotator user IDs to get agreement scores for
378
+
379
+ request_options : typing.Optional[RequestOptions]
380
+ Request-specific configuration.
381
+
382
+ Returns
383
+ -------
384
+ StatsAgreementAnnotatorsResponse
385
+ Multiple annotator agreement statistics
386
+
387
+ Examples
388
+ --------
389
+ from label_studio_sdk import LabelStudio
390
+
391
+ client = LabelStudio(
392
+ api_key="YOUR_API_KEY",
393
+ )
394
+ client.projects.stats.agreement_annotators(
395
+ id=1,
396
+ ids="ids",
397
+ )
398
+ """
399
+ _response = self._client_wrapper.httpx_client.request(
400
+ f"api/projects/{jsonable_encoder(id)}/stats/agreement_annotators",
401
+ method="GET",
402
+ params={
403
+ "ids": ids,
404
+ },
405
+ request_options=request_options,
406
+ )
407
+ try:
408
+ if 200 <= _response.status_code < 300:
409
+ return typing.cast(
410
+ StatsAgreementAnnotatorsResponse,
411
+ construct_type(
412
+ type_=StatsAgreementAnnotatorsResponse, # type: ignore
413
+ object_=_response.json(),
414
+ ),
415
+ )
416
+ _response_json = _response.json()
417
+ except JSONDecodeError:
418
+ raise ApiError(status_code=_response.status_code, body=_response.text)
419
+ raise ApiError(status_code=_response.status_code, body=_response_json)
420
+
150
421
  def data_filters(
151
422
  self, id: int, *, request_options: typing.Optional[RequestOptions] = None
152
423
  ) -> StatsDataFiltersResponse:
@@ -303,6 +574,12 @@ class StatsClient:
303
574
  request_options: typing.Optional[RequestOptions] = None,
304
575
  ) -> StatsTotalAgreementResponse:
305
576
  """
577
+ <Card href="https://humansignal.com/goenterprise">
578
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
579
+ <p style="margin-top: 10px; font-size: 14px;">
580
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
581
+ </p>
582
+ </Card>
306
583
  Overall or per-label total agreement across the project.
307
584
 
308
585
  NOTE: due to an open issue in Fern, SDK clients will raise ApiError upon handling a 204 response. As a workaround, wrap call to this function in a try-except block.
@@ -413,6 +690,76 @@ class StatsClient:
413
690
  raise ApiError(status_code=_response.status_code, body=_response.text)
414
691
  raise ApiError(status_code=_response.status_code, body=_response_json)
415
692
 
693
+ def users_prediction_agreement(
694
+ self,
695
+ id: int,
696
+ *,
697
+ ids: str,
698
+ per_label: typing.Optional[bool] = None,
699
+ request_options: typing.Optional[RequestOptions] = None,
700
+ ) -> StatsUsersPredictionAgreementResponse:
701
+ """
702
+ <Card href="https://humansignal.com/goenterprise">
703
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
704
+ <p style="margin-top: 10px; font-size: 14px;">
705
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
706
+ </p>
707
+ </Card>
708
+ Get prediction agreement statistics for multiple annotators within a project.
709
+
710
+ Parameters
711
+ ----------
712
+ id : int
713
+
714
+ ids : str
715
+ Comma separated list of annotator user IDs to get agreement scores for
716
+
717
+ per_label : typing.Optional[bool]
718
+ Per label
719
+
720
+ request_options : typing.Optional[RequestOptions]
721
+ Request-specific configuration.
722
+
723
+ Returns
724
+ -------
725
+ StatsUsersPredictionAgreementResponse
726
+ Prediction agreement statistics for multiple annotators
727
+
728
+ Examples
729
+ --------
730
+ from label_studio_sdk import LabelStudio
731
+
732
+ client = LabelStudio(
733
+ api_key="YOUR_API_KEY",
734
+ )
735
+ client.projects.stats.users_prediction_agreement(
736
+ id=1,
737
+ ids="ids",
738
+ )
739
+ """
740
+ _response = self._client_wrapper.httpx_client.request(
741
+ f"api/projects/{jsonable_encoder(id)}/user-stats/prediction",
742
+ method="GET",
743
+ params={
744
+ "ids": ids,
745
+ "per_label": per_label,
746
+ },
747
+ request_options=request_options,
748
+ )
749
+ try:
750
+ if 200 <= _response.status_code < 300:
751
+ return typing.cast(
752
+ StatsUsersPredictionAgreementResponse,
753
+ construct_type(
754
+ type_=StatsUsersPredictionAgreementResponse, # type: ignore
755
+ object_=_response.json(),
756
+ ),
757
+ )
758
+ _response_json = _response.json()
759
+ except JSONDecodeError:
760
+ raise ApiError(status_code=_response.status_code, body=_response.text)
761
+ raise ApiError(status_code=_response.status_code, body=_response_json)
762
+
416
763
  def user_prediction_agreement(
417
764
  self,
418
765
  id: int,
@@ -422,6 +769,12 @@ class StatsClient:
422
769
  request_options: typing.Optional[RequestOptions] = None,
423
770
  ) -> StatsUserPredictionAgreementResponse:
424
771
  """
772
+ <Card href="https://humansignal.com/goenterprise">
773
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
774
+ <p style="margin-top: 10px; font-size: 14px;">
775
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
776
+ </p>
777
+ </Card>
425
778
  Get prediction agreement statistics for a specific user within a project.
426
779
 
427
780
  Parameters
@@ -484,6 +837,12 @@ class StatsClient:
484
837
  request_options: typing.Optional[RequestOptions] = None,
485
838
  ) -> StatsUserReviewScoreResponse:
486
839
  """
840
+ <Card href="https://humansignal.com/goenterprise">
841
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
842
+ <p style="margin-top: 10px; font-size: 14px;">
843
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
844
+ </p>
845
+ </Card>
487
846
  Get review score statistics for a specific user within a project.
488
847
 
489
848
  Parameters
@@ -546,6 +905,12 @@ class StatsClient:
546
905
  request_options: typing.Optional[RequestOptions] = None,
547
906
  ) -> StatsUserGroundTruthAgreementResponse:
548
907
  """
908
+ <Card href="https://humansignal.com/goenterprise">
909
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
910
+ <p style="margin-top: 10px; font-size: 14px;">
911
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
912
+ </p>
913
+ </Card>
549
914
  Get ground truth agreement statistics for a specific user within a project.
550
915
 
551
916
  Parameters
@@ -604,6 +969,223 @@ class AsyncStatsClient:
604
969
  def __init__(self, *, client_wrapper: AsyncClientWrapper):
605
970
  self._client_wrapper = client_wrapper
606
971
 
972
+ async def model_version_annotator_agreement(
973
+ self, id: int, model_version: str, *, request_options: typing.Optional[RequestOptions] = None
974
+ ) -> StatsModelVersionAnnotatorAgreementResponse:
975
+ """
976
+ <Card href="https://humansignal.com/goenterprise">
977
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
978
+ <p style="margin-top: 10px; font-size: 14px;">
979
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
980
+ </p>
981
+ </Card>
982
+ Get agreement between a given model version and all annotators in the project for overlapping tasks.
983
+
984
+ Parameters
985
+ ----------
986
+ id : int
987
+
988
+ model_version : str
989
+
990
+ request_options : typing.Optional[RequestOptions]
991
+ Request-specific configuration.
992
+
993
+ Returns
994
+ -------
995
+ StatsModelVersionAnnotatorAgreementResponse
996
+ Model-version overall agreement vs annotators
997
+
998
+ Examples
999
+ --------
1000
+ import asyncio
1001
+
1002
+ from label_studio_sdk import AsyncLabelStudio
1003
+
1004
+ client = AsyncLabelStudio(
1005
+ api_key="YOUR_API_KEY",
1006
+ )
1007
+
1008
+
1009
+ async def main() -> None:
1010
+ await client.projects.stats.model_version_annotator_agreement(
1011
+ id=1,
1012
+ model_version="model_version",
1013
+ )
1014
+
1015
+
1016
+ asyncio.run(main())
1017
+ """
1018
+ _response = await self._client_wrapper.httpx_client.request(
1019
+ f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement",
1020
+ method="GET",
1021
+ request_options=request_options,
1022
+ )
1023
+ try:
1024
+ if 200 <= _response.status_code < 300:
1025
+ return typing.cast(
1026
+ StatsModelVersionAnnotatorAgreementResponse,
1027
+ construct_type(
1028
+ type_=StatsModelVersionAnnotatorAgreementResponse, # type: ignore
1029
+ object_=_response.json(),
1030
+ ),
1031
+ )
1032
+ _response_json = _response.json()
1033
+ except JSONDecodeError:
1034
+ raise ApiError(status_code=_response.status_code, body=_response.text)
1035
+ raise ApiError(status_code=_response.status_code, body=_response_json)
1036
+
1037
+ async def model_version_ground_truth_agreement(
1038
+ self,
1039
+ id: int,
1040
+ model_version: str,
1041
+ *,
1042
+ per_label: typing.Optional[bool] = None,
1043
+ request_options: typing.Optional[RequestOptions] = None,
1044
+ ) -> StatsModelVersionGroundTruthAgreementResponse:
1045
+ """
1046
+ <Card href="https://humansignal.com/goenterprise">
1047
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1048
+ <p style="margin-top: 10px; font-size: 14px;">
1049
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1050
+ </p>
1051
+ </Card>
1052
+ Get agreement between a given model version and ground truth annotations in the project for overlapping tasks.
1053
+
1054
+ Parameters
1055
+ ----------
1056
+ id : int
1057
+
1058
+ model_version : str
1059
+
1060
+ per_label : typing.Optional[bool]
1061
+ Calculate agreement per label
1062
+
1063
+ request_options : typing.Optional[RequestOptions]
1064
+ Request-specific configuration.
1065
+
1066
+ Returns
1067
+ -------
1068
+ StatsModelVersionGroundTruthAgreementResponse
1069
+ Model-version ground truth agreement
1070
+
1071
+ Examples
1072
+ --------
1073
+ import asyncio
1074
+
1075
+ from label_studio_sdk import AsyncLabelStudio
1076
+
1077
+ client = AsyncLabelStudio(
1078
+ api_key="YOUR_API_KEY",
1079
+ )
1080
+
1081
+
1082
+ async def main() -> None:
1083
+ await client.projects.stats.model_version_ground_truth_agreement(
1084
+ id=1,
1085
+ model_version="model_version",
1086
+ )
1087
+
1088
+
1089
+ asyncio.run(main())
1090
+ """
1091
+ _response = await self._client_wrapper.httpx_client.request(
1092
+ f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/agreement-groundtruth",
1093
+ method="GET",
1094
+ params={
1095
+ "per_label": per_label,
1096
+ },
1097
+ request_options=request_options,
1098
+ )
1099
+ try:
1100
+ if 200 <= _response.status_code < 300:
1101
+ return typing.cast(
1102
+ StatsModelVersionGroundTruthAgreementResponse,
1103
+ construct_type(
1104
+ type_=StatsModelVersionGroundTruthAgreementResponse, # type: ignore
1105
+ object_=_response.json(),
1106
+ ),
1107
+ )
1108
+ _response_json = _response.json()
1109
+ except JSONDecodeError:
1110
+ raise ApiError(status_code=_response.status_code, body=_response.text)
1111
+ raise ApiError(status_code=_response.status_code, body=_response_json)
1112
+
1113
+ async def model_version_prediction_agreement(
1114
+ self,
1115
+ id: int,
1116
+ model_version: str,
1117
+ *,
1118
+ per_label: typing.Optional[bool] = None,
1119
+ request_options: typing.Optional[RequestOptions] = None,
1120
+ ) -> StatsModelVersionPredictionAgreementResponse:
1121
+ """
1122
+ <Card href="https://humansignal.com/goenterprise">
1123
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1124
+ <p style="margin-top: 10px; font-size: 14px;">
1125
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1126
+ </p>
1127
+ </Card>
1128
+ Get agreement between a given model version and all other model versions in the project for overlapping tasks.
1129
+
1130
+ Parameters
1131
+ ----------
1132
+ id : int
1133
+
1134
+ model_version : str
1135
+
1136
+ per_label : typing.Optional[bool]
1137
+ Calculate agreement per label
1138
+
1139
+ request_options : typing.Optional[RequestOptions]
1140
+ Request-specific configuration.
1141
+
1142
+ Returns
1143
+ -------
1144
+ StatsModelVersionPredictionAgreementResponse
1145
+ Model-version prediction agreement
1146
+
1147
+ Examples
1148
+ --------
1149
+ import asyncio
1150
+
1151
+ from label_studio_sdk import AsyncLabelStudio
1152
+
1153
+ client = AsyncLabelStudio(
1154
+ api_key="YOUR_API_KEY",
1155
+ )
1156
+
1157
+
1158
+ async def main() -> None:
1159
+ await client.projects.stats.model_version_prediction_agreement(
1160
+ id=1,
1161
+ model_version="model_version",
1162
+ )
1163
+
1164
+
1165
+ asyncio.run(main())
1166
+ """
1167
+ _response = await self._client_wrapper.httpx_client.request(
1168
+ f"api/projects/{jsonable_encoder(id)}/model-stats/{jsonable_encoder(model_version)}/prediction",
1169
+ method="GET",
1170
+ params={
1171
+ "per_label": per_label,
1172
+ },
1173
+ request_options=request_options,
1174
+ )
1175
+ try:
1176
+ if 200 <= _response.status_code < 300:
1177
+ return typing.cast(
1178
+ StatsModelVersionPredictionAgreementResponse,
1179
+ construct_type(
1180
+ type_=StatsModelVersionPredictionAgreementResponse, # type: ignore
1181
+ object_=_response.json(),
1182
+ ),
1183
+ )
1184
+ _response_json = _response.json()
1185
+ except JSONDecodeError:
1186
+ raise ApiError(status_code=_response.status_code, body=_response.text)
1187
+ raise ApiError(status_code=_response.status_code, body=_response_json)
1188
+
607
1189
  async def iaa(
608
1190
  self,
609
1191
  id: int,
@@ -615,6 +1197,12 @@ class AsyncStatsClient:
615
1197
  request_options: typing.Optional[RequestOptions] = None,
616
1198
  ) -> StatsIaaResponse:
617
1199
  """
1200
+ <Card href="https://humansignal.com/goenterprise">
1201
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1202
+ <p style="margin-top: 10px; font-size: 14px;">
1203
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1204
+ </p>
1205
+ </Card>
618
1206
  Get Inter-Annotator Agreement (IAA) matrix for a project, showing agreement between all annotators.
619
1207
 
620
1208
  Parameters
@@ -689,6 +1277,12 @@ class AsyncStatsClient:
689
1277
  self, id: int, user_id: int, *, request_options: typing.Optional[RequestOptions] = None
690
1278
  ) -> StatsAgreementAnnotatorResponse:
691
1279
  """
1280
+ <Card href="https://humansignal.com/goenterprise">
1281
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1282
+ <p style="margin-top: 10px; font-size: 14px;">
1283
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1284
+ </p>
1285
+ </Card>
692
1286
  Get agreement statistics for a specific annotator within a project.
693
1287
 
694
1288
  Parameters
@@ -744,6 +1338,75 @@ class AsyncStatsClient:
744
1338
  raise ApiError(status_code=_response.status_code, body=_response.text)
745
1339
  raise ApiError(status_code=_response.status_code, body=_response_json)
746
1340
 
1341
+ async def agreement_annotators(
1342
+ self, id: int, *, ids: str, request_options: typing.Optional[RequestOptions] = None
1343
+ ) -> StatsAgreementAnnotatorsResponse:
1344
+ """
1345
+ <Card href="https://humansignal.com/goenterprise">
1346
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1347
+ <p style="margin-top: 10px; font-size: 14px;">
1348
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1349
+ </p>
1350
+ </Card>
1351
+ Get agreement statistics for multiple annotators within a project.
1352
+
1353
+ Parameters
1354
+ ----------
1355
+ id : int
1356
+
1357
+ ids : str
1358
+ Comma separated list of annotator user IDs to get agreement scores for
1359
+
1360
+ request_options : typing.Optional[RequestOptions]
1361
+ Request-specific configuration.
1362
+
1363
+ Returns
1364
+ -------
1365
+ StatsAgreementAnnotatorsResponse
1366
+ Multiple annotator agreement statistics
1367
+
1368
+ Examples
1369
+ --------
1370
+ import asyncio
1371
+
1372
+ from label_studio_sdk import AsyncLabelStudio
1373
+
1374
+ client = AsyncLabelStudio(
1375
+ api_key="YOUR_API_KEY",
1376
+ )
1377
+
1378
+
1379
+ async def main() -> None:
1380
+ await client.projects.stats.agreement_annotators(
1381
+ id=1,
1382
+ ids="ids",
1383
+ )
1384
+
1385
+
1386
+ asyncio.run(main())
1387
+ """
1388
+ _response = await self._client_wrapper.httpx_client.request(
1389
+ f"api/projects/{jsonable_encoder(id)}/stats/agreement_annotators",
1390
+ method="GET",
1391
+ params={
1392
+ "ids": ids,
1393
+ },
1394
+ request_options=request_options,
1395
+ )
1396
+ try:
1397
+ if 200 <= _response.status_code < 300:
1398
+ return typing.cast(
1399
+ StatsAgreementAnnotatorsResponse,
1400
+ construct_type(
1401
+ type_=StatsAgreementAnnotatorsResponse, # type: ignore
1402
+ object_=_response.json(),
1403
+ ),
1404
+ )
1405
+ _response_json = _response.json()
1406
+ except JSONDecodeError:
1407
+ raise ApiError(status_code=_response.status_code, body=_response.text)
1408
+ raise ApiError(status_code=_response.status_code, body=_response_json)
1409
+
747
1410
  async def data_filters(
748
1411
  self, id: int, *, request_options: typing.Optional[RequestOptions] = None
749
1412
  ) -> StatsDataFiltersResponse:
@@ -926,6 +1589,12 @@ class AsyncStatsClient:
926
1589
  request_options: typing.Optional[RequestOptions] = None,
927
1590
  ) -> StatsTotalAgreementResponse:
928
1591
  """
1592
+ <Card href="https://humansignal.com/goenterprise">
1593
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1594
+ <p style="margin-top: 10px; font-size: 14px;">
1595
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1596
+ </p>
1597
+ </Card>
929
1598
  Overall or per-label total agreement across the project.
930
1599
 
931
1600
  NOTE: due to an open issue in Fern, SDK clients will raise ApiError upon handling a 204 response. As a workaround, wrap call to this function in a try-except block.
@@ -1052,6 +1721,84 @@ class AsyncStatsClient:
1052
1721
  raise ApiError(status_code=_response.status_code, body=_response.text)
1053
1722
  raise ApiError(status_code=_response.status_code, body=_response_json)
1054
1723
 
1724
+ async def users_prediction_agreement(
1725
+ self,
1726
+ id: int,
1727
+ *,
1728
+ ids: str,
1729
+ per_label: typing.Optional[bool] = None,
1730
+ request_options: typing.Optional[RequestOptions] = None,
1731
+ ) -> StatsUsersPredictionAgreementResponse:
1732
+ """
1733
+ <Card href="https://humansignal.com/goenterprise">
1734
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1735
+ <p style="margin-top: 10px; font-size: 14px;">
1736
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1737
+ </p>
1738
+ </Card>
1739
+ Get prediction agreement statistics for multiple annotators within a project.
1740
+
1741
+ Parameters
1742
+ ----------
1743
+ id : int
1744
+
1745
+ ids : str
1746
+ Comma separated list of annotator user IDs to get agreement scores for
1747
+
1748
+ per_label : typing.Optional[bool]
1749
+ Per label
1750
+
1751
+ request_options : typing.Optional[RequestOptions]
1752
+ Request-specific configuration.
1753
+
1754
+ Returns
1755
+ -------
1756
+ StatsUsersPredictionAgreementResponse
1757
+ Prediction agreement statistics for multiple annotators
1758
+
1759
+ Examples
1760
+ --------
1761
+ import asyncio
1762
+
1763
+ from label_studio_sdk import AsyncLabelStudio
1764
+
1765
+ client = AsyncLabelStudio(
1766
+ api_key="YOUR_API_KEY",
1767
+ )
1768
+
1769
+
1770
+ async def main() -> None:
1771
+ await client.projects.stats.users_prediction_agreement(
1772
+ id=1,
1773
+ ids="ids",
1774
+ )
1775
+
1776
+
1777
+ asyncio.run(main())
1778
+ """
1779
+ _response = await self._client_wrapper.httpx_client.request(
1780
+ f"api/projects/{jsonable_encoder(id)}/user-stats/prediction",
1781
+ method="GET",
1782
+ params={
1783
+ "ids": ids,
1784
+ "per_label": per_label,
1785
+ },
1786
+ request_options=request_options,
1787
+ )
1788
+ try:
1789
+ if 200 <= _response.status_code < 300:
1790
+ return typing.cast(
1791
+ StatsUsersPredictionAgreementResponse,
1792
+ construct_type(
1793
+ type_=StatsUsersPredictionAgreementResponse, # type: ignore
1794
+ object_=_response.json(),
1795
+ ),
1796
+ )
1797
+ _response_json = _response.json()
1798
+ except JSONDecodeError:
1799
+ raise ApiError(status_code=_response.status_code, body=_response.text)
1800
+ raise ApiError(status_code=_response.status_code, body=_response_json)
1801
+
1055
1802
  async def user_prediction_agreement(
1056
1803
  self,
1057
1804
  id: int,
@@ -1061,6 +1808,12 @@ class AsyncStatsClient:
1061
1808
  request_options: typing.Optional[RequestOptions] = None,
1062
1809
  ) -> StatsUserPredictionAgreementResponse:
1063
1810
  """
1811
+ <Card href="https://humansignal.com/goenterprise">
1812
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1813
+ <p style="margin-top: 10px; font-size: 14px;">
1814
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1815
+ </p>
1816
+ </Card>
1064
1817
  Get prediction agreement statistics for a specific user within a project.
1065
1818
 
1066
1819
  Parameters
@@ -1131,6 +1884,12 @@ class AsyncStatsClient:
1131
1884
  request_options: typing.Optional[RequestOptions] = None,
1132
1885
  ) -> StatsUserReviewScoreResponse:
1133
1886
  """
1887
+ <Card href="https://humansignal.com/goenterprise">
1888
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1889
+ <p style="margin-top: 10px; font-size: 14px;">
1890
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1891
+ </p>
1892
+ </Card>
1134
1893
  Get review score statistics for a specific user within a project.
1135
1894
 
1136
1895
  Parameters
@@ -1201,6 +1960,12 @@ class AsyncStatsClient:
1201
1960
  request_options: typing.Optional[RequestOptions] = None,
1202
1961
  ) -> StatsUserGroundTruthAgreementResponse:
1203
1962
  """
1963
+ <Card href="https://humansignal.com/goenterprise">
1964
+ <img style="pointer-events: none; margin-left: 0px; margin-right: 0px;" src="https://docs.humansignal.com/images/badge.svg" alt="Label Studio Enterprise badge"/>
1965
+ <p style="margin-top: 10px; font-size: 14px;">
1966
+ This endpoint is not available in Label Studio Community Edition. [Learn more about Label Studio Enterprise](https://humansignal.com/goenterprise)
1967
+ </p>
1968
+ </Card>
1204
1969
  Get ground truth agreement statistics for a specific user within a project.
1205
1970
 
1206
1971
  Parameters