proofstate 4.15.6rc2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- proofstate-4.15.6rc2/LICENSE +21 -0
- proofstate-4.15.6rc2/PKG-INFO +96 -0
- proofstate-4.15.6rc2/README.md +73 -0
- proofstate-4.15.6rc2/proofstate/__init__.py +141 -0
- proofstate-4.15.6rc2/proofstate/_client/attributes.py +219 -0
- proofstate-4.15.6rc2/proofstate/_client/client.py +4249 -0
- proofstate-4.15.6rc2/proofstate/_client/constants.py +65 -0
- proofstate-4.15.6rc2/proofstate/_client/datasets.py +302 -0
- proofstate-4.15.6rc2/proofstate/_client/environment_variables.py +174 -0
- proofstate-4.15.6rc2/proofstate/_client/get_client.py +154 -0
- proofstate-4.15.6rc2/proofstate/_client/observe.py +800 -0
- proofstate-4.15.6rc2/proofstate/_client/propagation.py +775 -0
- proofstate-4.15.6rc2/proofstate/_client/resource_manager.py +688 -0
- proofstate-4.15.6rc2/proofstate/_client/span.py +1539 -0
- proofstate-4.15.6rc2/proofstate/_client/span_exporter.py +685 -0
- proofstate-4.15.6rc2/proofstate/_client/span_filter.py +121 -0
- proofstate-4.15.6rc2/proofstate/_client/span_processor.py +334 -0
- proofstate-4.15.6rc2/proofstate/_client/utils.py +158 -0
- proofstate-4.15.6rc2/proofstate/_task_manager/media_manager.py +536 -0
- proofstate-4.15.6rc2/proofstate/_task_manager/media_upload_consumer.py +46 -0
- proofstate-4.15.6rc2/proofstate/_task_manager/media_upload_queue.py +14 -0
- proofstate-4.15.6rc2/proofstate/_task_manager/score_ingestion_consumer.py +198 -0
- proofstate-4.15.6rc2/proofstate/_utils/__init__.py +19 -0
- proofstate-4.15.6rc2/proofstate/_utils/environment.py +34 -0
- proofstate-4.15.6rc2/proofstate/_utils/error_logging.py +47 -0
- proofstate-4.15.6rc2/proofstate/_utils/json_path.py +74 -0
- proofstate-4.15.6rc2/proofstate/_utils/parse_error.py +99 -0
- proofstate-4.15.6rc2/proofstate/_utils/prompt_cache.py +236 -0
- proofstate-4.15.6rc2/proofstate/_utils/request.py +137 -0
- proofstate-4.15.6rc2/proofstate/_utils/serializer.py +234 -0
- proofstate-4.15.6rc2/proofstate/_version.py +13 -0
- proofstate-4.15.6rc2/proofstate/api/.fern/metadata.json +14 -0
- proofstate-4.15.6rc2/proofstate/api/__init__.py +1353 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/__init__.py +82 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/client.py +1111 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/raw_client.py +2288 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/__init__.py +84 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/annotation_queue.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/annotation_queue_assignment_request.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/annotation_queue_item.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/annotation_queue_object_type.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/annotation_queue_status.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/create_annotation_queue_assignment_response.py +18 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/create_annotation_queue_item_request.py +25 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/create_annotation_queue_request.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/delete_annotation_queue_assignment_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/delete_annotation_queue_item_response.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/paginated_annotation_queue_items.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/paginated_annotation_queues.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/annotation_queues/types/update_annotation_queue_item_request.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/__init__.py +82 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/client.py +594 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/raw_client.py +1020 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/__init__.py +88 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_export_field_group.py +62 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_export_frequency.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_export_mode.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_export_source.py +35 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_integration_deletion_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_integration_file_type.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_integration_file_type_response.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_integration_response.py +80 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_integration_status_response.py +50 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_integration_type.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_integrations_response.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/blob_storage_sync_status.py +52 -0
- proofstate-4.15.6rc2/proofstate/api/blob_storage_integrations/types/create_blob_storage_integration_request.py +117 -0
- proofstate-4.15.6rc2/proofstate/api/client.py +804 -0
- proofstate-4.15.6rc2/proofstate/api/comments/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/comments/client.py +407 -0
- proofstate-4.15.6rc2/proofstate/api/comments/raw_client.py +750 -0
- proofstate-4.15.6rc2/proofstate/api/comments/types/__init__.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/comments/types/create_comment_request.py +47 -0
- proofstate-4.15.6rc2/proofstate/api/comments/types/create_comment_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/comments/types/get_comments_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/commons/__init__.py +252 -0
- proofstate-4.15.6rc2/proofstate/api/commons/errors/__init__.py +56 -0
- proofstate-4.15.6rc2/proofstate/api/commons/errors/access_denied_error.py +12 -0
- proofstate-4.15.6rc2/proofstate/api/commons/errors/error.py +12 -0
- proofstate-4.15.6rc2/proofstate/api/commons/errors/method_not_allowed_error.py +12 -0
- proofstate-4.15.6rc2/proofstate/api/commons/errors/not_found_error.py +12 -0
- proofstate-4.15.6rc2/proofstate/api/commons/errors/unauthorized_error.py +12 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/__init__.py +237 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/base_score.py +90 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/base_score_v1.py +70 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/boolean_score.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/boolean_score_v1.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/categorical_score.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/categorical_score_v1.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/comment.py +36 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/comment_object_type.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/config_category.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/correction_score.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/create_score_value.py +5 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset.py +49 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_item.py +65 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_item_media_reference.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_item_media_reference_field.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_item_media_reference_media.py +47 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_run.py +63 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_run_item.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_run_with_items.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/dataset_status.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/deprecation.py +42 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/map_value.py +11 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/model.py +125 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/model_price.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/model_usage_unit.py +42 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/numeric_score.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/numeric_score_v1.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/observation.py +142 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/observation_level.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/observation_v2.py +287 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/observations_view.py +89 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/observations_view_single.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier.py +91 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_attribute_condition.py +37 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_attribute_source.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_condition.py +10 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_condition_input.py +10 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_input.py +76 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_operator.py +42 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_usage_condition.py +39 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/pricing_tier_usage_condition_input.py +39 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/score.py +259 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/score_config.py +66 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/score_config_data_type.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/score_data_type.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/score_source.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/score_v1.py +168 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/session.py +25 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/session_with_traces.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/text_score.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/text_score_v1.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/trace.py +84 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/trace_with_details.py +43 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/trace_with_full_details.py +50 -0
- proofstate-4.15.6rc2/proofstate/api/commons/types/usage.py +59 -0
- proofstate-4.15.6rc2/proofstate/api/core/__init__.py +111 -0
- proofstate-4.15.6rc2/proofstate/api/core/api_error.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/core/client_wrapper.py +142 -0
- proofstate-4.15.6rc2/proofstate/api/core/datetime_utils.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/core/enum.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/core/file.py +70 -0
- proofstate-4.15.6rc2/proofstate/api/core/force_multipart.py +18 -0
- proofstate-4.15.6rc2/proofstate/api/core/http_client.py +711 -0
- proofstate-4.15.6rc2/proofstate/api/core/http_response.py +55 -0
- proofstate-4.15.6rc2/proofstate/api/core/http_sse/__init__.py +48 -0
- proofstate-4.15.6rc2/proofstate/api/core/http_sse/_api.py +114 -0
- proofstate-4.15.6rc2/proofstate/api/core/http_sse/_decoders.py +66 -0
- proofstate-4.15.6rc2/proofstate/api/core/http_sse/_exceptions.py +7 -0
- proofstate-4.15.6rc2/proofstate/api/core/http_sse/_models.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/core/jsonable_encoder.py +102 -0
- proofstate-4.15.6rc2/proofstate/api/core/pydantic_utilities.py +310 -0
- proofstate-4.15.6rc2/proofstate/api/core/query_encoder.py +60 -0
- proofstate-4.15.6rc2/proofstate/api/core/remove_none_from_dict.py +11 -0
- proofstate-4.15.6rc2/proofstate/api/core/request_options.py +35 -0
- proofstate-4.15.6rc2/proofstate/api/core/serialization.py +282 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_items/__init__.py +52 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_items/client.py +499 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_items/raw_client.py +973 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_items/types/__init__.py +50 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_items/types/create_dataset_item_request.py +37 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_items/types/delete_dataset_item_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_items/types/paginated_dataset_items.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_run_items/__init__.py +43 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_run_items/client.py +350 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_run_items/raw_client.py +574 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_run_items/types/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_run_items/types/create_dataset_run_item_request.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/dataset_run_items/types/paginated_dataset_run_items.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/__init__.py +55 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/client.py +686 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/raw_client.py +1393 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/types/__init__.py +53 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/types/create_dataset_request.py +31 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/types/delete_dataset_run_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/types/paginated_dataset_runs.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/datasets/types/paginated_datasets.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/__init__.py +196 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/__init__.py +212 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/array_options_evaluation_rule_filter.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/boolean_evaluation_rule_filter.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/category_options_evaluation_rule_filter.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/code_evaluator_source_code_language.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/date_time_evaluation_rule_filter.py +29 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_array_options_filter_operator.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_boolean_filter_operator.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_filter.py +680 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_null_filter_operator.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_number_filter_operator.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_options_filter_operator.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_read_filter.py +10 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_read_filter_base.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_read_filter_with_key.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluation_rule_string_filter_operator.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluator_output_definition.py +143 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluator_output_definition_base.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluator_output_score_type.py +35 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/evaluator_type.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/legacy_evaluation_object.py +66 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/legacy_prompt_variable_mapping.py +72 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/null_evaluation_rule_filter.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/number_evaluation_rule_filter.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/number_object_evaluation_rule_filter.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/prompt_variable_mapping.py +10 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/prompt_variable_mapping_input.py +70 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/prompt_variable_mapping_read.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/prompt_variable_mapping_source.py +52 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/public_evaluator_boolean_score.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/public_evaluator_categorical_score.py +33 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/public_evaluator_numeric_score.py +35 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/public_evaluator_output_definition.py +134 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/string_evaluation_rule_filter.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/string_object_evaluation_rule_filter.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_commons/types/string_options_evaluation_rule_filter.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/__init__.py +84 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/__init__.py +68 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/bad_request_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/conflict_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/evaluation_access_denied_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/evaluation_method_not_allowed_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/evaluation_not_found_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/evaluation_unauthorized_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/internal_server_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/precondition_failed_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/errors/too_many_requests_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/types/__init__.py +53 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/types/public_api_error.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/types/public_api_error_code.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/types/public_api_error_details.py +51 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_errors/types/public_api_validation_issue.py +31 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/__init__.py +64 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/client.py +671 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/raw_client.py +2038 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/__init__.py +64 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/create_evaluation_rule_request.py +48 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/deleted_evaluation_rule.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/evaluation_rule.py +83 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/evaluation_rule_evaluator_assignment_input.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/evaluation_rules_page.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/evaluator_assignment.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/evaluation_rules/types/update_evaluation_rule_request.py +52 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/__init__.py +142 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/client.py +674 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/raw_client.py +2300 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/__init__.py +142 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/code_evaluator.py +36 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/code_evaluator_version.py +36 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/create_code_evaluator_request.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/create_evaluator_request.py +70 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/create_llm_as_judge_evaluator_request.py +64 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/creator.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/cursor_meta.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/deleted_evaluator.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluation_rule_assignment.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator.py +143 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_base.py +114 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_chat_message.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_chat_message_role.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_chat_prompt.py +7 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_chat_prompt_input.py +7 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_model_config.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_status.py +29 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_version.py +77 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_version_base.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluator_versions_page.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/evaluators_page.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/llm_as_judge_evaluator.py +59 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/llm_as_judge_evaluator_version.py +63 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/update_code_evaluator_request.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/update_evaluator_metadata_request.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/update_evaluator_request.py +13 -0
- proofstate-4.15.6rc2/proofstate/api/evaluators/types/update_llm_as_judge_evaluator_request.py +64 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/__init__.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/client.py +456 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/raw_client.py +677 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/types/__init__.py +56 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/types/experiment.py +62 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/types/experiment_item.py +100 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/types/experiment_items_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/types/experiments_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/experiments/types/experiments_response_meta.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/__init__.py +51 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/client.py +188 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/errors/__init__.py +42 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/errors/feedback_not_configured_error.py +13 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/raw_client.py +295 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/types/__init__.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/types/feedback_target_type.py +38 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/types/submit_feedback_request.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/feedback/types/submit_feedback_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/health/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/health/client.py +112 -0
- proofstate-4.15.6rc2/proofstate/api/health/errors/__init__.py +42 -0
- proofstate-4.15.6rc2/proofstate/api/health/errors/service_unavailable_error.py +13 -0
- proofstate-4.15.6rc2/proofstate/api/health/raw_client.py +227 -0
- proofstate-4.15.6rc2/proofstate/api/health/types/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/health/types/health_response.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/__init__.py +169 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/client.py +228 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/raw_client.py +300 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/__init__.py +169 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/base_event.py +27 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/create_event_body.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/create_event_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/create_generation_body.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/create_generation_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/create_observation_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/create_span_body.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/create_span_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/ingestion_error.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/ingestion_event.py +155 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/ingestion_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/ingestion_success.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/ingestion_usage.py +8 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/observation_body.py +53 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/observation_type.py +54 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/open_ai_completion_usage_schema.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/open_ai_response_usage_schema.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/open_ai_usage.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/optional_observation_body.py +36 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/score_body.py +75 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/score_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/sdk_log_body.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/sdk_log_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/trace_body.py +36 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/trace_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/update_event_body.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/update_generation_body.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/update_generation_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/update_observation_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/update_span_body.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/update_span_event.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/ingestion/types/usage_details.py +10 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/__init__.py +63 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/client.py +105 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/metrics_v1/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/metrics_v1/client.py +221 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/metrics_v1/raw_client.py +329 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/metrics_v1/types/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/metrics_v1/types/metrics_response.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/observations_v1/__init__.py +43 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/observations_v1/client.py +540 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/observations_v1/raw_client.py +776 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/observations_v1/types/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/observations_v1/types/observations.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/observations_v1/types/observations_views.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/raw_client.py +13 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/score_v1/__init__.py +10 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/score_v1/client.py +212 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/score_v1/raw_client.py +214 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/score_v1/types/__init__.py +10 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/score_v1/types/create_score_request.py +5 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/score_v1/types/create_score_response.py +5 -0
- proofstate-4.15.6rc2/proofstate/api/legacy/score_v1/types/create_score_source.py +5 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/__init__.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/client.py +392 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/raw_client.py +743 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/types/__init__.py +56 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/types/delete_llm_connection_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/types/llm_adapter.py +38 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/types/llm_connection.py +77 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/types/paginated_llm_connections.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/llm_connections/types/upsert_llm_connection_request.py +69 -0
- proofstate-4.15.6rc2/proofstate/api/media/__init__.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/media/client.py +445 -0
- proofstate-4.15.6rc2/proofstate/api/media/raw_client.py +759 -0
- proofstate-4.15.6rc2/proofstate/api/media/types/__init__.py +56 -0
- proofstate-4.15.6rc2/proofstate/api/media/types/get_media_response.py +55 -0
- proofstate-4.15.6rc2/proofstate/api/media/types/get_media_upload_url_request.py +69 -0
- proofstate-4.15.6rc2/proofstate/api/media/types/get_media_upload_url_response.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/media/types/media_content_type.py +232 -0
- proofstate-4.15.6rc2/proofstate/api/media/types/patch_media_body.py +43 -0
- proofstate-4.15.6rc2/proofstate/api/metrics/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/metrics/client.py +466 -0
- proofstate-4.15.6rc2/proofstate/api/metrics/raw_client.py +574 -0
- proofstate-4.15.6rc2/proofstate/api/metrics/types/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/metrics/types/metrics_v2response.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/models/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/models/client.py +762 -0
- proofstate-4.15.6rc2/proofstate/api/models/raw_client.py +1348 -0
- proofstate-4.15.6rc2/proofstate/api/models/types/__init__.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/models/types/create_model_request.py +104 -0
- proofstate-4.15.6rc2/proofstate/api/models/types/model_tokenizer_id.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/models/types/paginated_models.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/observations/__init__.py +43 -0
- proofstate-4.15.6rc2/proofstate/api/observations/client.py +586 -0
- proofstate-4.15.6rc2/proofstate/api/observations/raw_client.py +705 -0
- proofstate-4.15.6rc2/proofstate/api/observations/types/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/observations/types/observations_v2meta.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/observations/types/observations_v2response.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/__init__.py +67 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/client.py +272 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/raw_client.py +287 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/__init__.py +65 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_attribute.py +27 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_attribute_value.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_resource.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_resource_span.py +32 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_scope.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_scope_span.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_span.py +76 -0
- proofstate-4.15.6rc2/proofstate/api/opentelemetry/types/otel_trace_response.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/__init__.py +73 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/client.py +756 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/raw_client.py +1707 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/__init__.py +71 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/delete_membership_request.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/membership_deletion_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/membership_request.py +18 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/membership_response.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/membership_role.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/memberships_response.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/organization_api_key.py +31 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/organization_api_keys_response.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/organization_project.py +25 -0
- proofstate-4.15.6rc2/proofstate/api/organizations/types/organization_projects_response.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/projects/__init__.py +67 -0
- proofstate-4.15.6rc2/proofstate/api/projects/client.py +760 -0
- proofstate-4.15.6rc2/proofstate/api/projects/raw_client.py +1577 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/__init__.py +65 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/api_key_deletion_response.py +18 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/api_key_list.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/api_key_response.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/api_key_summary.py +35 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/organization.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/project.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/project_deletion_response.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/projects/types/projects.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/prompt_version/__init__.py +4 -0
- proofstate-4.15.6rc2/proofstate/api/prompt_version/client.py +157 -0
- proofstate-4.15.6rc2/proofstate/api/prompt_version/raw_client.py +264 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/__init__.py +100 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/client.py +550 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/raw_client.py +987 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/__init__.py +96 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/base_prompt.py +42 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/chat_message.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/chat_message_type.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/chat_message_with_placeholders.py +8 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/chat_prompt.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/create_chat_prompt_request.py +37 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/create_chat_prompt_type.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/create_prompt_request.py +8 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/create_text_prompt_request.py +36 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/create_text_prompt_type.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/placeholder_message.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/placeholder_message_type.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/prompt.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/prompt_meta.py +35 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/prompt_meta_list_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/prompt_type.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/prompts/types/text_prompt.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/scim/__init__.py +94 -0
- proofstate-4.15.6rc2/proofstate/api/scim/client.py +686 -0
- proofstate-4.15.6rc2/proofstate/api/scim/raw_client.py +1528 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/__init__.py +92 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/authentication_scheme.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/bulk_config.py +22 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/empty_response.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/filter_config.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/resource_meta.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/resource_type.py +27 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/resource_types_response.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/schema_extension.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/schema_resource.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/schemas_response.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/scim_email.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/scim_feature_support.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/scim_name.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/scim_user.py +24 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/scim_users_list_response.py +25 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/service_provider_config.py +36 -0
- proofstate-4.15.6rc2/proofstate/api/scim/types/user_meta.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/score_configs/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/score_configs/client.py +528 -0
- proofstate-4.15.6rc2/proofstate/api/score_configs/raw_client.py +1014 -0
- proofstate-4.15.6rc2/proofstate/api/score_configs/types/__init__.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/score_configs/types/create_score_config_request.py +50 -0
- proofstate-4.15.6rc2/proofstate/api/score_configs/types/score_configs.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/score_configs/types/update_score_config_request.py +53 -0
- proofstate-4.15.6rc2/proofstate/api/scores/__init__.py +91 -0
- proofstate-4.15.6rc2/proofstate/api/scores/client.py +653 -0
- proofstate-4.15.6rc2/proofstate/api/scores/raw_client.py +1014 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/__init__.py +91 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/create_score_request.py +81 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/create_score_response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/create_score_source.py +28 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response_data.py +258 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response_data_boolean.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response_data_categorical.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response_data_correction.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response_data_numeric.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response_data_text.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/scores/types/get_scores_response_trace_data.py +38 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/__init__.py +112 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/client.py +355 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/raw_client.py +474 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/__init__.py +114 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/base_score_v3.py +71 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/boolean_score_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/categorical_score_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/correction_score_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/get_scores_v3meta.py +18 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/get_scores_v3response.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/numeric_score_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/score_subject_experiment_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/score_subject_observation_v3.py +26 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/score_subject_session_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/score_subject_trace_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/score_subject_v3.py +76 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/score_v3.py +200 -0
- proofstate-4.15.6rc2/proofstate/api/scores_v3/types/text_score_v3.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/sessions/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/sessions/client.py +299 -0
- proofstate-4.15.6rc2/proofstate/api/sessions/raw_client.py +537 -0
- proofstate-4.15.6rc2/proofstate/api/sessions/types/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/sessions/types/paginated_sessions.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/trace/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/trace/client.py +765 -0
- proofstate-4.15.6rc2/proofstate/api/trace/raw_client.py +1249 -0
- proofstate-4.15.6rc2/proofstate/api/trace/types/__init__.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/trace/types/delete_trace_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/trace/types/sort.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/trace/types/traces.py +23 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/__init__.py +192 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/client.py +91 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/__init__.py +91 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/client.py +735 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/raw_client.py +2114 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/__init__.py +89 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/create_dashboard_widget_request.py +41 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget.py +41 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_chart_config.py +33 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_chart_config_input.py +29 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_chart_type.py +50 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_default_sort.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_dimension.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_filter.py +37 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_list.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_metric.py +16 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_metric_aggregation.py +62 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_sort_order.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_view.py +30 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/dashboard_widget_view_with_legacy.py +39 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/delete_dashboard_widget_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboard_widgets/types/update_dashboard_widget_request.py +34 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/__init__.py +97 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/client.py +934 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/raw_client.py +3165 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/__init__.py +99 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/create_dashboard_placement_request.py +46 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/create_dashboard_request.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/create_preset_placement.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/create_widget_placement.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/dashboard.py +32 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/dashboard_definition.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/dashboard_list.py +17 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/dashboard_placement.py +60 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/delete_dashboard_placement_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/delete_dashboard_response.py +14 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/preset_placement.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/update_dashboard_placement_request.py +32 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/update_dashboard_request.py +19 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/dashboards/types/widget_placement.py +21 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/__init__.py +81 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/__init__.py +65 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/access_denied_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/bad_request_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/conflict_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/internal_server_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/method_not_allowed_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/not_found_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/too_many_requests_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/errors/unauthorized_error.py +15 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/types/__init__.py +53 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/types/public_api_error.py +33 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/types/public_api_error_code.py +58 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/types/public_api_error_details.py +38 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/errors/types/public_api_validation_issue.py +20 -0
- proofstate-4.15.6rc2/proofstate/api/unstable/raw_client.py +13 -0
- proofstate-4.15.6rc2/proofstate/api/utils/__init__.py +44 -0
- proofstate-4.15.6rc2/proofstate/api/utils/pagination/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/utils/pagination/types/__init__.py +40 -0
- proofstate-4.15.6rc2/proofstate/api/utils/pagination/types/meta_response.py +38 -0
- proofstate-4.15.6rc2/proofstate/batch_evaluation.py +1656 -0
- proofstate-4.15.6rc2/proofstate/experiment.py +1213 -0
- proofstate-4.15.6rc2/proofstate/langchain/CallbackHandler.py +1823 -0
- proofstate-4.15.6rc2/proofstate/langchain/__init__.py +5 -0
- proofstate-4.15.6rc2/proofstate/langchain/utils.py +245 -0
- proofstate-4.15.6rc2/proofstate/logger.py +28 -0
- proofstate-4.15.6rc2/proofstate/media.py +436 -0
- proofstate-4.15.6rc2/proofstate/model.py +479 -0
- proofstate-4.15.6rc2/proofstate/openai.py +1667 -0
- proofstate-4.15.6rc2/proofstate/py.typed +0 -0
- proofstate-4.15.6rc2/proofstate/span_filter.py +17 -0
- proofstate-4.15.6rc2/proofstate/types.py +311 -0
- proofstate-4.15.6rc2/pyproject.toml +156 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Finto Technologies
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: proofstate
|
|
3
|
+
Version: 4.15.6rc2
|
|
4
|
+
Summary: ProofState Python SDK for LLM observability, evaluation, and prompt management
|
|
5
|
+
Keywords: proofstate,llm,observability,tracing,opentelemetry,evaluation,llm-as-a-judge,prompt-management,datasets,experiments,openai,langchain,ai-agents,monitoring
|
|
6
|
+
Author: ProofState
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: httpx>=0.15.4,<1.0
|
|
10
|
+
Requires-Dist: pydantic>=2,<3
|
|
11
|
+
Requires-Dist: backoff>=1.10.0
|
|
12
|
+
Requires-Dist: wrapt>=1.14,<3
|
|
13
|
+
Requires-Dist: packaging>=23.2,<27.0
|
|
14
|
+
Requires-Dist: opentelemetry-api>=1.33.1,<2
|
|
15
|
+
Requires-Dist: opentelemetry-sdk>=1.33.1,<2
|
|
16
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.33.1,<2
|
|
17
|
+
Requires-Dist: typing-extensions>=4.12.2,<5
|
|
18
|
+
Requires-Python: >=3.10, <4.0
|
|
19
|
+
Project-URL: Homepage, https://proofstate.ai
|
|
20
|
+
Project-URL: Repository, https://github.com/JSOCIT-Inc/proofstate-python
|
|
21
|
+
Project-URL: Issues, https://github.com/JSOCIT-Inc/proofstate-python/issues
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# ProofState Python SDK
|
|
25
|
+
|
|
26
|
+
Python client for [ProofState](https://proofstate.ai): OpenTelemetry tracing, prompts, datasets, evaluations, scores, and the public REST API. The required MIT license notice is in [LICENSE](LICENSE).
|
|
27
|
+
|
|
28
|
+
The protocol used by the `4.15.6rc2` prerelease was checked against the ProofState server deployed at `https://proofstate.ai` on 2026-09-28. Use a test project when adopting a prerelease.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
Use Python 3.10 or newer. After publication, install the prerelease from PyPI:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
python -m pip install --pre proofstate==4.15.6rc2
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The distribution and import name are both `proofstate`. To install from a source checkout instead, run `python -m pip install .` in the repository root.
|
|
39
|
+
|
|
40
|
+
## Configure
|
|
41
|
+
|
|
42
|
+
Create a ProofState project and obtain its public and secret keys. Set:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
PROOFSTATE_PUBLIC_KEY=pk-ps-your-public-key
|
|
46
|
+
PROOFSTATE_SECRET_KEY=sk-ps-your-secret-key
|
|
47
|
+
PROOFSTATE_BASE_URL=https://proofstate.ai
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
`PROOFSTATE_BASE_URL` is optional because `https://proofstate.ai` is the default. You can also pass `public_key`, `secret_key`, and `base_url` to `ProofState(...)` directly. Keep the secret key on the server side.
|
|
51
|
+
|
|
52
|
+
## First trace and prompt
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from proofstate import get_client
|
|
56
|
+
|
|
57
|
+
client = get_client()
|
|
58
|
+
|
|
59
|
+
# Fetch the production prompt; fallback is used if it is unavailable.
|
|
60
|
+
prompt = client.get_prompt("greeting", fallback="Hello, {{name}}!")
|
|
61
|
+
message = prompt.compile(name="Ada")
|
|
62
|
+
|
|
63
|
+
with client.start_as_current_observation(
|
|
64
|
+
as_type="generation", name="greeting", input={"name": "Ada"}
|
|
65
|
+
) as generation:
|
|
66
|
+
generation.update(output=message)
|
|
67
|
+
|
|
68
|
+
client.flush() # Flush before a short-lived process exits.
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`ProofState` also provides `@observe`, OpenAI instrumentation through `proofstate.openai`, and a LangChain `CallbackHandler` through `proofstate.langchain`.
|
|
72
|
+
|
|
73
|
+
## Protocol and rollout
|
|
74
|
+
|
|
75
|
+
The SDK sends `x-proofstate-*` headers, `proofstate.*` OpenTelemetry attributes, `proofstate-sdk` instrumentation scope, `proofstate_` baggage, `@@@proofstateMedia` references, and ProofState JSON field names. Its SDK identity is `proofstate-python` and it requests ingestion version `4`.
|
|
76
|
+
|
|
77
|
+
Use a matching ProofState server protocol. Authenticated production checks on 2026-09-28 covered project authentication, prompts, datasets, scores, experiments, span ingestion, v2 observation readback, and MCP tool discovery. No legacy-named fallback is emitted by this SDK. On a server running v4 `events_only` mode, the legacy `client.api.trace.get()` endpoint returns 404; read spans through `client.api.observations.get_many(trace_id=...)` instead.
|
|
78
|
+
|
|
79
|
+
The generated `proofstate/api/` client mirrors an API schema snapshot. When ProofState's API diverges, regenerate it from ProofState's OpenAPI/Fern source before publishing a new SDK release.
|
|
80
|
+
|
|
81
|
+
The GitHub E2E job tests a local ProofState server checkout. It is skipped unless the repository variable `PROOFSTATE_E2E_ENABLED` is `true` and the `PROOFSTATE_SERVER_REPO_TOKEN` secret can read `JSOCIT-Inc/proofstate`. Live provider tests additionally need the provider API secrets. Unit, lint, type, and build checks run without those credentials.
|
|
82
|
+
|
|
83
|
+
## Development and release preparation
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uv sync --locked
|
|
87
|
+
uv run --frozen ruff check .
|
|
88
|
+
uv run --frozen mypy proofstate --no-error-summary
|
|
89
|
+
uv run --frozen pytest tests/unit
|
|
90
|
+
python scripts/sanitize_generated_api_docs.py --check
|
|
91
|
+
uv build --no-sources
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Before each release, review the generated API against the deployed server, run server-backed tests, and install the built wheel and source archive in clean environments. Tag only a version that passed those checks.
|
|
95
|
+
|
|
96
|
+
The `.github/workflows/publish.yml` workflow builds and checks the distributions, then publishes a GitHub prerelease whose tag matches the package version using PyPI Trusted Publishing. It does not use a stored PyPI token. The PyPI publisher is scoped to owner `JSOCIT-Inc`, repository `proofstate-python`, workflow `publish.yml`, and environment `pypi`. The GitHub `pypi` environment requires release review. After server-backed compatibility checks pass, set repository variable `PROOFSTATE_RELEASE_COMPAT_VERIFIED_TAG` to the exact release tag; the workflow fails closed if it does not match.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# ProofState Python SDK
|
|
2
|
+
|
|
3
|
+
Python client for [ProofState](https://proofstate.ai): OpenTelemetry tracing, prompts, datasets, evaluations, scores, and the public REST API. The required MIT license notice is in [LICENSE](LICENSE).
|
|
4
|
+
|
|
5
|
+
The protocol used by the `4.15.6rc2` prerelease was checked against the ProofState server deployed at `https://proofstate.ai` on 2026-09-28. Use a test project when adopting a prerelease.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
Use Python 3.10 or newer. After publication, install the prerelease from PyPI:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
python -m pip install --pre proofstate==4.15.6rc2
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The distribution and import name are both `proofstate`. To install from a source checkout instead, run `python -m pip install .` in the repository root.
|
|
16
|
+
|
|
17
|
+
## Configure
|
|
18
|
+
|
|
19
|
+
Create a ProofState project and obtain its public and secret keys. Set:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
PROOFSTATE_PUBLIC_KEY=pk-ps-your-public-key
|
|
23
|
+
PROOFSTATE_SECRET_KEY=sk-ps-your-secret-key
|
|
24
|
+
PROOFSTATE_BASE_URL=https://proofstate.ai
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
`PROOFSTATE_BASE_URL` is optional because `https://proofstate.ai` is the default. You can also pass `public_key`, `secret_key`, and `base_url` to `ProofState(...)` directly. Keep the secret key on the server side.
|
|
28
|
+
|
|
29
|
+
## First trace and prompt
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from proofstate import get_client
|
|
33
|
+
|
|
34
|
+
client = get_client()
|
|
35
|
+
|
|
36
|
+
# Fetch the production prompt; fallback is used if it is unavailable.
|
|
37
|
+
prompt = client.get_prompt("greeting", fallback="Hello, {{name}}!")
|
|
38
|
+
message = prompt.compile(name="Ada")
|
|
39
|
+
|
|
40
|
+
with client.start_as_current_observation(
|
|
41
|
+
as_type="generation", name="greeting", input={"name": "Ada"}
|
|
42
|
+
) as generation:
|
|
43
|
+
generation.update(output=message)
|
|
44
|
+
|
|
45
|
+
client.flush() # Flush before a short-lived process exits.
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`ProofState` also provides `@observe`, OpenAI instrumentation through `proofstate.openai`, and a LangChain `CallbackHandler` through `proofstate.langchain`.
|
|
49
|
+
|
|
50
|
+
## Protocol and rollout
|
|
51
|
+
|
|
52
|
+
The SDK sends `x-proofstate-*` headers, `proofstate.*` OpenTelemetry attributes, `proofstate-sdk` instrumentation scope, `proofstate_` baggage, `@@@proofstateMedia` references, and ProofState JSON field names. Its SDK identity is `proofstate-python` and it requests ingestion version `4`.
|
|
53
|
+
|
|
54
|
+
Use a matching ProofState server protocol. Authenticated production checks on 2026-09-28 covered project authentication, prompts, datasets, scores, experiments, span ingestion, v2 observation readback, and MCP tool discovery. No legacy-named fallback is emitted by this SDK. On a server running v4 `events_only` mode, the legacy `client.api.trace.get()` endpoint returns 404; read spans through `client.api.observations.get_many(trace_id=...)` instead.
|
|
55
|
+
|
|
56
|
+
The generated `proofstate/api/` client mirrors an API schema snapshot. When ProofState's API diverges, regenerate it from ProofState's OpenAPI/Fern source before publishing a new SDK release.
|
|
57
|
+
|
|
58
|
+
The GitHub E2E job tests a local ProofState server checkout. It is skipped unless the repository variable `PROOFSTATE_E2E_ENABLED` is `true` and the `PROOFSTATE_SERVER_REPO_TOKEN` secret can read `JSOCIT-Inc/proofstate`. Live provider tests additionally need the provider API secrets. Unit, lint, type, and build checks run without those credentials.
|
|
59
|
+
|
|
60
|
+
## Development and release preparation
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
uv sync --locked
|
|
64
|
+
uv run --frozen ruff check .
|
|
65
|
+
uv run --frozen mypy proofstate --no-error-summary
|
|
66
|
+
uv run --frozen pytest tests/unit
|
|
67
|
+
python scripts/sanitize_generated_api_docs.py --check
|
|
68
|
+
uv build --no-sources
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Before each release, review the generated API against the deployed server, run server-backed tests, and install the built wheel and source archive in clean environments. Tag only a version that passed those checks.
|
|
72
|
+
|
|
73
|
+
The `.github/workflows/publish.yml` workflow builds and checks the distributions, then publishes a GitHub prerelease whose tag matches the package version using PyPI Trusted Publishing. It does not use a stored PyPI token. The PyPI publisher is scoped to owner `JSOCIT-Inc`, repository `proofstate-python`, workflow `publish.yml`, and environment `pypi`. The GitHub `pypi` environment requires release review. After server-backed compatibility checks pass, set repository variable `PROOFSTATE_RELEASE_COMPAT_VERIFIED_TAG` to the exact release tag; the workflow fails closed if it does not match.
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""ProofState Python SDK — observability, evaluation, and prompt management for LLM applications.
|
|
2
|
+
|
|
3
|
+
Capabilities:
|
|
4
|
+
|
|
5
|
+
- **Tracing / observability**: `@observe` decorator, `ProofState.start_observation` /
|
|
6
|
+
`start_as_current_observation` context managers, OpenTelemetry-based; integrations
|
|
7
|
+
for OpenAI (`proofstate.openai`) and LangChain (`proofstate.langchain.CallbackHandler`).
|
|
8
|
+
- **Trace attributes**: `propagate_attributes` (top-level function) sets user_id,
|
|
9
|
+
session_id, tags, and metadata on all spans in a context.
|
|
10
|
+
- **Datasets & experiments**: `ProofState.get_dataset`, `ProofState.run_experiment` for
|
|
11
|
+
offline evaluation and regression testing of prompt/model changes.
|
|
12
|
+
- **Evaluation / LLM-as-a-judge**: `Evaluation` results from custom or model-based
|
|
13
|
+
evaluators; scores via `ProofState.create_score` / `span.score`.
|
|
14
|
+
- **Prompt management**: `ProofState.get_prompt`, `ProofState.create_prompt` with
|
|
15
|
+
client-side caching and version/label control.
|
|
16
|
+
- **Full REST API**: `ProofState.api` (sync) / `ProofState.async_api` (async) clients.
|
|
17
|
+
|
|
18
|
+
Quickstart:
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
# env: PROOFSTATE_PUBLIC_KEY, PROOFSTATE_SECRET_KEY, PROOFSTATE_BASE_URL
|
|
22
|
+
from proofstate import get_client
|
|
23
|
+
|
|
24
|
+
client = get_client()
|
|
25
|
+
|
|
26
|
+
# Create a span using a context manager
|
|
27
|
+
with client.start_as_current_observation(as_type="span", name="process-request") as span:
|
|
28
|
+
# Your processing logic here
|
|
29
|
+
span.update(output="Processing complete")
|
|
30
|
+
|
|
31
|
+
# Create a nested generation for an LLM call
|
|
32
|
+
with client.start_as_current_observation(as_type="generation", name="llm-response", model="gpt-3.5-turbo") as generation:
|
|
33
|
+
# Your LLM call logic here
|
|
34
|
+
generation.update(output="Generated response")
|
|
35
|
+
|
|
36
|
+
# All spans are automatically closed when exiting their context blocks
|
|
37
|
+
|
|
38
|
+
# Flush events in short-lived applications
|
|
39
|
+
client.flush()
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Configuration is via constructor args or environment variables: `PROOFSTATE_PUBLIC_KEY`,
|
|
43
|
+
`PROOFSTATE_SECRET_KEY`, `PROOFSTATE_BASE_URL` (defaults to https://proofstate.ai). See `proofstate._client.environment_variables`
|
|
44
|
+
for the full list.
|
|
45
|
+
|
|
46
|
+
Docs: https://proofstate.ai
|
|
47
|
+
|
|
48
|
+
.. include:: ../README.md
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
from proofstate.batch_evaluation import (
|
|
52
|
+
BatchEvaluationResult,
|
|
53
|
+
BatchEvaluationResumeToken,
|
|
54
|
+
CompositeEvaluatorFunction,
|
|
55
|
+
EvaluatorInputs,
|
|
56
|
+
EvaluatorStats,
|
|
57
|
+
MapperFunction,
|
|
58
|
+
)
|
|
59
|
+
from proofstate.experiment import Evaluation, RegressionError, RunnerContext
|
|
60
|
+
|
|
61
|
+
from ._client import client as _client_module
|
|
62
|
+
from ._client.attributes import ProofStateOtelSpanAttributes
|
|
63
|
+
from ._client.constants import ObservationTypeLiteral
|
|
64
|
+
from ._client.get_client import get_client
|
|
65
|
+
from ._client.observe import observe
|
|
66
|
+
from ._client.propagation import propagate_attributes
|
|
67
|
+
from ._client.span import (
|
|
68
|
+
ProofStateAgent,
|
|
69
|
+
ProofStateChain,
|
|
70
|
+
ProofStateEmbedding,
|
|
71
|
+
ProofStateEvaluator,
|
|
72
|
+
ProofStateEvent,
|
|
73
|
+
ProofStateGeneration,
|
|
74
|
+
ProofStateGuardrail,
|
|
75
|
+
ProofStateRetriever,
|
|
76
|
+
ProofStateSpan,
|
|
77
|
+
ProofStateTool,
|
|
78
|
+
)
|
|
79
|
+
from ._version import __version__
|
|
80
|
+
from .media import ProofStateMedia, ProofStateMediaReference
|
|
81
|
+
from .span_filter import (
|
|
82
|
+
KNOWN_LLM_INSTRUMENTATION_SCOPE_PREFIXES,
|
|
83
|
+
is_default_export_span,
|
|
84
|
+
is_genai_span,
|
|
85
|
+
is_known_llm_instrumentor,
|
|
86
|
+
is_proofstate_span,
|
|
87
|
+
)
|
|
88
|
+
from .types import (
|
|
89
|
+
MaskOtelSpansFunction,
|
|
90
|
+
MaskOtelSpansParams,
|
|
91
|
+
MaskOtelSpansResult,
|
|
92
|
+
OtelSpanData,
|
|
93
|
+
OtelSpanIdentifier,
|
|
94
|
+
OtelSpanPatch,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
ProofState = _client_module.ProofState
|
|
98
|
+
|
|
99
|
+
__all__ = [
|
|
100
|
+
"ProofState",
|
|
101
|
+
"ProofStateMedia",
|
|
102
|
+
"ProofStateMediaReference",
|
|
103
|
+
"get_client",
|
|
104
|
+
"observe",
|
|
105
|
+
"propagate_attributes",
|
|
106
|
+
"ObservationTypeLiteral",
|
|
107
|
+
"ProofStateSpan",
|
|
108
|
+
"ProofStateGeneration",
|
|
109
|
+
"ProofStateEvent",
|
|
110
|
+
"ProofStateOtelSpanAttributes",
|
|
111
|
+
"ProofStateAgent",
|
|
112
|
+
"ProofStateTool",
|
|
113
|
+
"ProofStateChain",
|
|
114
|
+
"ProofStateEmbedding",
|
|
115
|
+
"ProofStateEvaluator",
|
|
116
|
+
"ProofStateRetriever",
|
|
117
|
+
"ProofStateGuardrail",
|
|
118
|
+
"Evaluation",
|
|
119
|
+
"EvaluatorInputs",
|
|
120
|
+
"MapperFunction",
|
|
121
|
+
"CompositeEvaluatorFunction",
|
|
122
|
+
"EvaluatorStats",
|
|
123
|
+
"BatchEvaluationResumeToken",
|
|
124
|
+
"BatchEvaluationResult",
|
|
125
|
+
"RunnerContext",
|
|
126
|
+
"RegressionError",
|
|
127
|
+
"__version__",
|
|
128
|
+
"is_default_export_span",
|
|
129
|
+
"is_proofstate_span",
|
|
130
|
+
"is_genai_span",
|
|
131
|
+
"is_known_llm_instrumentor",
|
|
132
|
+
"KNOWN_LLM_INSTRUMENTATION_SCOPE_PREFIXES",
|
|
133
|
+
"MaskOtelSpansFunction",
|
|
134
|
+
"MaskOtelSpansParams",
|
|
135
|
+
"MaskOtelSpansResult",
|
|
136
|
+
"OtelSpanData",
|
|
137
|
+
"OtelSpanIdentifier",
|
|
138
|
+
"OtelSpanPatch",
|
|
139
|
+
"experiment",
|
|
140
|
+
"api",
|
|
141
|
+
]
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Span attribute management for ProofState OpenTelemetry integration.
|
|
2
|
+
|
|
3
|
+
This module defines constants and functions for managing OpenTelemetry span attributes
|
|
4
|
+
used by ProofState. It provides a structured approach to creating and manipulating
|
|
5
|
+
attributes for different span types (trace, span, generation) while ensuring consistency.
|
|
6
|
+
|
|
7
|
+
The module includes:
|
|
8
|
+
- Attribute name constants organized by category
|
|
9
|
+
- Functions to create attribute dictionaries for different entity types
|
|
10
|
+
- Utilities for serializing and processing attribute values
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
from typing import Any, Dict, Literal, Optional, Union
|
|
16
|
+
|
|
17
|
+
from proofstate._client.constants import (
|
|
18
|
+
ObservationTypeGenerationLike,
|
|
19
|
+
ObservationTypeSpanLike,
|
|
20
|
+
)
|
|
21
|
+
from proofstate._utils.serializer import EventSerializer
|
|
22
|
+
from proofstate.api import MapValue
|
|
23
|
+
from proofstate.model import PromptClient
|
|
24
|
+
from proofstate.types import SpanLevel
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ProofStateOtelSpanAttributes:
|
|
28
|
+
# ProofState-Trace attributes
|
|
29
|
+
TRACE_NAME = "proofstate.trace.name"
|
|
30
|
+
TRACE_USER_ID = "user.id"
|
|
31
|
+
TRACE_SESSION_ID = "session.id"
|
|
32
|
+
TRACE_TAGS = "proofstate.trace.tags"
|
|
33
|
+
TRACE_PUBLIC = "proofstate.trace.public"
|
|
34
|
+
TRACE_METADATA = "proofstate.trace.metadata"
|
|
35
|
+
TRACE_INPUT = "proofstate.trace.input"
|
|
36
|
+
TRACE_OUTPUT = "proofstate.trace.output"
|
|
37
|
+
|
|
38
|
+
# ProofState-observation attributes
|
|
39
|
+
OBSERVATION_TYPE = "proofstate.observation.type"
|
|
40
|
+
OBSERVATION_METADATA = "proofstate.observation.metadata"
|
|
41
|
+
OBSERVATION_LEVEL = "proofstate.observation.level"
|
|
42
|
+
OBSERVATION_STATUS_MESSAGE = "proofstate.observation.status_message"
|
|
43
|
+
OBSERVATION_INPUT = "proofstate.observation.input"
|
|
44
|
+
OBSERVATION_OUTPUT = "proofstate.observation.output"
|
|
45
|
+
|
|
46
|
+
# ProofState-observation of type Generation attributes
|
|
47
|
+
OBSERVATION_COMPLETION_START_TIME = "proofstate.observation.completion_start_time"
|
|
48
|
+
OBSERVATION_MODEL = "proofstate.observation.model.name"
|
|
49
|
+
OBSERVATION_MODEL_PARAMETERS = "proofstate.observation.model.parameters"
|
|
50
|
+
OBSERVATION_USAGE_DETAILS = "proofstate.observation.usage_details"
|
|
51
|
+
OBSERVATION_COST_DETAILS = "proofstate.observation.cost_details"
|
|
52
|
+
OBSERVATION_PROMPT_NAME = "proofstate.observation.prompt.name"
|
|
53
|
+
OBSERVATION_PROMPT_VERSION = "proofstate.observation.prompt.version"
|
|
54
|
+
|
|
55
|
+
# General
|
|
56
|
+
ENVIRONMENT = "proofstate.environment"
|
|
57
|
+
RELEASE = "proofstate.release"
|
|
58
|
+
VERSION = "proofstate.version"
|
|
59
|
+
|
|
60
|
+
# Internal
|
|
61
|
+
AS_ROOT = "proofstate.internal.as_root"
|
|
62
|
+
IS_APP_ROOT = "proofstate.internal.is_app_root"
|
|
63
|
+
|
|
64
|
+
# Experiments
|
|
65
|
+
EXPERIMENT_ID = "proofstate.experiment.id"
|
|
66
|
+
EXPERIMENT_NAME = "proofstate.experiment.name"
|
|
67
|
+
EXPERIMENT_DESCRIPTION = "proofstate.experiment.description"
|
|
68
|
+
EXPERIMENT_METADATA = "proofstate.experiment.metadata"
|
|
69
|
+
EXPERIMENT_DATASET_ID = "proofstate.experiment.dataset.id"
|
|
70
|
+
EXPERIMENT_ITEM_ID = "proofstate.experiment.item.id"
|
|
71
|
+
EXPERIMENT_ITEM_EXPECTED_OUTPUT = "proofstate.experiment.item.expected_output"
|
|
72
|
+
EXPERIMENT_ITEM_METADATA = "proofstate.experiment.item.metadata"
|
|
73
|
+
EXPERIMENT_ITEM_ROOT_OBSERVATION_ID = (
|
|
74
|
+
"proofstate.experiment.item.root_observation_id"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def create_trace_attributes(
|
|
79
|
+
*,
|
|
80
|
+
input: Optional[Any] = None,
|
|
81
|
+
output: Optional[Any] = None,
|
|
82
|
+
public: Optional[bool] = None,
|
|
83
|
+
) -> dict:
|
|
84
|
+
attributes = {
|
|
85
|
+
ProofStateOtelSpanAttributes.TRACE_INPUT: _serialize(input),
|
|
86
|
+
ProofStateOtelSpanAttributes.TRACE_OUTPUT: _serialize(output),
|
|
87
|
+
ProofStateOtelSpanAttributes.TRACE_PUBLIC: public,
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return {k: v for k, v in attributes.items() if v is not None}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def create_span_attributes(
|
|
94
|
+
*,
|
|
95
|
+
metadata: Optional[Any] = None,
|
|
96
|
+
input: Optional[Any] = None,
|
|
97
|
+
output: Optional[Any] = None,
|
|
98
|
+
level: Optional[SpanLevel] = None,
|
|
99
|
+
status_message: Optional[str] = None,
|
|
100
|
+
version: Optional[str] = None,
|
|
101
|
+
observation_type: Optional[
|
|
102
|
+
Union[ObservationTypeSpanLike, Literal["event"]]
|
|
103
|
+
] = "span",
|
|
104
|
+
) -> dict:
|
|
105
|
+
attributes = {
|
|
106
|
+
ProofStateOtelSpanAttributes.OBSERVATION_TYPE: observation_type,
|
|
107
|
+
ProofStateOtelSpanAttributes.OBSERVATION_LEVEL: level,
|
|
108
|
+
ProofStateOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE: status_message,
|
|
109
|
+
ProofStateOtelSpanAttributes.VERSION: version,
|
|
110
|
+
ProofStateOtelSpanAttributes.OBSERVATION_INPUT: _serialize(input),
|
|
111
|
+
ProofStateOtelSpanAttributes.OBSERVATION_OUTPUT: _serialize(output),
|
|
112
|
+
**_flatten_and_serialize_metadata(metadata, "observation"),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return {k: v for k, v in attributes.items() if v is not None}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def create_generation_attributes(
|
|
119
|
+
*,
|
|
120
|
+
name: Optional[str] = None,
|
|
121
|
+
completion_start_time: Optional[datetime] = None,
|
|
122
|
+
metadata: Optional[Any] = None,
|
|
123
|
+
level: Optional[SpanLevel] = None,
|
|
124
|
+
status_message: Optional[str] = None,
|
|
125
|
+
version: Optional[str] = None,
|
|
126
|
+
model: Optional[str] = None,
|
|
127
|
+
model_parameters: Optional[Dict[str, MapValue]] = None,
|
|
128
|
+
input: Optional[Any] = None,
|
|
129
|
+
output: Optional[Any] = None,
|
|
130
|
+
usage_details: Optional[Dict[str, int]] = None,
|
|
131
|
+
cost_details: Optional[Dict[str, float]] = None,
|
|
132
|
+
prompt: Optional[PromptClient] = None,
|
|
133
|
+
observation_type: Optional[ObservationTypeGenerationLike] = "generation",
|
|
134
|
+
) -> dict:
|
|
135
|
+
attributes = {
|
|
136
|
+
ProofStateOtelSpanAttributes.OBSERVATION_TYPE: observation_type,
|
|
137
|
+
ProofStateOtelSpanAttributes.OBSERVATION_LEVEL: level,
|
|
138
|
+
ProofStateOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE: status_message,
|
|
139
|
+
ProofStateOtelSpanAttributes.VERSION: version,
|
|
140
|
+
ProofStateOtelSpanAttributes.OBSERVATION_INPUT: _serialize(input),
|
|
141
|
+
ProofStateOtelSpanAttributes.OBSERVATION_OUTPUT: _serialize(output),
|
|
142
|
+
ProofStateOtelSpanAttributes.OBSERVATION_MODEL: model,
|
|
143
|
+
ProofStateOtelSpanAttributes.OBSERVATION_PROMPT_NAME: prompt.name
|
|
144
|
+
if prompt and not prompt.is_fallback
|
|
145
|
+
else None,
|
|
146
|
+
ProofStateOtelSpanAttributes.OBSERVATION_PROMPT_VERSION: prompt.version
|
|
147
|
+
if prompt and not prompt.is_fallback
|
|
148
|
+
else None,
|
|
149
|
+
ProofStateOtelSpanAttributes.OBSERVATION_USAGE_DETAILS: _serialize(
|
|
150
|
+
usage_details
|
|
151
|
+
),
|
|
152
|
+
ProofStateOtelSpanAttributes.OBSERVATION_COST_DETAILS: _serialize(cost_details),
|
|
153
|
+
ProofStateOtelSpanAttributes.OBSERVATION_COMPLETION_START_TIME: _serialize(
|
|
154
|
+
completion_start_time
|
|
155
|
+
),
|
|
156
|
+
ProofStateOtelSpanAttributes.OBSERVATION_MODEL_PARAMETERS: _serialize(
|
|
157
|
+
model_parameters
|
|
158
|
+
),
|
|
159
|
+
**_flatten_and_serialize_metadata(metadata, "observation"),
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return {k: v for k, v in attributes.items() if v is not None}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _serialize(obj: Any) -> Optional[str]:
|
|
166
|
+
if obj is None or isinstance(obj, str):
|
|
167
|
+
return obj
|
|
168
|
+
|
|
169
|
+
return json.dumps(obj, cls=EventSerializer)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _flatten_and_serialize_metadata_values(
|
|
173
|
+
metadata: Optional[Dict[str, Any]],
|
|
174
|
+
) -> Optional[Dict[str, str]]:
|
|
175
|
+
if metadata is None:
|
|
176
|
+
return None
|
|
177
|
+
|
|
178
|
+
flattened_metadata: Dict[str, str] = {}
|
|
179
|
+
|
|
180
|
+
def flatten_value(path: str, value: Any) -> None:
|
|
181
|
+
if isinstance(value, dict):
|
|
182
|
+
for nested_key, nested_value in value.items():
|
|
183
|
+
flatten_value(f"{path}.{nested_key}", nested_value)
|
|
184
|
+
|
|
185
|
+
return
|
|
186
|
+
|
|
187
|
+
serialized_value = _serialize(value)
|
|
188
|
+
|
|
189
|
+
if serialized_value is not None:
|
|
190
|
+
flattened_metadata[path] = serialized_value
|
|
191
|
+
|
|
192
|
+
for key, value in metadata.items():
|
|
193
|
+
flatten_value(str(key), value)
|
|
194
|
+
|
|
195
|
+
return flattened_metadata
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _flatten_and_serialize_metadata(
|
|
199
|
+
metadata: Any, type: Literal["observation", "trace"]
|
|
200
|
+
) -> dict:
|
|
201
|
+
prefix = (
|
|
202
|
+
ProofStateOtelSpanAttributes.OBSERVATION_METADATA
|
|
203
|
+
if type == "observation"
|
|
204
|
+
else ProofStateOtelSpanAttributes.TRACE_METADATA
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
metadata_attributes: Dict[str, Union[str, int, None]] = {}
|
|
208
|
+
|
|
209
|
+
if not isinstance(metadata, dict):
|
|
210
|
+
metadata_attributes[prefix] = _serialize(metadata)
|
|
211
|
+
else:
|
|
212
|
+
for key, value in metadata.items():
|
|
213
|
+
metadata_attributes[f"{prefix}.{key}"] = (
|
|
214
|
+
value
|
|
215
|
+
if isinstance(value, str) or isinstance(value, int)
|
|
216
|
+
else _serialize(value)
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
return metadata_attributes
|