octostar-python-client 0.1.759__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- octostar/__init__.py +9 -0
- octostar/api/__init__.py +1 -0
- octostar/api/apps/__init__.py +0 -0
- octostar/api/apps/deploy_app.py +210 -0
- octostar/api/apps/execute_app_job.py +188 -0
- octostar/api/apps/get_app_logs.py +210 -0
- octostar/api/apps/get_apps_url.py +188 -0
- octostar/api/apps/get_job_logs.py +210 -0
- octostar/api/apps/get_job_progress.py +162 -0
- octostar/api/apps/kill_job.py +160 -0
- octostar/api/apps/list_app_jobs.py +276 -0
- octostar/api/apps/list_apps.py +251 -0
- octostar/api/apps/set_job_progress.py +216 -0
- octostar/api/apps/undeploy_app.py +160 -0
- octostar/api/metadata/__init__.py +0 -0
- octostar/api/metadata/get_version.py +232 -0
- octostar/api/metadata/get_whoami.py +232 -0
- octostar/api/notifications/__init__.py +0 -0
- octostar/api/notifications/delete_stream.py +222 -0
- octostar/api/notifications/get_subscriptions.py +240 -0
- octostar/api/notifications/publish_notification.py +275 -0
- octostar/api/notifications/pull_events_from_stream.py +282 -0
- octostar/api/notifications/push_event_to_stream.py +265 -0
- octostar/api/notifications/toast.py +264 -0
- octostar/api/ontology/__init__.py +0 -0
- octostar/api/ontology/fetch_ontology_data.py +275 -0
- octostar/api/ontology/get_ontologies.py +237 -0
- octostar/api/ontology/multi_query.py +297 -0
- octostar/api/ontology/query.py +276 -0
- octostar/api/pipeline/__init__.py +1 -0
- octostar/api/pipeline/get_processing_status.py +185 -0
- octostar/api/pipeline/update_processing_status.py +164 -0
- octostar/api/search/__init__.py +0 -0
- octostar/api/search/get_annotations.py +153 -0
- octostar/api/workspace_data/__init__.py +0 -0
- octostar/api/workspace_data/delete_blob.py +212 -0
- octostar/api/workspace_data/delete_entities.py +326 -0
- octostar/api/workspace_data/download_blob.py +235 -0
- octostar/api/workspace_data/get_attachment.py +336 -0
- octostar/api/workspace_data/get_files_tree.py +397 -0
- octostar/api/workspace_data/upload_blob.py +235 -0
- octostar/api/workspace_data/upsert_entities.py +284 -0
- octostar/api/workspace_permissions/__init__.py +0 -0
- octostar/api/workspace_permissions/get_permissions.py +325 -0
- octostar/api/workspace_tags/__init__.py +0 -0
- octostar/api/workspace_tags/delete_tag_from_entities.py +141 -0
- octostar/api/workspace_tags/tag_entities.py +180 -0
- octostar/client.py +492 -0
- octostar/errors.py +50 -0
- octostar/models/__init__.py +249 -0
- octostar/models/acknowledgement.py +74 -0
- octostar/models/acknowledgement_with_data.py +82 -0
- octostar/models/app_status.py +239 -0
- octostar/models/app_status_annotations.py +66 -0
- octostar/models/app_status_labels.py +69 -0
- octostar/models/app_with_url.py +82 -0
- octostar/models/child_processing_status.py +118 -0
- octostar/models/delete_entities_response_401.py +74 -0
- octostar/models/delete_entities_response_409.py +82 -0
- octostar/models/delete_entities_response_500.py +82 -0
- octostar/models/delete_stream_response_401.py +74 -0
- octostar/models/delete_tag_from_entities_response_401.py +74 -0
- octostar/models/deploy_app_json_body.py +90 -0
- octostar/models/deploy_app_json_body_secrets.py +65 -0
- octostar/models/deploy_app_response_200.py +98 -0
- octostar/models/deploy_app_response_200_data.py +60 -0
- octostar/models/deploy_app_response_400.py +82 -0
- octostar/models/deploy_app_response_403.py +82 -0
- octostar/models/deploy_app_response_404.py +82 -0
- octostar/models/deploy_app_response_409.py +82 -0
- octostar/models/deploy_app_response_500.py +82 -0
- octostar/models/entity.py +80 -0
- octostar/models/entity_response.py +99 -0
- octostar/models/entity_response_s3_urls.py +93 -0
- octostar/models/entity_response_s3_urls_additional_property.py +105 -0
- octostar/models/entity_response_s3_urls_additional_property_fields.py +114 -0
- octostar/models/execute_app_job_json_body.py +151 -0
- octostar/models/execute_app_job_json_body_annotation.py +65 -0
- octostar/models/execute_app_job_response_401.py +74 -0
- octostar/models/fetch_ontology_data_response_200.py +60 -0
- octostar/models/fetch_ontology_data_response_401.py +74 -0
- octostar/models/fetch_ontology_data_response_500.py +82 -0
- octostar/models/get_app_logs_response_401.py +74 -0
- octostar/models/get_app_logs_response_404.py +74 -0
- octostar/models/get_app_logs_response_500.py +82 -0
- octostar/models/get_apps_url_json_body.py +76 -0
- octostar/models/get_apps_url_response_401.py +74 -0
- octostar/models/get_apps_url_response_500.py +82 -0
- octostar/models/get_attachment_response_200.py +74 -0
- octostar/models/get_attachment_response_401.py +74 -0
- octostar/models/get_files_tree_response_200.py +106 -0
- octostar/models/get_files_tree_response_200_status.py +8 -0
- octostar/models/get_files_tree_response_400.py +111 -0
- octostar/models/get_files_tree_response_400_data.py +60 -0
- octostar/models/get_files_tree_response_400_status.py +8 -0
- octostar/models/get_files_tree_response_401.py +74 -0
- octostar/models/get_files_tree_response_500.py +111 -0
- octostar/models/get_files_tree_response_500_data.py +60 -0
- octostar/models/get_files_tree_response_500_status.py +8 -0
- octostar/models/get_job_logs_response_401.py +74 -0
- octostar/models/get_job_logs_response_404.py +74 -0
- octostar/models/get_job_logs_response_500.py +82 -0
- octostar/models/get_job_progress_response_401.py +74 -0
- octostar/models/get_object_response_401.py +74 -0
- octostar/models/get_ontologies_response_401.py +74 -0
- octostar/models/get_ontologies_response_500.py +81 -0
- octostar/models/get_permissions_response_200.py +98 -0
- octostar/models/get_permissions_response_400.py +82 -0
- octostar/models/get_permissions_response_401.py +74 -0
- octostar/models/get_permissions_response_500.py +82 -0
- octostar/models/get_processing_status_response_200.py +104 -0
- octostar/models/get_processing_status_response_200_data.py +87 -0
- octostar/models/get_processing_status_response_400.py +82 -0
- octostar/models/get_processing_status_response_500.py +82 -0
- octostar/models/get_subscriptions_response_200_item.py +74 -0
- octostar/models/get_version_response_200.py +74 -0
- octostar/models/get_version_response_404.py +74 -0
- octostar/models/get_whoami_response_200.py +129 -0
- octostar/models/get_whoami_response_401.py +74 -0
- octostar/models/insert_entity.py +114 -0
- octostar/models/insert_entity_base.py +266 -0
- octostar/models/insert_entity_relationships_item.py +107 -0
- octostar/models/insert_entity_request.py +94 -0
- octostar/models/internal_server_error.py +82 -0
- octostar/models/job_execution_result.py +146 -0
- octostar/models/job_status.py +196 -0
- octostar/models/job_status_labels.py +60 -0
- octostar/models/job_with_url.py +82 -0
- octostar/models/kill_job_response_401.py +74 -0
- octostar/models/list_app_jobs_response_401.py +74 -0
- octostar/models/list_app_jobs_response_500.py +82 -0
- octostar/models/list_apps_response_401.py +74 -0
- octostar/models/list_apps_response_500.py +82 -0
- octostar/models/multi_query_json_body.py +100 -0
- octostar/models/multi_query_json_body_queries_item.py +80 -0
- octostar/models/multi_query_response_400.py +82 -0
- octostar/models/multi_query_response_401.py +74 -0
- octostar/models/not_found_error.py +74 -0
- octostar/models/octostar_event.py +96 -0
- octostar/models/octostar_event_octostar_payload.py +100 -0
- octostar/models/octostar_event_octostar_payload_level.py +11 -0
- octostar/models/os_notification.py +122 -0
- octostar/models/processing_status.py +262 -0
- octostar/models/processing_status_code.py +14 -0
- octostar/models/progress_request.py +73 -0
- octostar/models/publish_notification_response_401.py +74 -0
- octostar/models/pull_events_from_stream_response_401.py +74 -0
- octostar/models/push_event_to_stream_response_401.py +74 -0
- octostar/models/query_json_body.py +101 -0
- octostar/models/query_json_body_params.py +60 -0
- octostar/models/query_response_400.py +82 -0
- octostar/models/query_response_401.py +74 -0
- octostar/models/set_job_progress_response_401.py +74 -0
- octostar/models/string_to_value_label_map.py +99 -0
- octostar/models/string_to_value_label_map_data.py +89 -0
- octostar/models/string_to_value_label_map_data_additional_property.py +80 -0
- octostar/models/successful_get_tags.py +103 -0
- octostar/models/successful_insertion.py +98 -0
- octostar/models/tag_entities_response_401.py +74 -0
- octostar/models/toast_level.py +11 -0
- octostar/models/toast_response_401.py +74 -0
- octostar/models/undeploy_app_response_401.py +74 -0
- octostar/models/update_processing_status_response_200.py +82 -0
- octostar/models/update_processing_status_response_400.py +82 -0
- octostar/models/update_processing_status_response_500.py +82 -0
- octostar/models/upsert_entities_response_401.py +74 -0
- octostar/models/upsert_entity.py +114 -0
- octostar/models/upsert_entity_base.py +266 -0
- octostar/models/upsert_entity_relationships_item.py +107 -0
- octostar/py.typed +1 -0
- octostar/types.py +54 -0
- octostar/utils/__init__.py +15 -0
- octostar/utils/chat/__init__.py +0 -0
- octostar/utils/chat/chat.py +513 -0
- octostar/utils/chat/detokenize.py +105 -0
- octostar/utils/chat/get_default_model.py +50 -0
- octostar/utils/chat/list_models.py +91 -0
- octostar/utils/chat/tokenize.py +105 -0
- octostar/utils/commons.py +226 -0
- octostar/utils/exceptions.py +134 -0
- octostar/utils/jobs/__init__.py +0 -0
- octostar/utils/jobs/apps/__init__.py +0 -0
- octostar/utils/jobs/apps/deploy_app.py +81 -0
- octostar/utils/jobs/apps/execute_app_job.py +114 -0
- octostar/utils/jobs/apps/get_app_logs.py +113 -0
- octostar/utils/jobs/apps/get_app_secret.py +102 -0
- octostar/utils/jobs/apps/get_apps_url.py +73 -0
- octostar/utils/jobs/apps/list_app_jobs.py +62 -0
- octostar/utils/jobs/apps/list_apps.py +126 -0
- octostar/utils/jobs/apps/undeploy_app.py +48 -0
- octostar/utils/jobs/get_job_logs.py +113 -0
- octostar/utils/jobs/get_job_progress.py +76 -0
- octostar/utils/jobs/kill_job.py +47 -0
- octostar/utils/jobs/set_job_progress.py +67 -0
- octostar/utils/meta/__init__.py +0 -0
- octostar/utils/meta/get_version.py +30 -0
- octostar/utils/meta/get_whoami.py +30 -0
- octostar/utils/notifications/__init__.py +0 -0
- octostar/utils/notifications/delete_stream.py +58 -0
- octostar/utils/notifications/get_my_subscriptions.py +49 -0
- octostar/utils/notifications/publish_notification.py +73 -0
- octostar/utils/notifications/pull_event_from_stream.py +63 -0
- octostar/utils/notifications/pull_events_from_stream.py +64 -0
- octostar/utils/notifications/push_event_to_stream.py +109 -0
- octostar/utils/notifications/push_events_to_stream.py +137 -0
- octostar/utils/notifications/toast.py +92 -0
- octostar/utils/ontology/__init__.py +10 -0
- octostar/utils/ontology/fetch_ontology_data.py +141 -0
- octostar/utils/ontology/get_ontologies.py +55 -0
- octostar/utils/ontology/multiquery_ontology.py +287 -0
- octostar/utils/ontology/query_ontology.py +186 -0
- octostar/utils/pipeline/__init__.py +1 -0
- octostar/utils/pipeline/get_processing_status.py +230 -0
- octostar/utils/pipeline/update_processing_status.py +286 -0
- octostar/utils/search/__init__.py +11 -0
- octostar/utils/search/bulk_update.py +138 -0
- octostar/utils/search/count.py +117 -0
- octostar/utils/search/get_entity_annotations.py +304 -0
- octostar/utils/search/get_index_definition.py +111 -0
- octostar/utils/search/multi_search.py +129 -0
- octostar/utils/workspace/__init__.py +0 -0
- octostar/utils/workspace/delete_entities.py +247 -0
- octostar/utils/workspace/delete_entity.py +81 -0
- octostar/utils/workspace/delete_relationship.py +78 -0
- octostar/utils/workspace/delete_relationships.py +85 -0
- octostar/utils/workspace/delete_temporary_blob.py +85 -0
- octostar/utils/workspace/extract_entities.py +140 -0
- octostar/utils/workspace/get_filepath_from_item.py +85 -0
- octostar/utils/workspace/get_filepaths_from_items.py +100 -0
- octostar/utils/workspace/get_files_tree.py +102 -0
- octostar/utils/workspace/get_item_from_filepath.py +102 -0
- octostar/utils/workspace/get_items_from_filepaths.py +108 -0
- octostar/utils/workspace/linkcharts/__init__.py +0 -0
- octostar/utils/workspace/linkcharts/create_linkchart.py +241 -0
- octostar/utils/workspace/permissions/PermissionLevel.py +8 -0
- octostar/utils/workspace/permissions/__init__.py +1 -0
- octostar/utils/workspace/permissions/get_permissions.py +81 -0
- octostar/utils/workspace/read_attachment.py +284 -0
- octostar/utils/workspace/read_file.py +113 -0
- octostar/utils/workspace/read_temporary_blob.py +428 -0
- octostar/utils/workspace/saved_searches/__init__.py +0 -0
- octostar/utils/workspace/saved_searches/create_saved_search.py +183 -0
- octostar/utils/workspace/tags/__init__.py +0 -0
- octostar/utils/workspace/tags/delete_tag_from_entities.py +96 -0
- octostar/utils/workspace/tags/tag_entities.py +175 -0
- octostar/utils/workspace/upsert_entities.py +268 -0
- octostar/utils/workspace/upsert_entity.py +110 -0
- octostar/utils/workspace/upsert_relationship.py +128 -0
- octostar/utils/workspace/upsert_relationships.py +194 -0
- octostar/utils/workspace/write_attachment.py +263 -0
- octostar/utils/workspace/write_file.py +335 -0
- octostar/utils/workspace/write_temporary_blob.py +218 -0
- octostar_python_client-0.1.759.dist-info/METADATA +159 -0
- octostar_python_client-0.1.759.dist-info/RECORD +257 -0
- octostar_python_client-0.1.759.dist-info/WHEEL +5 -0
- octostar_python_client-0.1.759.dist-info/licenses/LICENSE +21 -0
- octostar_python_client-0.1.759.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Get processing status utility functions."""
|
|
2
|
+
|
|
3
|
+
from typing import List, Dict, Any, Union
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
_logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
from ...api.pipeline import get_processing_status as api_get_processing_status
|
|
9
|
+
from ...client import Client
|
|
10
|
+
from ..exceptions import ApiConnectionError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def sync(
|
|
14
|
+
entity_ids: List[str],
|
|
15
|
+
client: Client = None,
|
|
16
|
+
) -> Dict[str, Union[Dict[str, Any], None]]:
|
|
17
|
+
"""
|
|
18
|
+
Retrieve the processing status for a list of entities.
|
|
19
|
+
|
|
20
|
+
.. note::
|
|
21
|
+
|
|
22
|
+
**Trust contract.** The persisted snapshot is fully trustworthy
|
|
23
|
+
only when ``status_code`` is **``COMPLETED``** or **``FAILED``**
|
|
24
|
+
(non-stale terminal states). Other codes carry caveats:
|
|
25
|
+
|
|
26
|
+
- ``RUNNING`` / ``UNKNOWN``: the pipeline is still in flight; the
|
|
27
|
+
data may be catching up.
|
|
28
|
+
- ``STALE``: terminal but signals an *unsettled* prior run -- the
|
|
29
|
+
redeploy cleanup transitions stuck-partial roots to ``STALE``
|
|
30
|
+
so a fresh run can reprocess them; whatever data
|
|
31
|
+
is attached reflects whatever the previous run managed to
|
|
32
|
+
write before being torn down, not a settled processing status.
|
|
33
|
+
- ``PARTIALLY_COMPLETED``: Sparse override
|
|
34
|
+
at ``detailed_status.status_code`` whenever child fragment
|
|
35
|
+
(``tree_live`` or ``remaining_floor > 0``); when the override is
|
|
36
|
+
absent, callers should treat the parent's ``status_code`` as the
|
|
37
|
+
effective value:
|
|
38
|
+
``detailed_status.status_code or status_code``.
|
|
39
|
+
|
|
40
|
+
Callers that need a definitive answer (e.g. "is this entity
|
|
41
|
+
finished?") should gate on
|
|
42
|
+
``status_code in {"COMPLETED", "FAILED"} plus detailed_status lookup``.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
entity_ids: List of entity UIDs to get processing status for.
|
|
46
|
+
client: The Client with which to connect to Octostar. If None, the default is used.
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
A dictionary mapping entity IDs to their processing status information.
|
|
50
|
+
Each status mirrors the backend's stored ``processing_status``
|
|
51
|
+
document and may contain:
|
|
52
|
+
|
|
53
|
+
- ``entity_id``: The entity UID.
|
|
54
|
+
- ``entity_type``: The entity type/concept.
|
|
55
|
+
- ``entity_label``: Human-readable label.
|
|
56
|
+
- ``status_code``: "RUNNING", "COMPLETED", "FAILED", "STALE", or
|
|
57
|
+
"UNKNOWN".
|
|
58
|
+
- ``reason``: Reason for the status.
|
|
59
|
+
- ``timestamp``: When the status was last updated.
|
|
60
|
+
- ``do_not_reindex_before``: ISO 8601 timestamp guard for reindex.
|
|
61
|
+
- ``do_not_update_before``: ISO 8601 timestamp; older incoming
|
|
62
|
+
writes are silently nooped.
|
|
63
|
+
- ``timeout``: Redis cache TTL in seconds.
|
|
64
|
+
- ``n_child_fragments``: Number of direct child fragments declared.
|
|
65
|
+
- ``source_entity_uid``: UID of the global root (``None`` for
|
|
66
|
+
global roots).
|
|
67
|
+
- ``os_parent_uid``: UID of the direct parent (``None`` for global
|
|
68
|
+
roots).
|
|
69
|
+
- ``os_workspace``: UID of the workspace.
|
|
70
|
+
- ``fragment_errors``: List of child error dicts.
|
|
71
|
+
- ``fragment_warnings``: List of child warning dicts.
|
|
72
|
+
- ``warnings``: List of entity-level warning dicts.
|
|
73
|
+
- ``detailed_status``: Persisted recursive-rollup payload. Always
|
|
74
|
+
returned in-line for entities the debounced rollup job has
|
|
75
|
+
visited. Fields:
|
|
76
|
+
|
|
77
|
+
- ``total_indexed`` (int): docs visited by the aggregation.
|
|
78
|
+
- ``total_terminal`` (int): visited docs in a terminal status.
|
|
79
|
+
- ``total_running`` (int): visited docs not yet terminal.
|
|
80
|
+
- ``total_missing`` (int): sum over visited parents of
|
|
81
|
+
``max(0, n_child_fragments - actual_children_indexed)``;
|
|
82
|
+
counts children declared by a parent but never indexed.
|
|
83
|
+
- ``total_failed`` (int): visited docs in a terminal status that
|
|
84
|
+
isn't COMPLETED (i.e. FAILED or STALE).
|
|
85
|
+
- ``known_floor`` (int): lower bound on eventual tree size.
|
|
86
|
+
- ``remaining_floor`` (int): ``max(0, known_floor - total_terminal)``.
|
|
87
|
+
- ``fully_complete`` (bool): fixpoint completion on the visited snapshot.
|
|
88
|
+
- ``tree_live`` (bool): at least one non-terminal node present.
|
|
89
|
+
- ``has_missing_branches`` (bool): orphan children or declared shortfall.
|
|
90
|
+
- ``partial`` (bool): visit cap hit or query error; annotates trust and
|
|
91
|
+
does NOT veto ``fully_complete``.
|
|
92
|
+
- ``status_code`` (str, optional): sparse rollup-effective status
|
|
93
|
+
override. Set only when it differs from the parent's
|
|
94
|
+
``status_code`` -- in practice, only ``"PARTIALLY_COMPLETED"``
|
|
95
|
+
ever appears here (when the rollup is unsettled for a
|
|
96
|
+
``COMPLETED`` parent). Consumers should compute the effective
|
|
97
|
+
status as
|
|
98
|
+
``detailed_status.status_code or status_code`` so a missing
|
|
99
|
+
override falls through to the parent value.
|
|
100
|
+
|
|
101
|
+
- ``detailed_status_hash``: SHA-256 of ``detailed_status``.
|
|
102
|
+
|
|
103
|
+
If an entity has no processing status, its value will be None.
|
|
104
|
+
|
|
105
|
+
Raises:
|
|
106
|
+
ApiConnectionError: If the request was unsuccessful on the server.
|
|
107
|
+
"""
|
|
108
|
+
response = api_get_processing_status.sync_detailed(
|
|
109
|
+
entity_ids=entity_ids, client=client
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
if response.status_code != 200:
|
|
113
|
+
raise ApiConnectionError("get_processing_status", response, client)
|
|
114
|
+
|
|
115
|
+
parsed = response.parsed
|
|
116
|
+
if parsed is None:
|
|
117
|
+
return {eid: None for eid in entity_ids}
|
|
118
|
+
|
|
119
|
+
return parsed.get("data", {})
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
async def asyncio(
|
|
123
|
+
entity_ids: List[str],
|
|
124
|
+
client: Client = None,
|
|
125
|
+
) -> Dict[str, Union[Dict[str, Any], None]]:
|
|
126
|
+
"""
|
|
127
|
+
Retrieve the processing status for a list of entities (async).
|
|
128
|
+
|
|
129
|
+
.. note::
|
|
130
|
+
|
|
131
|
+
**Trust contract.** The persisted snapshot is fully trustworthy
|
|
132
|
+
only when ``status_code`` is **``COMPLETED``** or **``FAILED``**
|
|
133
|
+
(non-stale terminal states). Other codes carry caveats:
|
|
134
|
+
|
|
135
|
+
- ``RUNNING`` / ``UNKNOWN``: the pipeline is still in flight; the
|
|
136
|
+
data may be catching up.
|
|
137
|
+
- ``STALE``: terminal but signals an *unsettled* prior run -- the
|
|
138
|
+
redeploy cleanup transitions stuck-partial roots to ``STALE``
|
|
139
|
+
so a fresh run can reprocess them; whatever data
|
|
140
|
+
is attached reflects whatever the previous run managed to
|
|
141
|
+
write before being torn down, not a settled processing status.
|
|
142
|
+
- ``PARTIALLY_COMPLETED``: synthetic value. It never appears as the
|
|
143
|
+
parent ``status_code`` (the doc is stored as ``COMPLETED`` with an
|
|
144
|
+
unsettled ``detailed_status``). It does appear as a sparse override
|
|
145
|
+
at ``detailed_status.status_code`` whenever the rollup is unsettled
|
|
146
|
+
(``tree_live`` or ``remaining_floor > 0``); when the override is
|
|
147
|
+
absent, callers should treat the parent's ``status_code`` as the
|
|
148
|
+
effective value:
|
|
149
|
+
``detailed_status.status_code or status_code``.
|
|
150
|
+
|
|
151
|
+
Callers that need a definitive answer (e.g. "is this entity
|
|
152
|
+
finished?") should gate on
|
|
153
|
+
``status_code in {"COMPLETED", "FAILED"} plus detailed_status lookup``.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
entity_ids: List of entity UIDs to get processing status for.
|
|
157
|
+
client: The Client with which to connect to Octostar. If None, the default is used.
|
|
158
|
+
|
|
159
|
+
Returns:
|
|
160
|
+
A dictionary mapping entity IDs to their processing status information.
|
|
161
|
+
Each status mirrors the backend's stored ``processing_status``
|
|
162
|
+
document and may contain:
|
|
163
|
+
|
|
164
|
+
- ``entity_id``: The entity UID.
|
|
165
|
+
- ``entity_type``: The entity type/concept.
|
|
166
|
+
- ``entity_label``: Human-readable label.
|
|
167
|
+
- ``status_code``: "RUNNING", "COMPLETED", "FAILED", "STALE", or
|
|
168
|
+
"UNKNOWN".
|
|
169
|
+
- ``reason``: Reason for the status.
|
|
170
|
+
- ``timestamp``: When the status was last updated.
|
|
171
|
+
- ``do_not_reindex_before``: ISO 8601 timestamp guard for reindex.
|
|
172
|
+
- ``do_not_update_before``: ISO 8601 timestamp; older incoming
|
|
173
|
+
writes are silently nooped.
|
|
174
|
+
- ``timeout``: Redis cache TTL in seconds.
|
|
175
|
+
- ``n_child_fragments``: Number of direct child fragments declared.
|
|
176
|
+
- ``source_entity_uid``: UID of the global root (``None`` for
|
|
177
|
+
global roots).
|
|
178
|
+
- ``os_parent_uid``: UID of the direct parent (``None`` for global
|
|
179
|
+
roots).
|
|
180
|
+
- ``os_workspace``: UID of the workspace.
|
|
181
|
+
- ``fragment_errors``: List of child error dicts.
|
|
182
|
+
- ``fragment_warnings``: List of child warning dicts.
|
|
183
|
+
- ``warnings``: List of entity-level warning dicts.
|
|
184
|
+
- ``detailed_status``: Persisted recursive-rollup payload. Always
|
|
185
|
+
returned in-line for entities the debounced rollup job has
|
|
186
|
+
visited. Fields:
|
|
187
|
+
|
|
188
|
+
- ``total_indexed`` (int): docs visited by the aggregation.
|
|
189
|
+
- ``total_terminal`` (int): visited docs in a terminal status.
|
|
190
|
+
- ``total_running`` (int): visited docs not yet terminal.
|
|
191
|
+
- ``total_missing`` (int): sum over visited parents of
|
|
192
|
+
``max(0, n_child_fragments - actual_children_indexed)``;
|
|
193
|
+
counts children declared by a parent but never indexed.
|
|
194
|
+
- ``total_failed`` (int): visited docs in a terminal status that
|
|
195
|
+
isn't COMPLETED (i.e. FAILED or STALE).
|
|
196
|
+
- ``known_floor`` (int): lower bound on eventual tree size.
|
|
197
|
+
- ``remaining_floor`` (int): ``max(0, known_floor - total_terminal)``.
|
|
198
|
+
- ``fully_complete`` (bool): fixpoint completion on the visited snapshot.
|
|
199
|
+
- ``tree_live`` (bool): at least one non-terminal node present.
|
|
200
|
+
- ``has_missing_branches`` (bool): orphan children or declared shortfall.
|
|
201
|
+
- ``partial`` (bool): visit cap hit or query error; annotates trust and
|
|
202
|
+
does NOT veto ``fully_complete``.
|
|
203
|
+
- ``status_code`` (str, optional): sparse rollup-effective status
|
|
204
|
+
override. Set only when it differs from the parent's
|
|
205
|
+
``status_code`` -- in practice, only ``"PARTIALLY_COMPLETED"``
|
|
206
|
+
ever appears here (when the rollup is unsettled for a
|
|
207
|
+
``COMPLETED`` parent). Consumers should compute the effective
|
|
208
|
+
status as
|
|
209
|
+
``detailed_status.status_code or status_code`` so a missing
|
|
210
|
+
override falls through to the parent value.
|
|
211
|
+
|
|
212
|
+
- ``detailed_status_hash``: SHA-256 of ``detailed_status``.
|
|
213
|
+
|
|
214
|
+
If an entity has no processing status, its value will be None.
|
|
215
|
+
|
|
216
|
+
Raises:
|
|
217
|
+
ApiConnectionError: If the request was unsuccessful on the server.
|
|
218
|
+
"""
|
|
219
|
+
response = await api_get_processing_status.asyncio_detailed(
|
|
220
|
+
entity_ids=entity_ids, client=client
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
if response.status_code != 200:
|
|
224
|
+
raise ApiConnectionError("get_processing_status", response, client)
|
|
225
|
+
|
|
226
|
+
parsed = response.parsed
|
|
227
|
+
if parsed is None:
|
|
228
|
+
return {eid: None for eid in entity_ids}
|
|
229
|
+
|
|
230
|
+
return parsed.get("data", {})
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"""Update processing status utility functions."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import Any, Dict, List, Optional, TypedDict, Union
|
|
5
|
+
|
|
6
|
+
_logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
from ...api.pipeline import update_processing_status as api_update_processing_status
|
|
9
|
+
from ...client import Client
|
|
10
|
+
from ..exceptions import ApiConnectionError
|
|
11
|
+
|
|
12
|
+
APPEND_TOKEN = "$append"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ChildProcessingStatusUpdate(TypedDict, total=False):
|
|
16
|
+
"""
|
|
17
|
+
Type definition for a child entry inside ``fragment_errors``,
|
|
18
|
+
``fragment_warnings``, or ``warnings``.
|
|
19
|
+
|
|
20
|
+
All fields are optional to support cases where the child entity cannot
|
|
21
|
+
be fully identified (e.g. missing fragments known only by index).
|
|
22
|
+
|
|
23
|
+
Fields:
|
|
24
|
+
- ``entity_id``: The child entity UID.
|
|
25
|
+
- ``entity_type``: The child entity type/concept.
|
|
26
|
+
- ``entity_label``: Human-readable label for the child entity.
|
|
27
|
+
- ``status_code``: e.g. "RUNNING", "COMPLETED", "FAILED", "STALE", "UNKNOWN".
|
|
28
|
+
- ``reason``: Reason for the status.
|
|
29
|
+
- ``timestamp``: ISO 8601 timestamp of when the entry was produced.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
entity_id: Optional[str]
|
|
33
|
+
entity_type: Optional[str]
|
|
34
|
+
entity_label: Optional[str]
|
|
35
|
+
status_code: Optional[str]
|
|
36
|
+
reason: Optional[str]
|
|
37
|
+
timestamp: Optional[str]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class _ProcessingStatusUpdateRequired(TypedDict):
|
|
41
|
+
"""Required fields for processing status updates."""
|
|
42
|
+
|
|
43
|
+
entity_id: str
|
|
44
|
+
entity_type: str
|
|
45
|
+
do_not_update_before: str
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ProcessingStatusUpdate(_ProcessingStatusUpdateRequired, total=False):
|
|
49
|
+
"""
|
|
50
|
+
Type definition for processing status updates.
|
|
51
|
+
|
|
52
|
+
Mirrors the backend's ``ProcessingStatus`` Pydantic model. The endpoint
|
|
53
|
+
performs partial updates: only the keys you set are written, all other
|
|
54
|
+
persisted fields are preserved.
|
|
55
|
+
|
|
56
|
+
Required fields on every write:
|
|
57
|
+
|
|
58
|
+
- ``entity_id``: The entity UID.
|
|
59
|
+
- ``entity_type``: The entity type/concept.
|
|
60
|
+
- ``do_not_update_before``: ISO 8601 timestamp guard. If the incoming
|
|
61
|
+
value is older than the stored one, the entire update for this
|
|
62
|
+
entity is silently nooped.
|
|
63
|
+
|
|
64
|
+
Recommended on every status write:
|
|
65
|
+
|
|
66
|
+
- ``os_workspace``: UID of the workspace this entity belongs to. The
|
|
67
|
+
per-workspace stats aggregation buckets by this field, so omitting
|
|
68
|
+
it strands the entry from those rollups.
|
|
69
|
+
|
|
70
|
+
Required on every fragment write (i.e. any non-root entity):
|
|
71
|
+
|
|
72
|
+
- ``source_entity_uid``: UID of the global root the fragment belongs
|
|
73
|
+
to. The recursive rollup scopes its tree fetch by this field, so
|
|
74
|
+
omitting it strands the fragment outside its parent's rollup view.
|
|
75
|
+
- ``os_parent_uid``: UID of the direct parent in the fragment tree.
|
|
76
|
+
Pair with ``source_entity_uid``. Root entities leave both as
|
|
77
|
+
``None`` (the default).
|
|
78
|
+
|
|
79
|
+
Other optional fields:
|
|
80
|
+
|
|
81
|
+
- ``entity_label``: Human-readable label.
|
|
82
|
+
- ``status_code``: "RUNNING", "COMPLETED", "FAILED", "STALE", or
|
|
83
|
+
"UNKNOWN" (default for new entities; preserved on partial updates
|
|
84
|
+
that omit it).
|
|
85
|
+
- ``reason``: Reason for the status.
|
|
86
|
+
- ``timestamp``: ISO 8601 timestamp annotation.
|
|
87
|
+
- ``do_not_reindex_before``: ISO 8601 timestamp guard for the
|
|
88
|
+
search-side reindex.
|
|
89
|
+
- ``timeout``: Redis cache TTL in seconds (default: 5 minutes).
|
|
90
|
+
- ``n_child_fragments``: Number of direct child fragments this entity
|
|
91
|
+
declares; used by the recursive rollup to detect missing branches.
|
|
92
|
+
|
|
93
|
+
Appendable list fields -- ``fragment_errors``, ``fragment_warnings``,
|
|
94
|
+
``warnings`` -- share identical semantics:
|
|
95
|
+
|
|
96
|
+
- ``["$append", item1, item2]`` -> append items to the existing list
|
|
97
|
+
- ``[item1, item2]`` -> replace the entire list
|
|
98
|
+
- ``[]`` or ``None`` -> clear the list
|
|
99
|
+
- omitted -> no change
|
|
100
|
+
|
|
101
|
+
Each item is a ``ChildProcessingStatusUpdate`` (or plain dict with the
|
|
102
|
+
same shape). Example:
|
|
103
|
+
``{"fragment_errors": ["$append", {"entity_id": "c1", "reason": "bad"}]}``.
|
|
104
|
+
|
|
105
|
+
**Trust contract on read.** The persisted snapshot returned by
|
|
106
|
+
``get_processing_status`` is fully trustworthy only when
|
|
107
|
+
``status_code`` is **``COMPLETED``** or **``FAILED``** (non-stale
|
|
108
|
+
terminal states). ``STALE`` is also terminal but signals an
|
|
109
|
+
*unsettled* prior run -- the redeploy cleanup transitions
|
|
110
|
+
stuck-partial roots to ``STALE`` so a fresh run can reprocess
|
|
111
|
+
them, and whatever data is attached reflects whatever
|
|
112
|
+
the previous run managed to write before being torn down. Callers
|
|
113
|
+
that need a definitive answer (e.g. "is this entity finished?")
|
|
114
|
+
should gate on ``status_code in {"COMPLETED", "FAILED"} + detailed_status``; treating
|
|
115
|
+
``STALE`` as authoritative will silently surface stale data.
|
|
116
|
+
|
|
117
|
+
Internal fields (normally written by the backend rollup job, not by
|
|
118
|
+
callers):
|
|
119
|
+
|
|
120
|
+
- ``detailed_status``: Persisted recursive-rollup payload.
|
|
121
|
+
- ``detailed_status_hash``: SHA-256 used by the scripted update to
|
|
122
|
+
short-circuit no-op writes.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
entity_label: Optional[str]
|
|
126
|
+
status_code: Optional[str]
|
|
127
|
+
reason: Optional[str]
|
|
128
|
+
timestamp: Optional[str]
|
|
129
|
+
do_not_reindex_before: Optional[str]
|
|
130
|
+
timeout: Optional[int]
|
|
131
|
+
n_child_fragments: Optional[int]
|
|
132
|
+
source_entity_uid: Optional[str]
|
|
133
|
+
os_parent_uid: Optional[str]
|
|
134
|
+
os_workspace: Optional[str]
|
|
135
|
+
fragment_errors: Optional[
|
|
136
|
+
List[Union[str, ChildProcessingStatusUpdate, Dict[str, Any]]]
|
|
137
|
+
]
|
|
138
|
+
fragment_warnings: Optional[
|
|
139
|
+
List[Union[str, ChildProcessingStatusUpdate, Dict[str, Any]]]
|
|
140
|
+
]
|
|
141
|
+
warnings: Optional[List[Union[str, ChildProcessingStatusUpdate, Dict[str, Any]]]]
|
|
142
|
+
detailed_status: Optional[Dict[str, Any]]
|
|
143
|
+
detailed_status_hash: Optional[str]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
_REQUIRED_KEYS = ("entity_id", "entity_type", "do_not_update_before")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _validate(statuses: List[Union[ProcessingStatusUpdate, Dict[str, Any]]]) -> None:
|
|
150
|
+
for status in statuses:
|
|
151
|
+
for key in _REQUIRED_KEYS:
|
|
152
|
+
if key not in status:
|
|
153
|
+
raise ValueError(f"Each status must have a '{key}' field")
|
|
154
|
+
value = status[key] # type: ignore[literal-required]
|
|
155
|
+
if not isinstance(value, str) or not value.strip():
|
|
156
|
+
raise ValueError(f"Each status's '{key}' must be a non-empty string")
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def sync(
|
|
160
|
+
statuses: List[Union[ProcessingStatusUpdate, Dict[str, Any]]],
|
|
161
|
+
client: Client = None,
|
|
162
|
+
) -> Dict[str, Any]:
|
|
163
|
+
"""
|
|
164
|
+
Update the processing status for a list of entities.
|
|
165
|
+
|
|
166
|
+
Only provided fields are updated; existing fields are preserved unless
|
|
167
|
+
explicitly set (e.g. an empty list ``[]`` clears an appendable list,
|
|
168
|
+
``None`` clears a scalar where the backend allows it).
|
|
169
|
+
|
|
170
|
+
Args:
|
|
171
|
+
statuses: List of status updates. See :class:`ProcessingStatusUpdate`
|
|
172
|
+
for the full set of supported keys, including the ``$append``
|
|
173
|
+
semantics for ``fragment_errors`` / ``fragment_warnings`` /
|
|
174
|
+
``warnings`` and the fragment-write contract
|
|
175
|
+
(``source_entity_uid`` + ``os_parent_uid`` required for any
|
|
176
|
+
non-root entity). At minimum each dict must contain:
|
|
177
|
+
|
|
178
|
+
- ``entity_id`` (str): The entity UID.
|
|
179
|
+
- ``entity_type`` (str): The entity type/concept.
|
|
180
|
+
- ``do_not_update_before`` (str): ISO 8601 timestamp guard;
|
|
181
|
+
older incoming values cause the whole entry to be nooped.
|
|
182
|
+
|
|
183
|
+
See also :func:`get_processing_status` for the trust
|
|
184
|
+
contract on what reads back as authoritative
|
|
185
|
+
(``COMPLETED`` / ``FAILED`` only -- ``STALE`` is terminal
|
|
186
|
+
but signals an unsettled prior run).
|
|
187
|
+
client: The Client with which to connect to Octostar. If None, the default is used.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
Response dict with ``status``, ``message``, ``count``,
|
|
191
|
+
``skipped_entity_ids``, and ``errors``. The ``errors`` list
|
|
192
|
+
contains per-entity dicts with ``entity_id``, ``type``, and
|
|
193
|
+
``detail`` for any entities that failed.
|
|
194
|
+
|
|
195
|
+
Raises:
|
|
196
|
+
ApiConnectionError: If the request was unsuccessful.
|
|
197
|
+
ValueError: If any status is missing one of the required
|
|
198
|
+
identity fields (``entity_id``, ``entity_type``,
|
|
199
|
+
``do_not_update_before``) or sets it to an empty string.
|
|
200
|
+
"""
|
|
201
|
+
if not statuses:
|
|
202
|
+
return {
|
|
203
|
+
"status": "success",
|
|
204
|
+
"message": "No statuses to update",
|
|
205
|
+
"count": 0,
|
|
206
|
+
"skipped_entity_ids": [],
|
|
207
|
+
"errors": [],
|
|
208
|
+
}
|
|
209
|
+
_validate(statuses)
|
|
210
|
+
json_body = [dict(s) for s in statuses]
|
|
211
|
+
response = api_update_processing_status.sync_detailed(
|
|
212
|
+
json_body=json_body, client=client
|
|
213
|
+
)
|
|
214
|
+
if response.status_code != 200:
|
|
215
|
+
raise ApiConnectionError("update_processing_status", response, client)
|
|
216
|
+
return response.parsed or {
|
|
217
|
+
"status": "success",
|
|
218
|
+
"count": len(statuses),
|
|
219
|
+
"skipped_entity_ids": [],
|
|
220
|
+
"errors": [],
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
async def asyncio(
|
|
225
|
+
statuses: List[Union[ProcessingStatusUpdate, Dict[str, Any]]],
|
|
226
|
+
client: Client = None,
|
|
227
|
+
) -> Dict[str, Any]:
|
|
228
|
+
"""
|
|
229
|
+
Update the processing status for a list of entities (async).
|
|
230
|
+
|
|
231
|
+
Only provided fields are updated; existing fields are preserved unless
|
|
232
|
+
explicitly set (e.g. an empty list ``[]`` clears an appendable list,
|
|
233
|
+
``None`` clears a scalar where the backend allows it).
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
statuses: List of status updates. See :class:`ProcessingStatusUpdate`
|
|
237
|
+
for the full set of supported keys, including the ``$append``
|
|
238
|
+
semantics for ``fragment_errors`` / ``fragment_warnings`` /
|
|
239
|
+
``warnings`` and the fragment-write contract
|
|
240
|
+
(``source_entity_uid`` + ``os_parent_uid`` required for any
|
|
241
|
+
non-root entity). At minimum each dict must contain:
|
|
242
|
+
|
|
243
|
+
- ``entity_id`` (str): The entity UID.
|
|
244
|
+
- ``entity_type`` (str): The entity type/concept.
|
|
245
|
+
- ``do_not_update_before`` (str): ISO 8601 timestamp guard;
|
|
246
|
+
older incoming values cause the whole entry to be nooped.
|
|
247
|
+
|
|
248
|
+
See also :func:`get_processing_status` for the trust
|
|
249
|
+
contract on what reads back as authoritative
|
|
250
|
+
(``COMPLETED`` / ``FAILED`` only -- ``STALE`` is terminal
|
|
251
|
+
but signals an unsettled prior run).
|
|
252
|
+
client: The Client with which to connect to Octostar. If None, the default is used.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
Response dict with ``status``, ``message``, ``count``,
|
|
256
|
+
``skipped_entity_ids``, and ``errors``. The ``errors`` list
|
|
257
|
+
contains per-entity dicts with ``entity_id``, ``type``, and
|
|
258
|
+
``detail`` for any entities that failed.
|
|
259
|
+
|
|
260
|
+
Raises:
|
|
261
|
+
ApiConnectionError: If the request was unsuccessful.
|
|
262
|
+
ValueError: If any status is missing one of the required
|
|
263
|
+
identity fields (``entity_id``, ``entity_type``,
|
|
264
|
+
``do_not_update_before``) or sets it to an empty string.
|
|
265
|
+
"""
|
|
266
|
+
if not statuses:
|
|
267
|
+
return {
|
|
268
|
+
"status": "success",
|
|
269
|
+
"message": "No statuses to update",
|
|
270
|
+
"count": 0,
|
|
271
|
+
"skipped_entity_ids": [],
|
|
272
|
+
"errors": [],
|
|
273
|
+
}
|
|
274
|
+
_validate(statuses)
|
|
275
|
+
json_body = [dict(s) for s in statuses]
|
|
276
|
+
response = await api_update_processing_status.asyncio_detailed(
|
|
277
|
+
json_body=json_body, client=client
|
|
278
|
+
)
|
|
279
|
+
if response.status_code != 200:
|
|
280
|
+
raise ApiConnectionError("update_processing_status", response, client)
|
|
281
|
+
return response.parsed or {
|
|
282
|
+
"status": "success",
|
|
283
|
+
"count": len(statuses),
|
|
284
|
+
"skipped_entity_ids": [],
|
|
285
|
+
"errors": [],
|
|
286
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""
|
|
2
|
+
# General queries against the search engine.
|
|
3
|
+
Use this to query the search engine for data. Do note that the search engine can find entities by
|
|
4
|
+
keywords, full text search, vector search (e.g. faces, rag), which are features not available in the ontology.
|
|
5
|
+
|
|
6
|
+
Example of a query:
|
|
7
|
+
```sql
|
|
8
|
+
{ "query": { "bool": { "must": [{ "terms": { "_id": [your_ids] } }] } }, "_source": ["annotations"] }
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
"""
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Scripted bulk update utility -- POST /internal/opensearch/_bulk_update
|
|
2
|
+
|
|
3
|
+
Send arbitrary Painless scripted updates to OpenSearch documents through
|
|
4
|
+
the octostar-api. When ``do_not_update_before`` is provided, the server
|
|
5
|
+
wraps the script with a guard that noops stale updates.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import httpx
|
|
9
|
+
from typing import Any, Dict, List, Optional, TypedDict, Union
|
|
10
|
+
|
|
11
|
+
from ...client import Client, get_default_client
|
|
12
|
+
from ..commons import network_retry_strategy
|
|
13
|
+
from ..exceptions import ApiConnectionError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ScriptedUpdateDict(TypedDict, total=False):
|
|
17
|
+
entity_id: str # required
|
|
18
|
+
script_source: str # required
|
|
19
|
+
script_params: Dict[str, Any]
|
|
20
|
+
do_not_update_before: str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def sync(
|
|
24
|
+
updates: List[Union[ScriptedUpdateDict, Dict[str, Any]]],
|
|
25
|
+
timeout: Optional[float] = 60.0,
|
|
26
|
+
client: Optional[Client] = None,
|
|
27
|
+
) -> Dict[str, Any]:
|
|
28
|
+
"""Send scripted bulk updates to OpenSearch through octostar-api.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
updates: List of update dicts. Each must contain ``entity_id`` and
|
|
32
|
+
``script_source``. Optionally ``script_params`` and
|
|
33
|
+
``do_not_update_before``.
|
|
34
|
+
timeout: HTTP timeout in seconds.
|
|
35
|
+
client: Octostar client. Falls back to the default client.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
Dict with ``status``, ``updated``, ``nooped_entity_ids``, ``errors``.
|
|
39
|
+
|
|
40
|
+
Raises:
|
|
41
|
+
ApiConnectionError: If the request fails.
|
|
42
|
+
"""
|
|
43
|
+
if not updates:
|
|
44
|
+
return {
|
|
45
|
+
"status": "success",
|
|
46
|
+
"updated": 0,
|
|
47
|
+
"nooped_entity_ids": [],
|
|
48
|
+
"errors": [],
|
|
49
|
+
}
|
|
50
|
+
if not client:
|
|
51
|
+
client = get_default_client()
|
|
52
|
+
|
|
53
|
+
endpoint_url = f"{client.get_base_url_v1()}/internal/opensearch/_bulk_update"
|
|
54
|
+
headers = {
|
|
55
|
+
"Content-Type": "application/json",
|
|
56
|
+
"Authorization": f"Bearer {client.token}",
|
|
57
|
+
"x-ontology": client.ontology,
|
|
58
|
+
}
|
|
59
|
+
json_body = [dict(u) for u in updates]
|
|
60
|
+
|
|
61
|
+
response = None
|
|
62
|
+
try:
|
|
63
|
+
for attempt in network_retry_strategy():
|
|
64
|
+
with attempt:
|
|
65
|
+
with httpx.Client() as httpx_client:
|
|
66
|
+
response = httpx_client.post(
|
|
67
|
+
endpoint_url,
|
|
68
|
+
json=json_body,
|
|
69
|
+
headers=headers,
|
|
70
|
+
timeout=timeout,
|
|
71
|
+
)
|
|
72
|
+
if response.status_code != 200:
|
|
73
|
+
raise ConnectionError(
|
|
74
|
+
f"bulk_update failed (status {response.status_code}): "
|
|
75
|
+
+ str(response.content)
|
|
76
|
+
)
|
|
77
|
+
return response.json()
|
|
78
|
+
except Exception:
|
|
79
|
+
raise ApiConnectionError("bulk_update", response, client)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
async def asyncio(
|
|
83
|
+
updates: List[Union[ScriptedUpdateDict, Dict[str, Any]]],
|
|
84
|
+
timeout: Optional[float] = 60.0,
|
|
85
|
+
client: Optional[Client] = None,
|
|
86
|
+
) -> Dict[str, Any]:
|
|
87
|
+
"""Send scripted bulk updates to OpenSearch through octostar-api (async).
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
updates: List of update dicts. Each must contain ``entity_id`` and
|
|
91
|
+
``script_source``. Optionally ``script_params`` and
|
|
92
|
+
``do_not_update_before``.
|
|
93
|
+
timeout: HTTP timeout in seconds.
|
|
94
|
+
client: Octostar client. Falls back to the default client.
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
Dict with ``status``, ``updated``, ``nooped_entity_ids``, ``errors``.
|
|
98
|
+
|
|
99
|
+
Raises:
|
|
100
|
+
ApiConnectionError: If the request fails.
|
|
101
|
+
"""
|
|
102
|
+
if not updates:
|
|
103
|
+
return {
|
|
104
|
+
"status": "success",
|
|
105
|
+
"updated": 0,
|
|
106
|
+
"nooped_entity_ids": [],
|
|
107
|
+
"errors": [],
|
|
108
|
+
}
|
|
109
|
+
if not client:
|
|
110
|
+
client = get_default_client()
|
|
111
|
+
|
|
112
|
+
endpoint_url = f"{client.get_base_url_v1()}/internal/opensearch/_bulk_update"
|
|
113
|
+
headers = {
|
|
114
|
+
"Content-Type": "application/json",
|
|
115
|
+
"Authorization": f"Bearer {client.token}",
|
|
116
|
+
"x-ontology": client.ontology,
|
|
117
|
+
}
|
|
118
|
+
json_body = [dict(u) for u in updates]
|
|
119
|
+
|
|
120
|
+
response = None
|
|
121
|
+
try:
|
|
122
|
+
for attempt in network_retry_strategy():
|
|
123
|
+
with attempt:
|
|
124
|
+
async with httpx.AsyncClient() as httpx_client:
|
|
125
|
+
response = await httpx_client.post(
|
|
126
|
+
endpoint_url,
|
|
127
|
+
json=json_body,
|
|
128
|
+
headers=headers,
|
|
129
|
+
timeout=timeout,
|
|
130
|
+
)
|
|
131
|
+
if response.status_code != 200:
|
|
132
|
+
raise ConnectionError(
|
|
133
|
+
f"bulk_update failed (status {response.status_code}): "
|
|
134
|
+
+ str(response.content)
|
|
135
|
+
)
|
|
136
|
+
return response.json()
|
|
137
|
+
except Exception:
|
|
138
|
+
raise ApiConnectionError("bulk_update", response, client)
|