Office365-REST-Python-Client 3.1.0__py3-none-any.whl → 3.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- office365/communications/callrecords/call_record.py +18 -0
- office365/communications/calls/call.py +3 -0
- office365/communications/presences/presence.py +3 -0
- office365/communications/quotas.py +66 -0
- office365/communications/virtualevents/virtual_event.py +3 -0
- office365/directory/domains/domain.py +3 -0
- office365/directory/licenses/subscribed_sku.py +3 -0
- office365/directory/objects/object.py +3 -0
- office365/directory/quotas.py +34 -0
- office365/entity_collection.py +2 -2
- office365/graph_client.py +46 -2
- office365/limits.py +66 -0
- office365/migration/__init__.py +6 -1
- office365/migration/adapters/__init__.py +14 -4
- office365/migration/adapters/filesystem.py +4 -1
- office365/migration/assessment/export.py +5 -5
- office365/migration/assessment/issue.py +19 -0
- office365/migration/assessment/report.py +11 -0
- office365/migration/assessment/scanners/base.py +37 -49
- office365/migration/base.py +33 -8
- office365/migration/checkpoint.py +1 -1
- office365/migration/outlook/assessor.py +4 -4
- office365/migration/package/__init__.py +82 -0
- office365/migration/package/builder.py +603 -0
- office365/migration/package/crypto.py +54 -0
- office365/migration/package/manifest_xml.py +495 -0
- office365/migration/package/staging.py +180 -0
- office365/migration/report.py +7 -2
- office365/migration/report_io.py +79 -0
- office365/migration/runner.py +117 -10
- office365/migration/server_job.py +126 -14
- office365/migration/session.py +132 -49
- office365/migration/settings.py +67 -0
- office365/migration/sharepoint/__init__.py +2 -0
- office365/migration/sharepoint/adapters.py +207 -41
- office365/migration/sharepoint/assessor.py +45 -9
- office365/migration/sharepoint/options.py +57 -0
- office365/migration/sharepoint/package_target.py +209 -0
- office365/migration/sharepoint/registry.py +22 -0
- office365/migration/sharepoint/resolvers.py +96 -0
- office365/migration/sharepoint/scanners/__init__.py +29 -0
- office365/migration/sharepoint/scanners/browser_file_handling.py +55 -0
- office365/migration/sharepoint/scanners/checked_out_files.py +57 -0
- office365/migration/sharepoint/scanners/file_versions.py +56 -0
- office365/migration/sharepoint/scanners/files.py +5 -2
- office365/migration/sharepoint/scanners/large_excel_files.py +56 -0
- office365/migration/sharepoint/scanners/large_lists.py +58 -0
- office365/migration/sharepoint/scanners/large_sites.py +4 -3
- office365/migration/sharepoint/scanners/long_onedrive_urls.py +52 -0
- office365/migration/sharepoint/scanners/lookup.py +38 -0
- office365/migration/sharepoint/scanners/paths.py +9 -4
- office365/migration/sharepoint/scanners/permissions.py +20 -2
- office365/migration/sharepoint/scanners/smat.py +51 -0
- office365/migration/sharepoint/scanners/summary.py +1 -0
- office365/migration/sharepoint/scanners/thicket_folders.py +38 -0
- office365/migration/sharepoint/scanners/unsupported_site_templates.py +90 -0
- office365/migration/sharepoint/tenant_assessor.py +5 -3
- office365/migration/sharepoint/transfer.py +4 -2
- office365/migration/tasks.py +121 -0
- office365/migration/validators.py +2 -19
- office365/onedrive/driveitems/driveItem.py +116 -12
- office365/onedrive/termstore/groups/collection.py +2 -9
- office365/onedrive/termstore/sets/collection.py +2 -9
- office365/onedrive/termstore/terms/collection.py +3 -10
- office365/onedrive/workbooks/workbook.py +7 -0
- office365/outlook/contacts/folders/collection.py +4 -13
- office365/runtime/auth/authentication_context.py +5 -0
- office365/runtime/client_object.py +18 -0
- office365/runtime/client_object_collection.py +75 -245
- office365/runtime/client_request.py +77 -1
- office365/runtime/client_request_exception.py +127 -39
- office365/runtime/client_runtime_context.py +79 -8
- office365/runtime/converters/csv_writer.py +10 -4
- office365/runtime/converters/dataframe.py +84 -4
- office365/runtime/converters/json_file.py +12 -3
- office365/runtime/converters/parquet.py +67 -0
- office365/runtime/converters/records.py +31 -12
- office365/runtime/converters/registry.py +185 -0
- office365/runtime/converters/scalars.py +31 -0
- office365/runtime/converters/sql.py +69 -0
- office365/runtime/converters/streamers.py +105 -0
- office365/runtime/converters/streams.py +42 -0
- office365/runtime/converters/upsert.py +101 -0
- office365/runtime/exceptions.py +105 -0
- office365/runtime/http/prefer.py +26 -0
- office365/runtime/http/throttling.py +258 -56
- office365/runtime/imports.py +571 -0
- office365/runtime/limits.py +455 -0
- office365/runtime/odata/request.py +11 -2
- office365/runtime/odata/type.py +1 -0
- office365/runtime/odata/v3/batch_request.py +4 -2
- office365/runtime/odata/v4/batch_request.py +22 -7
- office365/runtime/operations.py +29 -0
- office365/runtime/parallel.py +12 -14
- office365/runtime/paths/v4/entity.py +2 -2
- office365/runtime/queries/batch.py +18 -1
- office365/runtime/queries/get_or_create.py +109 -0
- office365/runtime/record_collection.py +443 -0
- office365/runtime/retry.py +92 -26
- office365/runtime/transport/base.py +6 -1
- office365/runtime/transport/requests_transport.py +4 -0
- office365/runtime/transport/throttled_transport.py +67 -0
- office365/runtime/verification.py +40 -0
- office365/sharepoint/activities/logger.py +118 -8
- office365/sharepoint/administration/analytics/usage_service.py +15 -7
- office365/sharepoint/administration/sitemove/service.py +791 -8
- office365/sharepoint/alerts/alert.py +79 -0
- office365/sharepoint/apps/app_collection.py +25 -0
- office365/sharepoint/apps/launcher.py +16 -0
- office365/sharepoint/authpolicy/tenant/compliancereport.py +5 -0
- office365/sharepoint/businessdata/app_bdc_catalog.py +135 -0
- office365/sharepoint/businessdata/entityfieldvaluedictionary.py +65 -0
- office365/sharepoint/businessdata/entityidentifier.py +23 -0
- office365/sharepoint/businessdata/entityinstance.py +52 -0
- office365/sharepoint/businessdata/entityview.py +31 -0
- office365/sharepoint/businessdata/externalsubscriptionstore.py +9 -0
- office365/sharepoint/businessdata/typedescriptor.py +37 -0
- office365/sharepoint/client_context.py +40 -23
- office365/sharepoint/clientsidecomponent/hostedapps/app.py +21 -0
- office365/sharepoint/clientsidecomponent/hostedapps/manager.py +16 -0
- office365/sharepoint/comments/comment.py +84 -2
- office365/sharepoint/contentcenter/machinelearning/models/model.py +0 -23
- office365/sharepoint/contentcenter/machinelearning/publications/publication.py +0 -9
- office365/sharepoint/contentcenter/machinelearning/samples/sample.py +0 -14
- office365/sharepoint/contentcenter/machinelearning/workitems/item.py +0 -80
- office365/sharepoint/contentsolution/template_api.py +148 -0
- office365/sharepoint/contenttypes/collection.py +32 -10
- office365/sharepoint/convergence/addnoliveblobsinodcservicestateapi.py +9 -0
- office365/sharepoint/convergence/consumer_permissions.py +15 -0
- office365/sharepoint/convergence/migration_complete_state_api.py +25 -0
- office365/sharepoint/convergence/odcmetadatacleanedupapi.py +9 -0
- office365/sharepoint/documentmanagement/document_id.py +12 -0
- office365/sharepoint/entity.py +11 -2
- office365/sharepoint/entity_collection.py +2 -2
- office365/sharepoint/exceptions.py +194 -0
- office365/sharepoint/fields/builtin_field_id.py +45 -34
- office365/sharepoint/fields/builtin_field_name.py +123 -0
- office365/sharepoint/fields/coercion.py +131 -0
- office365/sharepoint/fields/collection.py +54 -37
- office365/sharepoint/fields/field.py +17 -0
- office365/sharepoint/fields/name.py +46 -2
- office365/sharepoint/files/collection.py +7 -1
- office365/sharepoint/files/file.py +54 -8
- office365/sharepoint/files/versions/version.py +38 -13
- office365/sharepoint/folders/collection.py +4 -0
- office365/sharepoint/folders/folder.py +91 -21
- office365/sharepoint/htmlpages/html_page.py +16 -0
- office365/sharepoint/htmlpages/html_page_service.py +19 -0
- office365/sharepoint/ir/migration.py +9 -0
- office365/sharepoint/listitems/caml/__init__.py +31 -0
- office365/sharepoint/listitems/caml/builder.py +129 -0
- office365/sharepoint/listitems/caml/expressions.py +424 -0
- office365/sharepoint/listitems/caml/fields.py +68 -0
- office365/sharepoint/listitems/caml/guard.py +63 -0
- office365/sharepoint/listitems/caml/query.py +71 -2
- office365/sharepoint/listitems/caml/values.py +113 -0
- office365/sharepoint/listitems/collection.py +21 -20
- office365/sharepoint/listitems/upsert.py +75 -0
- office365/sharepoint/lists/collection.py +29 -22
- office365/sharepoint/lists/exporter.py +4 -3
- office365/sharepoint/lists/list.py +397 -39
- office365/sharepoint/lists/smart_wiki_library_utility.py +58 -0
- office365/sharepoint/logger/export.py +12 -0
- office365/sharepoint/marketplace/app_metadata.py +97 -1
- office365/sharepoint/marketplace/appstatequeryjobdefinition.py +9 -0
- office365/sharepoint/marketplace/corporatecuratedgallery/carddesigns/card_designs.py +35 -0
- office365/sharepoint/marketplace/sitecollection/appcatalog/allowed_item.py +5 -0
- office365/sharepoint/marketplace/tenant/appcatalog/accessor.py +107 -30
- office365/sharepoint/microfeed/data.py +0 -6
- office365/sharepoint/microservice/manager.py +61 -0
- office365/sharepoint/migrationcenter/common/task.py +193 -4
- office365/sharepoint/migrationcenter/common/task_entity_data.py +1 -1
- office365/sharepoint/migrationcenter/deploy_status.py +13 -0
- office365/sharepoint/migrationcenter/service/properties.py +20 -0
- office365/sharepoint/migrationcenter/service/storage.py +34 -0
- office365/sharepoint/migrationcenter/spflight.py +13 -0
- office365/sharepoint/migrationcenter/urlparseresult.py +77 -0
- office365/sharepoint/multigeo/service/spmultigeoproxy.py +13 -0
- office365/sharepoint/navigation/servicerest.py +127 -0
- office365/sharepoint/oauth/authentication.py +13 -0
- office365/sharepoint/portal/theme_manager.py +120 -8
- office365/sharepoint/principal/users/user.py +10 -0
- office365/sharepoint/publishing/campaign/campaign.py +36 -1
- office365/sharepoint/publishing/feedvideopage.py +0 -7
- office365/sharepoint/publishing/pages/3d/page.py +0 -6
- office365/sharepoint/publishing/pages/page.py +0 -117
- office365/sharepoint/publishing/pages/reposts/repost.py +0 -7
- office365/sharepoint/publishing/pages/topics/topic.py +0 -7
- office365/sharepoint/publishing/subtitlefile.py +0 -40
- office365/sharepoint/quotamanagement/consumer/consumer_quota_info_api.py +14 -0
- office365/sharepoint/quotamanagement/consumer/non_quota_backfill_api.py +23 -0
- office365/sharepoint/quotamanagement/consumer/nonquotamigrationapi.py +9 -0
- office365/sharepoint/quotamanagement/consumer/quotamigrationapi.py +13 -0
- office365/sharepoint/request.py +57 -16
- office365/sharepoint/search/administration/document_crawl_log.py +24 -7
- office365/sharepoint/search/administration/providers/crawl_versions_info.py +29 -0
- office365/sharepoint/search/administration/providers/recycle_bin_info.py +23 -0
- office365/sharepoint/search/administration/providers/site_content_processing_info.py +67 -0
- office365/sharepoint/sites/site.py +976 -65
- office365/sharepoint/sites/siteprovisioninginfoprovider.py +13 -0
- office365/sharepoint/tenant/administration/datagovernance/insight_client.py +292 -23
- office365/sharepoint/tenant/administration/ddiadapter.py +97 -0
- office365/sharepoint/tenant/administration/internal/aad/collection.py +20 -0
- office365/sharepoint/tenant/administration/internal/aad/permission_grant.py +0 -19
- office365/sharepoint/tenant/administration/internal/appservice/permission_request.py +72 -1
- office365/sharepoint/tenant/administration/internal/appservice/principal.py +1 -3
- office365/sharepoint/tenant/administration/odb_license_enforcement_service.py +26 -0
- office365/sharepoint/tenant/administration/sites/properties.py +564 -6
- office365/sharepoint/tenant/administration/spdatagovernancesarrestapiclient.py +60 -1
- office365/sharepoint/tenant/insights/reports/manager.py +24 -12
- office365/sharepoint/tenant/management/office365_tenant.py +451 -2
- office365/sharepoint/tenantadministration/catalog_management_collection.py +69 -0
- office365/sharepoint/tenantadministration/catalog_management_custom_properties_collection.py +237 -0
- office365/sharepoint/tenantadministration/catalog_management_restricted_content_discovery_collection.py +56 -0
- office365/sharepoint/tenantadministration/sp_data_governance_client_exception.py +7 -0
- office365/sharepoint/thresholds.py +529 -0
- office365/sharepoint/translation/jobs/job.py +72 -2
- office365/sharepoint/translation/jobs/status.py +47 -9
- office365/sharepoint/translation/sync_translator.py +15 -2
- office365/sharepoint/utilities/move_copy_util.py +2 -1
- office365/sharepoint/viva/connections/page.py +27 -0
- office365/sharepoint/viva/home.py +38 -0
- office365/sharepoint/viva/learningmanager.py +9 -0
- office365/sharepoint/viva/vivaresources.py +60 -0
- office365/sharepoint/webhooks/subscription.py +0 -11
- office365/sharepoint/webparts/definitions/definition.py +13 -0
- office365/sharepoint/webparts/limited_manager.py +29 -7
- office365/sharepoint/webs/context_web_information.py +14 -9
- office365/sharepoint/webs/templates/__init__.py +11 -0
- office365/sharepoint/webs/templates/type.py +74 -15
- office365/sharepoint/workflow/definition.py +6 -0
- office365/sharepoint/workflow/deployment_service.py +155 -0
- office365/sharepoint/workflow/instances/service.py +33 -0
- office365/sharepoint/workflow/interopservice.py +68 -0
- office365/sharepoint/workflow/messagingservice.py +24 -0
- office365/sharepoint/workflow/servicesmanager.py +41 -2
- office365/sharepoint/workflow/subscriptions/service.py +67 -0
- office365/sharepoint/workflow/subscriptions/subscription.py +31 -0
- office365/sharepoint/workflow/task.py +425 -0
- office365_rest_python_client-3.2.0.dist-info/METADATA +605 -0
- {office365_rest_python_client-3.1.0.dist-info → office365_rest_python_client-3.2.0.dist-info}/RECORD +242 -185
- office365/migration/_util.py +0 -73
- office365/sharepoint/listitems/caml/types.py +0 -70
- office365_rest_python_client-3.1.0.dist-info/METADATA +0 -475
- {office365_rest_python_client-3.1.0.dist-info → office365_rest_python_client-3.2.0.dist-info}/WHEEL +0 -0
|
@@ -1,12 +1,69 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
from dataclasses import dataclass
|
|
3
4
|
from typing import Optional
|
|
4
5
|
|
|
5
6
|
from requests import RequestException, Response
|
|
6
7
|
|
|
8
|
+
from office365.runtime.converters.scalars import parse_int
|
|
9
|
+
|
|
7
10
|
_HEADER_REQUEST_IDS = ("request-id", "client-request-id", "SPRequestGuid")
|
|
8
11
|
|
|
9
12
|
|
|
13
|
+
@dataclass(frozen=True)
|
|
14
|
+
class ErrorPayload:
|
|
15
|
+
"""Normalized view of an OData error response, parsed once for classification."""
|
|
16
|
+
|
|
17
|
+
code: str = ""
|
|
18
|
+
message: str = ""
|
|
19
|
+
details: tuple[dict, ...] = ()
|
|
20
|
+
status: Optional[int] = None
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def hresult(self) -> Optional[str]:
|
|
24
|
+
"""The leading HRESULT of a SharePoint ``code`` (e.g. ``-2147018894``).
|
|
25
|
+
|
|
26
|
+
SharePoint encodes the error as ``"<hresult>, <dotnet-type>"`` (locale
|
|
27
|
+
independent), while Graph uses symbolic codes; returns ``None`` when the
|
|
28
|
+
code has no numeric prefix.
|
|
29
|
+
"""
|
|
30
|
+
token = self.code.split(",", 1)[0].strip()
|
|
31
|
+
return token if token.lstrip("-").isdigit() else None
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def error_type(self) -> Optional[str]:
|
|
35
|
+
"""The .NET type name embedded in a SharePoint ``code`` after the comma.
|
|
36
|
+
|
|
37
|
+
For ``"-2147018894, Microsoft.SharePoint.SPFileLockException"`` this is
|
|
38
|
+
``"Microsoft.SharePoint.SPFileLockException"`` — a locale-independent,
|
|
39
|
+
self-describing discriminator that avoids matching on translated messages.
|
|
40
|
+
"""
|
|
41
|
+
if "," not in self.code:
|
|
42
|
+
return None
|
|
43
|
+
type_name = self.code.split(",", 1)[1].strip()
|
|
44
|
+
return type_name or None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _parse_error(response: Response) -> dict:
|
|
48
|
+
try:
|
|
49
|
+
error = response.json().get("error", {})
|
|
50
|
+
except Exception:
|
|
51
|
+
return {}
|
|
52
|
+
return error if isinstance(error, dict) else {}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _to_payload(error: dict, response: Response) -> ErrorPayload:
|
|
56
|
+
msg = error.get("message")
|
|
57
|
+
message = str(msg.get("value", "")) if isinstance(msg, dict) else str(msg or "")
|
|
58
|
+
details = error.get("details")
|
|
59
|
+
return ErrorPayload(
|
|
60
|
+
code=str(error.get("code") or ""),
|
|
61
|
+
message=message,
|
|
62
|
+
details=tuple(d for d in details if isinstance(d, dict)) if isinstance(details, list) else (),
|
|
63
|
+
status=getattr(response, "status_code", None),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
10
67
|
class ClientRequestException(RequestException):
|
|
11
68
|
"""Custom exception for client requests with enhanced error handling.
|
|
12
69
|
|
|
@@ -20,38 +77,37 @@ class ClientRequestException(RequestException):
|
|
|
20
77
|
super().__init__(*args, **kwargs)
|
|
21
78
|
self._error: dict = {}
|
|
22
79
|
|
|
80
|
+
#: Specificity of this type when several match a payload; the highest wins
|
|
81
|
+
#: (ties broken by registration order). Lets catch-alls coexist with the
|
|
82
|
+
#: concrete types without depending on import order.
|
|
83
|
+
MATCH_PRIORITY: int = 0
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def matches(cls, payload: ErrorPayload) -> bool:
|
|
87
|
+
"""Whether this exception type describes the given error payload."""
|
|
88
|
+
return False
|
|
89
|
+
|
|
23
90
|
@classmethod
|
|
24
91
|
def from_response(cls, response: Response) -> ClientRequestException:
|
|
25
92
|
"""Factory: parse error response, dispatch to the right exception type.
|
|
26
93
|
|
|
27
|
-
Inspects the error payload once and returns
|
|
28
|
-
(e.g. DuplicatedObjectException), so callers never deal with
|
|
29
|
-
HTTP status codes or error JSON.
|
|
94
|
+
Inspects the error payload once and returns the most specific matching
|
|
95
|
+
subclass (e.g. ``DuplicatedObjectException``), so callers never deal with
|
|
96
|
+
HTTP status codes or error JSON. ``MATCH_PRIORITY`` decides between
|
|
97
|
+
overlapping types and registration order breaks ties.
|
|
30
98
|
"""
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
if (
|
|
44
|
-
code in {"nameAlreadyExists", "ErrorFolderExists"}
|
|
45
|
-
or any(d.get("code") == "ConflictingObjects" for d in details)
|
|
46
|
-
or "183" in code
|
|
47
|
-
or "-2130575342" in code # SharePoint: list/survey/document library already exists
|
|
48
|
-
or "already exists" in msg_text.lower()
|
|
49
|
-
):
|
|
50
|
-
exc: ClientRequestException = DuplicatedObjectException(response=response)
|
|
51
|
-
elif "-2147024809" in code or code.lower() in {"itemnotfound", "resourcenotfound", "notfound"}:
|
|
52
|
-
exc: ClientRequestException = ObjectNotFoundException(response=response)
|
|
53
|
-
else:
|
|
54
|
-
exc = cls(response=response)
|
|
99
|
+
from office365.runtime import exceptions # noqa: F401 — registers the built-in error types
|
|
100
|
+
|
|
101
|
+
error = _parse_error(response)
|
|
102
|
+
payload = _to_payload(error, response)
|
|
103
|
+
matches = [
|
|
104
|
+
(getattr(exc_type, "MATCH_PRIORITY", 0), -index, exc_type)
|
|
105
|
+
for index, exc_type in enumerate(_ERROR_TYPES)
|
|
106
|
+
if exc_type.matches(payload)
|
|
107
|
+
]
|
|
108
|
+
exc: ClientRequestException = (
|
|
109
|
+
max(matches, key=lambda item: item[:2])[2](response=response) if matches else cls(response=response)
|
|
110
|
+
)
|
|
55
111
|
|
|
56
112
|
exc._error = error
|
|
57
113
|
if error:
|
|
@@ -65,6 +121,24 @@ class ClientRequestException(RequestException):
|
|
|
65
121
|
def code(self) -> Optional[str]:
|
|
66
122
|
return self._error.get("code")
|
|
67
123
|
|
|
124
|
+
@property
|
|
125
|
+
def hresult(self) -> Optional[str]:
|
|
126
|
+
"""The leading HRESULT of a SharePoint ``code`` (``None`` for Graph codes)."""
|
|
127
|
+
code = self.code
|
|
128
|
+
if not code:
|
|
129
|
+
return None
|
|
130
|
+
token = code.split(",", 1)[0].strip()
|
|
131
|
+
return token if token.lstrip("-").isdigit() else None
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def error_type(self) -> Optional[str]:
|
|
135
|
+
"""The .NET type name embedded in a SharePoint ``code`` (after the comma)."""
|
|
136
|
+
code = self.code
|
|
137
|
+
if not code or "," not in code:
|
|
138
|
+
return None
|
|
139
|
+
type_name = code.split(",", 1)[1].strip()
|
|
140
|
+
return type_name or None
|
|
141
|
+
|
|
68
142
|
@property
|
|
69
143
|
def message(self) -> str:
|
|
70
144
|
msg = self._error.get("message")
|
|
@@ -107,25 +181,39 @@ class ClientRequestException(RequestException):
|
|
|
107
181
|
@property
|
|
108
182
|
def duration_ms(self) -> Optional[int]:
|
|
109
183
|
"""SharePoint server-side processing time (``SPRequestDuration``), ms."""
|
|
110
|
-
return
|
|
184
|
+
return parse_int((getattr(self.response, "headers", None) or {}).get("SPRequestDuration"))
|
|
111
185
|
|
|
112
186
|
@property
|
|
113
187
|
def health_score(self) -> Optional[int]:
|
|
114
188
|
"""SharePoint server health score (``X-SharePointHealthScore``)."""
|
|
115
|
-
return
|
|
189
|
+
return parse_int((getattr(self.response, "headers", None) or {}).get("X-SharePointHealthScore"))
|
|
116
190
|
|
|
117
191
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
192
|
+
# Error types consulted by ``from_response``. The generic runtime types live in
|
|
193
|
+
# ``office365.runtime.exceptions`` and product packages (e.g.
|
|
194
|
+
# ``office365.sharepoint.exceptions``) register their own; both call
|
|
195
|
+
# ``register_error_type`` so the dispatcher stays product-agnostic. Specificity is
|
|
196
|
+
# decided by ``MATCH_PRIORITY`` (registration order breaks ties), not import order.
|
|
197
|
+
_ERROR_TYPES: list[type[ClientRequestException]] = []
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def register_error_type(exc_type: type[ClientRequestException]) -> None:
|
|
201
|
+
"""Register an error type for ``from_response`` dispatch.
|
|
202
|
+
|
|
203
|
+
Modules call this at import time; the dispatcher picks the highest
|
|
204
|
+
``MATCH_PRIORITY`` match, breaking ties by registration order (earlier wins).
|
|
205
|
+
"""
|
|
206
|
+
if exc_type not in _ERROR_TYPES:
|
|
207
|
+
_ERROR_TYPES.append(exc_type)
|
|
208
|
+
|
|
124
209
|
|
|
210
|
+
_LAZY_EXPORTS = ("DuplicatedObjectException", "ObjectNotFoundException")
|
|
125
211
|
|
|
126
|
-
class DuplicatedObjectException(ClientRequestException):
|
|
127
|
-
"""Raised when creating an object that already exists (HTTP 400 + ConflictingObjects)."""
|
|
128
212
|
|
|
213
|
+
def __getattr__(name: str):
|
|
214
|
+
"""Backward-compatible re-export of the generic error types (now in ``exceptions``)."""
|
|
215
|
+
if name in _LAZY_EXPORTS:
|
|
216
|
+
from office365.runtime import exceptions
|
|
129
217
|
|
|
130
|
-
|
|
131
|
-
"
|
|
218
|
+
return getattr(exceptions, name)
|
|
219
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -12,11 +12,13 @@ from office365.runtime.client_request_exception import ClientRequestException
|
|
|
12
12
|
from office365.runtime.client_result import ClientResult
|
|
13
13
|
from office365.runtime.http.http_method import HttpMethod
|
|
14
14
|
from office365.runtime.http.request_options import RequestOptions
|
|
15
|
+
from office365.runtime.limits import Limit, LimitDecl, collect_class_limits, collect_limit_meta
|
|
15
16
|
from office365.runtime.queries.client_query import ClientQuery
|
|
16
17
|
from office365.runtime.queries.read_entity import ReadEntityQuery
|
|
17
18
|
|
|
18
19
|
if TYPE_CHECKING:
|
|
19
20
|
from office365.runtime.client_object import ClientObject
|
|
21
|
+
from office365.runtime.http.throttling import RateLimiter
|
|
20
22
|
from office365.runtime.queries.batch import BatchQuery
|
|
21
23
|
|
|
22
24
|
|
|
@@ -26,11 +28,26 @@ class ClientRuntimeContext(ABC):
|
|
|
26
28
|
Provides core functionality for executing queries and managing request lifecycle.
|
|
27
29
|
"""
|
|
28
30
|
|
|
31
|
+
_limit_meta: dict[str, Tuple[LimitDecl, ...]] = {}
|
|
32
|
+
_class_limit_decls: Tuple[LimitDecl, ...] = ()
|
|
33
|
+
|
|
34
|
+
def __init_subclass__(cls, **kwargs: object) -> None:
|
|
35
|
+
super().__init_subclass__(**kwargs)
|
|
36
|
+
cls._limit_meta = collect_limit_meta(cls)
|
|
37
|
+
cls._class_limit_decls = collect_class_limits(cls)
|
|
38
|
+
|
|
29
39
|
def __init__(self) -> None:
|
|
30
40
|
self._queries: deque[ClientQuery] = deque()
|
|
31
41
|
self._current_query = None
|
|
32
42
|
self._pending_request: ClientRequest | None = None
|
|
33
43
|
|
|
44
|
+
@classmethod
|
|
45
|
+
def declared_limits(cls) -> Tuple[Limit, ...]:
|
|
46
|
+
"""The limits declared on this class and its methods/properties (``@limit``)."""
|
|
47
|
+
class_limits = tuple(decl.limit for decl in cls._class_limit_decls)
|
|
48
|
+
member_limits = tuple(decl.limit for decls in cls._limit_meta.values() for decl in decls)
|
|
49
|
+
return (*class_limits, *member_limits)
|
|
50
|
+
|
|
34
51
|
@property
|
|
35
52
|
def service_root_url(self) -> str:
|
|
36
53
|
"""Get the API service root URL"""
|
|
@@ -54,6 +71,7 @@ class ClientRuntimeContext(ABC):
|
|
|
54
71
|
success_callback: Optional[Callable[[ClientObject | None], None]] = None,
|
|
55
72
|
failure_callback: Optional[Callable[[int, Exception], Optional[int]]] = None,
|
|
56
73
|
exceptions: Tuple[Type[Exception], ...] = (ClientRequestException,),
|
|
74
|
+
is_retriable: Optional[Callable[[Exception], bool]] = None,
|
|
57
75
|
) -> None:
|
|
58
76
|
"""Executes pending queries with retry logic.
|
|
59
77
|
|
|
@@ -74,8 +92,12 @@ class ClientRuntimeContext(ABC):
|
|
|
74
92
|
failure_callback: Called after each failed attempt; may return a
|
|
75
93
|
retry delay in seconds to override the backoff
|
|
76
94
|
exceptions: Exception types that trigger retries
|
|
95
|
+
is_retriable: Optional predicate deciding whether a caught exception
|
|
96
|
+
is retried. Defaults to :func:`~office365.runtime.retry.is_transient_error`;
|
|
97
|
+
pass :func:`~office365.runtime.retry.retry_on` to also retry
|
|
98
|
+
otherwise-permanent errors such as a locked file (HTTP 423).
|
|
77
99
|
"""
|
|
78
|
-
from office365.runtime.retry import retry
|
|
100
|
+
from office365.runtime.retry import is_transient_error, retry
|
|
79
101
|
|
|
80
102
|
def _on_failure(_attempt: int, ex: Exception) -> Optional[int]:
|
|
81
103
|
# Re-queue the failed query for a retry, except on the last attempt —
|
|
@@ -96,6 +118,7 @@ class ClientRuntimeContext(ABC):
|
|
|
96
118
|
max_delay=max_delay,
|
|
97
119
|
jitter=jitter,
|
|
98
120
|
exceptions=exceptions,
|
|
121
|
+
is_retriable=is_retriable or is_transient_error,
|
|
99
122
|
on_failure=_on_failure,
|
|
100
123
|
on_success=_on_success,
|
|
101
124
|
)
|
|
@@ -188,6 +211,12 @@ class ClientRuntimeContext(ABC):
|
|
|
188
211
|
return self
|
|
189
212
|
|
|
190
213
|
def on_error(self, action: Callable[[ClientRequestException], None], once: bool = True) -> Self:
|
|
214
|
+
"""Attach an error handler for the pending query — the error is swallowed.
|
|
215
|
+
|
|
216
|
+
This is a control hook (see :meth:`ClientRequest.on_error`): once a
|
|
217
|
+
handler is attached, a failing query is considered handled and does not
|
|
218
|
+
re-raise. Use ``after_execute`` / ``throttle_guard`` for observation.
|
|
219
|
+
"""
|
|
191
220
|
if len(self._queries) == 0:
|
|
192
221
|
return self
|
|
193
222
|
query = self._queries[-1]
|
|
@@ -210,6 +239,33 @@ class ClientRuntimeContext(ABC):
|
|
|
210
239
|
"""
|
|
211
240
|
return self.pending_request().execute_request(path)
|
|
212
241
|
|
|
242
|
+
@property
|
|
243
|
+
def rate_limiter(self) -> "RateLimiter | None":
|
|
244
|
+
"""The shared rate limiter pacing this context's requests, if any.
|
|
245
|
+
|
|
246
|
+
Returns ``None`` until :meth:`with_rate_limit` (or a shared limiter via
|
|
247
|
+
``ClientRequest.with_rate_limiter``) is configured.
|
|
248
|
+
"""
|
|
249
|
+
return self.pending_request().rate_limiter
|
|
250
|
+
|
|
251
|
+
def with_rate_limit(self, health_threshold: int = 80, min_interval: float = 0.0) -> Self:
|
|
252
|
+
"""Enable fleet-wide pacing for this context (opt-in).
|
|
253
|
+
|
|
254
|
+
Wraps the context transport with a shared rate limiter, so every request
|
|
255
|
+
— including the parallel batches dispatched by ``execute_batch`` — is
|
|
256
|
+
gated by ``Retry-After`` / ``X-SharePointHealthScore`` as a group. Call
|
|
257
|
+
this after any transport configuration (``with_transport``).
|
|
258
|
+
|
|
259
|
+
Args:
|
|
260
|
+
health_threshold: Health score at/above which the group paces.
|
|
261
|
+
min_interval: Minimum pause applied on a high health score (seconds).
|
|
262
|
+
|
|
263
|
+
Returns:
|
|
264
|
+
Self: Supports method chaining
|
|
265
|
+
"""
|
|
266
|
+
self.pending_request().with_rate_limit(health_threshold=health_threshold, min_interval=min_interval)
|
|
267
|
+
return self
|
|
268
|
+
|
|
213
269
|
def execute_query(self) -> Self:
|
|
214
270
|
"""Executes all pending queries.
|
|
215
271
|
|
|
@@ -428,24 +484,39 @@ class ClientRuntimeContext(ABC):
|
|
|
428
484
|
"""Execute batch units concurrently on a thread pool.
|
|
429
485
|
|
|
430
486
|
Reuses the generic :func:`~office365.runtime.parallel.run_parallel`
|
|
431
|
-
primitive; ``success_callback`` is invoked
|
|
432
|
-
(
|
|
487
|
+
primitive; ``success_callback`` is invoked **live, as each batch
|
|
488
|
+
completes** (on the calling thread), so callers can report progress.
|
|
489
|
+
After the pool drains, the first failure is re-raised.
|
|
433
490
|
|
|
434
491
|
Args:
|
|
435
492
|
batches: Batch units to execute
|
|
436
493
|
concurrency: Maximum number of concurrent batch requests
|
|
437
|
-
success_callback: Called with each batch's
|
|
494
|
+
success_callback: Called with each successfully completed batch's
|
|
495
|
+
return types, in completion order
|
|
438
496
|
"""
|
|
439
497
|
from office365.runtime.parallel import run_parallel
|
|
440
498
|
|
|
441
|
-
|
|
499
|
+
errors: list[BaseException] = []
|
|
500
|
+
|
|
501
|
+
def _on_progress(snapshot: Any) -> None:
|
|
502
|
+
if not callable(success_callback):
|
|
503
|
+
return
|
|
504
|
+
for return_types in snapshot.items or []:
|
|
505
|
+
if return_types is not None:
|
|
506
|
+
success_callback(return_types)
|
|
507
|
+
|
|
508
|
+
def _on_error(_task: Any, error: BaseException) -> None:
|
|
509
|
+
errors.append(error)
|
|
510
|
+
|
|
511
|
+
run_parallel(
|
|
442
512
|
lambda _ctx, batch_qry: self._execute_batch(batch_qry),
|
|
443
513
|
batches,
|
|
444
514
|
concurrency=concurrency,
|
|
515
|
+
progress=_on_progress,
|
|
516
|
+
on_error=_on_error,
|
|
445
517
|
)
|
|
446
|
-
if
|
|
447
|
-
|
|
448
|
-
success_callback(return_types)
|
|
518
|
+
if errors:
|
|
519
|
+
raise errors[0]
|
|
449
520
|
|
|
450
521
|
def _execute_batch(self, batch_qry: "BatchQuery") -> List[Any]:
|
|
451
522
|
"""Execute a single batch unit (implemented by concrete contexts)."""
|
|
@@ -22,7 +22,7 @@ def _cell(value: Any) -> str:
|
|
|
22
22
|
return str(value)
|
|
23
23
|
|
|
24
24
|
|
|
25
|
-
def write_csv(collection: ClientObjectCollection, file: IO[str]) -> None:
|
|
25
|
+
def write_csv(collection: ClientObjectCollection, file: IO[str], delimiter: str = ",") -> None:
|
|
26
26
|
"""Write collection items to CSV using the shared record projection.
|
|
27
27
|
|
|
28
28
|
The projection (``records.iter_records``) mirrors the previous behaviour:
|
|
@@ -38,10 +38,10 @@ def write_csv(collection: ClientObjectCollection, file: IO[str]) -> None:
|
|
|
38
38
|
... .to_csv(f) \\
|
|
39
39
|
... .execute_query()
|
|
40
40
|
"""
|
|
41
|
-
write_records(iter_records(collection), file)
|
|
41
|
+
write_records(iter_records(collection), file, delimiter=delimiter)
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
def write_records(records: list[dict], file: IO[str], columns: Optional[List[str]] = None) -> None:
|
|
44
|
+
def write_records(records: list[dict], file: IO[str], columns: Optional[List[str]] = None, delimiter: str = ",") -> None:
|
|
45
45
|
"""Write a list of dict records to CSV.
|
|
46
46
|
|
|
47
47
|
Args:
|
|
@@ -54,7 +54,13 @@ def write_records(records: list[dict], file: IO[str], columns: Optional[List[str
|
|
|
54
54
|
return
|
|
55
55
|
if columns is None:
|
|
56
56
|
columns = list(dict.fromkeys(key for record in records for key in record))
|
|
57
|
-
writer = csv.writer(file)
|
|
57
|
+
writer = csv.writer(file, delimiter=delimiter)
|
|
58
58
|
writer.writerow(columns)
|
|
59
|
+
write_rows(records, file, columns, delimiter)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def write_rows(records: list[dict], file: IO[str], columns: list[str], delimiter: str = ",") -> None:
|
|
63
|
+
"""Write data rows only (no header) — used by the paged/streaming exporter."""
|
|
64
|
+
writer = csv.writer(file, delimiter=delimiter)
|
|
59
65
|
for record in records:
|
|
60
66
|
writer.writerow([_cell(record.get(key)) for key in columns])
|
|
@@ -9,7 +9,9 @@ risk of requiring the ``[pandas]`` extra.
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import math
|
|
12
|
-
from
|
|
12
|
+
from datetime import date, datetime
|
|
13
|
+
from os import PathLike
|
|
14
|
+
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, cast
|
|
13
15
|
|
|
14
16
|
from typing_extensions import Self
|
|
15
17
|
|
|
@@ -74,6 +76,43 @@ def read_dataframe(df) -> List[Dict[str, Any]]:
|
|
|
74
76
|
return records_from_dataframe(df)
|
|
75
77
|
|
|
76
78
|
|
|
79
|
+
def dataframe_to_bytes(df, format: str = "csv", index: bool = False, **opts: Any) -> bytes: # noqa: A002
|
|
80
|
+
"""Serialize a DataFrame to file bytes (``csv``/``xlsx``/``json``/``parquet``).
|
|
81
|
+
|
|
82
|
+
CSV is written UTF-8 with a BOM (``utf-8-sig``) so Excel keeps the columns.
|
|
83
|
+
"""
|
|
84
|
+
import io
|
|
85
|
+
|
|
86
|
+
if format in ("xlsx", "excel"):
|
|
87
|
+
buffer = io.BytesIO()
|
|
88
|
+
df.to_excel(buffer, index=index, **opts)
|
|
89
|
+
return buffer.getvalue()
|
|
90
|
+
if format == "json":
|
|
91
|
+
return df.to_json(**opts).encode("utf-8")
|
|
92
|
+
if format in ("parquet", "orc", "feather"):
|
|
93
|
+
buffer = io.BytesIO()
|
|
94
|
+
writer = {"parquet": df.to_parquet, "orc": df.to_orc, "feather": df.to_feather}[format]
|
|
95
|
+
writer(buffer, **opts)
|
|
96
|
+
return buffer.getvalue()
|
|
97
|
+
return df.to_csv(index=index, **opts).encode("utf-8-sig")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def dataframe_from_bytes(content: bytes, format: str = "csv", **opts: Any) -> "pd.DataFrame": # noqa: A002
|
|
101
|
+
"""Parse in-memory file bytes into a DataFrame (``csv``/``xlsx``/``json``/``parquet``)."""
|
|
102
|
+
import io
|
|
103
|
+
|
|
104
|
+
pd = require_pandas()
|
|
105
|
+
buffer = io.BytesIO(content)
|
|
106
|
+
if format in ("xlsx", "excel"):
|
|
107
|
+
return pd.read_excel(buffer, **opts)
|
|
108
|
+
if format == "json":
|
|
109
|
+
return pd.read_json(buffer, **opts)
|
|
110
|
+
if format in ("parquet", "orc", "feather"):
|
|
111
|
+
reader = {"parquet": pd.read_parquet, "orc": pd.read_orc, "feather": pd.read_feather}[format]
|
|
112
|
+
return reader(buffer, **opts)
|
|
113
|
+
return pd.read_csv(buffer, **opts)
|
|
114
|
+
|
|
115
|
+
|
|
77
116
|
def series_kind(pd, series) -> str:
|
|
78
117
|
"""Categorize a pandas column dtype into a generic kind.
|
|
79
118
|
|
|
@@ -81,19 +120,36 @@ def series_kind(pd, series) -> str:
|
|
|
81
120
|
Kept in the pandas boundary so callers can map the generic kind onto their
|
|
82
121
|
own schema (e.g. a SharePoint ``FieldType``) without importing pandas.
|
|
83
122
|
|
|
123
|
+
Nullable dtypes are handled; complex and timedelta map to ``"text"`` (no
|
|
124
|
+
SharePoint equivalent), and an object column holding only ``datetime``/``date``
|
|
125
|
+
values is treated as ``"datetime"``.
|
|
126
|
+
|
|
84
127
|
Args:
|
|
85
128
|
pd: The pandas module (e.g. from :func:`require_pandas`).
|
|
86
129
|
series: A pandas Series (a DataFrame column).
|
|
87
130
|
"""
|
|
88
|
-
|
|
131
|
+
dtype = series.dtype
|
|
132
|
+
if pd.api.types.is_bool_dtype(dtype):
|
|
89
133
|
return "boolean"
|
|
90
|
-
if pd.api.types.is_datetime64_any_dtype(
|
|
134
|
+
if pd.api.types.is_datetime64_any_dtype(dtype):
|
|
91
135
|
return "datetime"
|
|
92
|
-
if pd.api.types.
|
|
136
|
+
if pd.api.types.is_complex_dtype(dtype):
|
|
137
|
+
return "text"
|
|
138
|
+
if pd.api.types.is_numeric_dtype(dtype):
|
|
93
139
|
return "number"
|
|
140
|
+
if pd.api.types.is_object_dtype(dtype) and _holds_datetimes(series):
|
|
141
|
+
return "datetime"
|
|
94
142
|
return "text"
|
|
95
143
|
|
|
96
144
|
|
|
145
|
+
def _holds_datetimes(series) -> bool:
|
|
146
|
+
"""Whether an object-dtype column holds only ``datetime``/``date`` values."""
|
|
147
|
+
non_null = series.dropna()
|
|
148
|
+
if non_null.empty:
|
|
149
|
+
return False
|
|
150
|
+
return all(isinstance(value, (datetime, date)) for value in non_null.head(100))
|
|
151
|
+
|
|
152
|
+
|
|
97
153
|
def records_from_dataframe(
|
|
98
154
|
df,
|
|
99
155
|
key_fn: Optional[Callable[[str], str]] = None,
|
|
@@ -120,3 +176,27 @@ def records_from_dataframe(
|
|
|
120
176
|
record[key_fn(key) if key_fn is not None else key] = value
|
|
121
177
|
records.append(record)
|
|
122
178
|
return records
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def dataframe_chunks(source: Any, chunksize: int) -> tuple[Iterator[Any], Optional[int]]:
|
|
182
|
+
"""Split a DataFrame/CSV source into chunks, with the total when known.
|
|
183
|
+
|
|
184
|
+
Accepts a ``DataFrame`` (sliced into ``chunksize``-row slices), a CSV
|
|
185
|
+
path/URL/file (read with ``pandas.read_csv(chunksize=)``), a pandas chunk
|
|
186
|
+
reader (``pd.read_csv(..., chunksize=)``, used as-is), or any iterable of
|
|
187
|
+
chunks (used as-is). Returns ``(chunks, total)`` — ``total`` is ``None`` when
|
|
188
|
+
it can't be known upfront (a CSV stream or an opaque iterable).
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
source: A DataFrame, a CSV path/URL/file, or an iterable of chunks.
|
|
192
|
+
chunksize: Rows per chunk for a DataFrame or CSV source.
|
|
193
|
+
"""
|
|
194
|
+
if hasattr(source, "iloc"): # a pandas DataFrame — slice it (bounded queue)
|
|
195
|
+
total = len(source)
|
|
196
|
+
return (source.iloc[start : start + chunksize] for start in range(0, total, chunksize)), total
|
|
197
|
+
if hasattr(source, "get_chunk"): # a pandas TextFileReader (already chunked) — iterate
|
|
198
|
+
return iter(source), None
|
|
199
|
+
if isinstance(source, (str, PathLike)) or hasattr(source, "read"):
|
|
200
|
+
pd = require_pandas()
|
|
201
|
+
return pd.read_csv(source, chunksize=chunksize), None
|
|
202
|
+
return iter(source), None
|
|
@@ -7,15 +7,24 @@ of records — useful for round-tripping an export verbatim.
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import json
|
|
10
|
-
from typing import IO, Any, Dict, List
|
|
10
|
+
from typing import IO, Any, Dict, List, Optional
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
def write_json(records: List[Dict[str, Any]], file: IO[str]) -> None:
|
|
13
|
+
def write_json(records: List[Dict[str, Any]], file: IO[str], indent: Optional[int] = None) -> None:
|
|
14
14
|
"""Write records as a JSON array."""
|
|
15
|
-
json.dump(records, file)
|
|
15
|
+
json.dump(records, file, indent=indent, default=str)
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
def read_json(file: IO[str]) -> List[Dict[str, Any]]:
|
|
19
19
|
"""Read a JSON array of records."""
|
|
20
20
|
data = json.load(file)
|
|
21
21
|
return data if isinstance(data, list) else []
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def record_to_json(payload: Any) -> str:
|
|
25
|
+
"""Serialize a record to the canonical JSON text used when persisting it.
|
|
26
|
+
|
|
27
|
+
Non-JSON values (entity/datetime objects) fall back to ``str`` so the text is
|
|
28
|
+
stable — source and target checksums both use this form.
|
|
29
|
+
"""
|
|
30
|
+
return json.dumps(payload, indent=2, default=str)
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Parquet/ORC/Feather reader/writer via pyarrow — the optional-dependency boundary.
|
|
2
|
+
|
|
3
|
+
Every function imports pyarrow lazily, so the core library never depends on it
|
|
4
|
+
(``pip install office365-rest-python-client[parquet]``). Sources/targets accept a
|
|
5
|
+
path or an open binary stream.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any, Dict, List
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def require_pyarrow():
|
|
14
|
+
"""Import pyarrow or raise an actionable error."""
|
|
15
|
+
try:
|
|
16
|
+
import pyarrow # type: ignore[import-not-found]
|
|
17
|
+
except ImportError:
|
|
18
|
+
raise ImportError("pip install office365-rest-python-client[parquet]") from None
|
|
19
|
+
return pyarrow
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _read(source: Any, reader: str) -> List[Dict[str, Any]]:
|
|
23
|
+
require_pyarrow()
|
|
24
|
+
from office365.runtime.converters.streams import open_binary
|
|
25
|
+
|
|
26
|
+
module = __import__(f"pyarrow.{reader}", fromlist=["read_table"])
|
|
27
|
+
with open_binary(source, "rb") as f:
|
|
28
|
+
return module.read_table(f).to_pylist()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _write(records: List[Dict[str, Any]], target: Any, writer: str) -> None:
|
|
32
|
+
import pyarrow as pa # type: ignore[import-not-found]
|
|
33
|
+
|
|
34
|
+
require_pyarrow()
|
|
35
|
+
from office365.runtime.converters.streams import open_binary
|
|
36
|
+
|
|
37
|
+
module = __import__(f"pyarrow.{writer}", fromlist=["write_table"])
|
|
38
|
+
table = pa.Table.from_pylist(records)
|
|
39
|
+
with open_binary(target, "wb") as f:
|
|
40
|
+
if writer == "feather":
|
|
41
|
+
module.write_feather(table, f)
|
|
42
|
+
else:
|
|
43
|
+
module.write_table(table, f)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def read_parquet(source: Any) -> List[Dict[str, Any]]:
|
|
47
|
+
return _read(source, "parquet")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def write_parquet(records: List[Dict[str, Any]], target: Any) -> None:
|
|
51
|
+
_write(records, target, "parquet")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def read_orc(source: Any) -> List[Dict[str, Any]]:
|
|
55
|
+
return _read(source, "orc")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def write_orc(records: List[Dict[str, Any]], target: Any) -> None:
|
|
59
|
+
_write(records, target, "orc")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def read_feather(source: Any) -> List[Dict[str, Any]]:
|
|
63
|
+
return _read(source, "feather")
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def write_feather(records: List[Dict[str, Any]], target: Any) -> None:
|
|
67
|
+
_write(records, target, "feather")
|