nc-py-api 0.20.2__tar.gz → 0.21.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/CHANGELOG.md +20 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/PKG-INFO +2 -3
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/__init__.py +1 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_exceptions.py +10 -4
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_session.py +116 -72
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_version.py +1 -1
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/calendar_api.py +1 -1
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/integration_fastapi.py +99 -58
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/files/_files.py +1 -1
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/files/files.py +17 -15
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/files/files_async.py +19 -17
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/loginflow_v2.py +2 -2
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/nextcloud.py +5 -5
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/notes.py +2 -2
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/options.py +3 -12
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/talk_bot.py +20 -13
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/users.py +4 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/pyproject.toml +1 -2
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/.gitignore +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/AUTHORS +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/LICENSE.txt +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/README.md +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_deffered_error.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_misc.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_preferences.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_preferences_ex.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_talk_api.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/_theming.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/activity.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/apps.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/__init__.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/defs.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/logger.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/misc.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/occ_commands.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/persist_transformers_cache.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/providers/__init__.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/providers/providers.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/providers/task_processing.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/ui/__init__.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/ui/files_actions.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/ui/resources.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/ui/settings.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/ui/top_menu.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/ui/ui.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/ex_app/uvicorn_fastapi.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/files/__init__.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/files/sharing.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/notifications.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/talk.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/user_status.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/users_groups.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/weather_status.py +0 -0
- {nc_py_api-0.20.2 → nc_py_api-0.21.1}/nc_py_api/webhooks.py +0 -0
|
@@ -2,6 +2,26 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.21.1 - 2025-08-27]
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
|
|
9
|
+
- ExApps: Now exception is raised during model download if there was error to prevent incorrect ExApp installation. #376
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- ExApps: Broken models download ue a typo in the previous PRs. #376
|
|
14
|
+
|
|
15
|
+
## [0.21.0 - 2025-08-26]
|
|
16
|
+
|
|
17
|
+
### Added
|
|
18
|
+
|
|
19
|
+
- Added wipe method to UsersAPI. #368 Thanks to @MrAalen
|
|
20
|
+
|
|
21
|
+
### Changed
|
|
22
|
+
|
|
23
|
+
- Switch from `httpx` library to the `niquests` library. #375 Thanks to @Ousret
|
|
24
|
+
|
|
5
25
|
## [0.20.2 - 2025-05-28]
|
|
6
26
|
|
|
7
27
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nc-py-api
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.21.1
|
|
4
4
|
Summary: Nextcloud Python Framework
|
|
5
5
|
Project-URL: Changelog, https://github.com/cloud-py-api/nc_py_api/blob/main/CHANGELOG.md
|
|
6
6
|
Project-URL: Documentation, https://cloud-py-api.github.io/nc_py_api/
|
|
@@ -31,10 +31,9 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
|
31
31
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
32
32
|
Requires-Python: >=3.10
|
|
33
33
|
Requires-Dist: fastapi>=0.109.2
|
|
34
|
-
Requires-Dist:
|
|
34
|
+
Requires-Dist: niquests<4,>=3
|
|
35
35
|
Requires-Dist: pydantic>=2.1.1
|
|
36
36
|
Requires-Dist: python-dotenv>=1
|
|
37
|
-
Requires-Dist: truststore==0.10
|
|
38
37
|
Requires-Dist: xmltodict>=0.13
|
|
39
38
|
Provides-Extra: app
|
|
40
39
|
Requires-Dist: uvicorn[standard]>=0.23.2; extra == 'app'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Exceptions for the Nextcloud API."""
|
|
2
2
|
|
|
3
|
-
from
|
|
3
|
+
from niquests import HTTPError, Response
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
class NextcloudException(Exception):
|
|
@@ -60,6 +60,12 @@ def check_error(response: Response, info: str = ""):
|
|
|
60
60
|
else:
|
|
61
61
|
phrase = "Unknown error"
|
|
62
62
|
raise NextcloudException(status_code, reason=phrase, info=info)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
|
|
64
|
+
try:
|
|
65
|
+
response.raise_for_status()
|
|
66
|
+
except HTTPError as e:
|
|
67
|
+
raise NextcloudException(status_code, reason=response.reason, info=info) from e
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class ModelFetchError(Exception):
|
|
71
|
+
"""Exception raised when model fetching fails."""
|
|
@@ -10,9 +10,11 @@ from dataclasses import dataclass
|
|
|
10
10
|
from enum import IntEnum
|
|
11
11
|
from json import loads
|
|
12
12
|
from os import environ
|
|
13
|
+
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
13
14
|
|
|
14
|
-
from
|
|
15
|
-
from
|
|
15
|
+
from niquests import AsyncSession, ReadTimeout, Request, Response, Session
|
|
16
|
+
from niquests import __version__ as niquests_version
|
|
17
|
+
from niquests.structures import CaseInsensitiveDict
|
|
16
18
|
from starlette.requests import HTTPConnection
|
|
17
19
|
|
|
18
20
|
from . import options
|
|
@@ -49,6 +51,13 @@ class ServerVersion(typing.TypedDict):
|
|
|
49
51
|
"""Indicates if the subscription has extended support"""
|
|
50
52
|
|
|
51
53
|
|
|
54
|
+
@dataclass
|
|
55
|
+
class Limits:
|
|
56
|
+
max_keepalive_connections: int | None = 20
|
|
57
|
+
max_connections: int | None = 100
|
|
58
|
+
keepalive_expiry: int | float | None = 5
|
|
59
|
+
|
|
60
|
+
|
|
52
61
|
@dataclass
|
|
53
62
|
class RuntimeOptions:
|
|
54
63
|
xdebug_session: str
|
|
@@ -134,11 +143,11 @@ class AppConfig(BasicConfig):
|
|
|
134
143
|
|
|
135
144
|
|
|
136
145
|
class NcSessionBase(ABC):
|
|
137
|
-
adapter:
|
|
138
|
-
adapter_dav:
|
|
146
|
+
adapter: AsyncSession | Session
|
|
147
|
+
adapter_dav: AsyncSession | Session
|
|
139
148
|
cfg: BasicConfig
|
|
140
149
|
custom_headers: dict
|
|
141
|
-
response_headers:
|
|
150
|
+
response_headers: CaseInsensitiveDict
|
|
142
151
|
_user: str
|
|
143
152
|
_capabilities: dict
|
|
144
153
|
|
|
@@ -150,7 +159,7 @@ class NcSessionBase(ABC):
|
|
|
150
159
|
self.limits = Limits(max_keepalive_connections=20, max_connections=20, keepalive_expiry=60.0)
|
|
151
160
|
self.init_adapter()
|
|
152
161
|
self.init_adapter_dav()
|
|
153
|
-
self.response_headers =
|
|
162
|
+
self.response_headers = CaseInsensitiveDict()
|
|
154
163
|
self._ocs_regexp = re.compile(r"/ocs/v[12]\.php/|/apps/groupfolders/")
|
|
155
164
|
|
|
156
165
|
def init_adapter(self, restart=False) -> None:
|
|
@@ -172,7 +181,7 @@ class NcSessionBase(ABC):
|
|
|
172
181
|
self.adapter_dav.cookies.set("XDEBUG_SESSION", options.XDEBUG_SESSION)
|
|
173
182
|
|
|
174
183
|
@abstractmethod
|
|
175
|
-
def _create_adapter(self, dav: bool = False) ->
|
|
184
|
+
def _create_adapter(self, dav: bool = False) -> AsyncSession | Session:
|
|
176
185
|
pass # pragma: no cover
|
|
177
186
|
|
|
178
187
|
@property
|
|
@@ -187,8 +196,8 @@ class NcSessionBase(ABC):
|
|
|
187
196
|
|
|
188
197
|
|
|
189
198
|
class NcSessionBasic(NcSessionBase, ABC):
|
|
190
|
-
adapter:
|
|
191
|
-
adapter_dav:
|
|
199
|
+
adapter: Session
|
|
200
|
+
adapter_dav: Session
|
|
192
201
|
|
|
193
202
|
def ocs(
|
|
194
203
|
self,
|
|
@@ -206,9 +215,7 @@ class NcSessionBasic(NcSessionBase, ABC):
|
|
|
206
215
|
info = f"request: {method} {path}"
|
|
207
216
|
nested_req = kwargs.pop("nested_req", False)
|
|
208
217
|
try:
|
|
209
|
-
response = self.adapter.request(
|
|
210
|
-
method, path, content=content, json=json, params=params, files=files, **kwargs
|
|
211
|
-
)
|
|
218
|
+
response = self.adapter.request(method, path, data=content, json=json, params=params, files=files, **kwargs)
|
|
212
219
|
except ReadTimeout:
|
|
213
220
|
raise NextcloudException(408, info=info) from None
|
|
214
221
|
|
|
@@ -281,18 +288,18 @@ class NcSessionBasic(NcSessionBase, ABC):
|
|
|
281
288
|
return {
|
|
282
289
|
"base_url": self.cfg.dav_endpoint,
|
|
283
290
|
"timeout": self.cfg.options.timeout_dav,
|
|
284
|
-
"event_hooks": {"
|
|
291
|
+
"event_hooks": {"pre_request": [], "response": [self._response_event]},
|
|
285
292
|
}
|
|
286
293
|
return {
|
|
287
294
|
"base_url": self.cfg.endpoint,
|
|
288
295
|
"timeout": self.cfg.options.timeout,
|
|
289
|
-
"event_hooks": {"
|
|
296
|
+
"event_hooks": {"pre_request": [self._request_event_ocs], "response": [self._response_event]},
|
|
290
297
|
}
|
|
291
298
|
|
|
292
299
|
def _request_event_ocs(self, request: Request) -> None:
|
|
293
300
|
str_url = str(request.url)
|
|
294
301
|
if re.search(self._ocs_regexp, str_url) is not None: # this is OCS call
|
|
295
|
-
request.url = request.url
|
|
302
|
+
request.url = patch_param(request.url, "format", "json")
|
|
296
303
|
request.headers["Accept"] = "application/json"
|
|
297
304
|
|
|
298
305
|
def _response_event(self, response: Response) -> None:
|
|
@@ -305,15 +312,15 @@ class NcSessionBasic(NcSessionBase, ABC):
|
|
|
305
312
|
|
|
306
313
|
def download2fp(self, url_path: str, fp, dav: bool, params=None, **kwargs):
|
|
307
314
|
adapter = self.adapter_dav if dav else self.adapter
|
|
308
|
-
with adapter.
|
|
315
|
+
with adapter.get(url_path, params=params, headers=kwargs.get("headers"), stream=True) as response:
|
|
309
316
|
check_error(response)
|
|
310
|
-
for data_chunk in response.
|
|
317
|
+
for data_chunk in response.iter_raw(chunk_size=kwargs.get("chunk_size", -1)):
|
|
311
318
|
fp.write(data_chunk)
|
|
312
319
|
|
|
313
320
|
|
|
314
321
|
class AsyncNcSessionBasic(NcSessionBase, ABC):
|
|
315
|
-
adapter:
|
|
316
|
-
adapter_dav:
|
|
322
|
+
adapter: AsyncSession
|
|
323
|
+
adapter_dav: AsyncSession
|
|
317
324
|
|
|
318
325
|
async def ocs(
|
|
319
326
|
self,
|
|
@@ -332,7 +339,7 @@ class AsyncNcSessionBasic(NcSessionBase, ABC):
|
|
|
332
339
|
nested_req = kwargs.pop("nested_req", False)
|
|
333
340
|
try:
|
|
334
341
|
response = await self.adapter.request(
|
|
335
|
-
method, path,
|
|
342
|
+
method, path, data=content, json=json, params=params, files=files, **kwargs
|
|
336
343
|
)
|
|
337
344
|
except ReadTimeout:
|
|
338
345
|
raise NextcloudException(408, info=info) from None
|
|
@@ -350,7 +357,7 @@ class AsyncNcSessionBasic(NcSessionBase, ABC):
|
|
|
350
357
|
and ocs_meta["statuscode"] == 403
|
|
351
358
|
and str(ocs_meta["message"]).lower().find("password confirmation is required") != -1
|
|
352
359
|
):
|
|
353
|
-
await self.adapter.
|
|
360
|
+
await self.adapter.close()
|
|
354
361
|
self.init_adapter(restart=True)
|
|
355
362
|
return await self.ocs(
|
|
356
363
|
method, path, **kwargs, content=content, json=json, params=params, nested_req=True
|
|
@@ -408,18 +415,18 @@ class AsyncNcSessionBasic(NcSessionBase, ABC):
|
|
|
408
415
|
return {
|
|
409
416
|
"base_url": self.cfg.dav_endpoint,
|
|
410
417
|
"timeout": self.cfg.options.timeout_dav,
|
|
411
|
-
"event_hooks": {"
|
|
418
|
+
"event_hooks": {"pre_request": [], "response": [self._response_event]},
|
|
412
419
|
}
|
|
413
420
|
return {
|
|
414
421
|
"base_url": self.cfg.endpoint,
|
|
415
422
|
"timeout": self.cfg.options.timeout,
|
|
416
|
-
"event_hooks": {"
|
|
423
|
+
"event_hooks": {"pre_request": [self._request_event_ocs], "response": [self._response_event]},
|
|
417
424
|
}
|
|
418
425
|
|
|
419
426
|
async def _request_event_ocs(self, request: Request) -> None:
|
|
420
427
|
str_url = str(request.url)
|
|
421
428
|
if re.search(self._ocs_regexp, str_url) is not None: # this is OCS call
|
|
422
|
-
request.url = request.url
|
|
429
|
+
request.url = patch_param(request.url, "format", "json")
|
|
423
430
|
request.headers["Accept"] = "application/json"
|
|
424
431
|
|
|
425
432
|
async def _response_event(self, response: Response) -> None:
|
|
@@ -432,10 +439,12 @@ class AsyncNcSessionBasic(NcSessionBase, ABC):
|
|
|
432
439
|
|
|
433
440
|
async def download2fp(self, url_path: str, fp, dav: bool, params=None, **kwargs):
|
|
434
441
|
adapter = self.adapter_dav if dav else self.adapter
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
442
|
+
response = await adapter.get(url_path, params=params, headers=kwargs.get("headers"), stream=True)
|
|
443
|
+
|
|
444
|
+
check_error(response)
|
|
445
|
+
|
|
446
|
+
async for data_chunk in await response.iter_raw(chunk_size=kwargs.get("chunk_size", -1)):
|
|
447
|
+
fp.write(data_chunk)
|
|
439
448
|
|
|
440
449
|
|
|
441
450
|
class NcSession(NcSessionBasic):
|
|
@@ -445,15 +454,20 @@ class NcSession(NcSessionBasic):
|
|
|
445
454
|
self.cfg = Config(**kwargs)
|
|
446
455
|
super().__init__()
|
|
447
456
|
|
|
448
|
-
def _create_adapter(self, dav: bool = False) ->
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
auth=self.cfg.auth,
|
|
457
|
+
def _create_adapter(self, dav: bool = False) -> AsyncSession | Session:
|
|
458
|
+
session_kwargs = self._get_adapter_kwargs(dav)
|
|
459
|
+
hooks = session_kwargs.pop("event_hooks")
|
|
460
|
+
|
|
461
|
+
session = Session(
|
|
462
|
+
keepalive_delay=self.limits.keepalive_expiry, pool_maxsize=self.limits.max_connections, **session_kwargs
|
|
455
463
|
)
|
|
456
464
|
|
|
465
|
+
session.auth = self.cfg.auth
|
|
466
|
+
session.verify = self.cfg.options.nc_cert
|
|
467
|
+
session.hooks.update(hooks)
|
|
468
|
+
|
|
469
|
+
return session
|
|
470
|
+
|
|
457
471
|
|
|
458
472
|
class AsyncNcSession(AsyncNcSessionBasic):
|
|
459
473
|
cfg: Config
|
|
@@ -462,21 +476,28 @@ class AsyncNcSession(AsyncNcSessionBasic):
|
|
|
462
476
|
self.cfg = Config(**kwargs)
|
|
463
477
|
super().__init__()
|
|
464
478
|
|
|
465
|
-
def _create_adapter(self, dav: bool = False) ->
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
479
|
+
def _create_adapter(self, dav: bool = False) -> AsyncSession | Session:
|
|
480
|
+
session_kwargs = self._get_adapter_kwargs(dav)
|
|
481
|
+
hooks = session_kwargs.pop("event_hooks")
|
|
482
|
+
|
|
483
|
+
session = AsyncSession(
|
|
484
|
+
keepalive_delay=self.limits.keepalive_expiry,
|
|
485
|
+
pool_maxsize=self.limits.max_connections,
|
|
486
|
+
**session_kwargs,
|
|
472
487
|
)
|
|
473
488
|
|
|
489
|
+
session.verify = self.cfg.options.nc_cert
|
|
490
|
+
session.auth = self.cfg.auth
|
|
491
|
+
session.hooks.update(hooks)
|
|
492
|
+
|
|
493
|
+
return session
|
|
494
|
+
|
|
474
495
|
|
|
475
496
|
class NcSessionAppBasic(ABC):
|
|
476
497
|
cfg: AppConfig
|
|
477
498
|
_user: str
|
|
478
|
-
adapter:
|
|
479
|
-
adapter_dav:
|
|
499
|
+
adapter: AsyncSession | Session
|
|
500
|
+
adapter_dav: AsyncSession | Session
|
|
480
501
|
|
|
481
502
|
def __init__(self, **kwargs):
|
|
482
503
|
self.cfg = AppConfig(**kwargs)
|
|
@@ -505,22 +526,29 @@ class NcSessionAppBasic(ABC):
|
|
|
505
526
|
class NcSessionApp(NcSessionAppBasic, NcSessionBasic):
|
|
506
527
|
cfg: AppConfig
|
|
507
528
|
|
|
508
|
-
def _create_adapter(self, dav: bool = False) ->
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
"EX-APP-ID": self.cfg.app_name,
|
|
519
|
-
"EX-APP-VERSION": self.cfg.app_version,
|
|
520
|
-
"user-agent": f"ExApp/{self.cfg.app_name}/{self.cfg.app_version} (httpx/{httpx_version})",
|
|
521
|
-
},
|
|
529
|
+
def _create_adapter(self, dav: bool = False) -> AsyncSession | Session:
|
|
530
|
+
session_kwargs = self._get_adapter_kwargs(dav)
|
|
531
|
+
session_kwargs["event_hooks"]["pre_request"].append(self._add_auth)
|
|
532
|
+
|
|
533
|
+
hooks = session_kwargs.pop("event_hooks")
|
|
534
|
+
|
|
535
|
+
session = Session(
|
|
536
|
+
keepalive_delay=self.limits.keepalive_expiry,
|
|
537
|
+
pool_maxsize=self.limits.max_connections,
|
|
538
|
+
**session_kwargs,
|
|
522
539
|
)
|
|
523
540
|
|
|
541
|
+
session.verify = self.cfg.options.nc_cert
|
|
542
|
+
session.headers = {
|
|
543
|
+
"AA-VERSION": self.cfg.aa_version,
|
|
544
|
+
"EX-APP-ID": self.cfg.app_name,
|
|
545
|
+
"EX-APP-VERSION": self.cfg.app_version,
|
|
546
|
+
"user-agent": f"ExApp/{self.cfg.app_name}/{self.cfg.app_version} (niquests/{niquests_version})",
|
|
547
|
+
}
|
|
548
|
+
session.hooks.update(hooks)
|
|
549
|
+
|
|
550
|
+
return session
|
|
551
|
+
|
|
524
552
|
def _add_auth(self, request: Request):
|
|
525
553
|
request.headers.update(
|
|
526
554
|
{"AUTHORIZATION-APP-API": b64encode(f"{self._user}:{self.cfg.app_secret}".encode("UTF=8"))}
|
|
@@ -530,23 +558,39 @@ class NcSessionApp(NcSessionAppBasic, NcSessionBasic):
|
|
|
530
558
|
class AsyncNcSessionApp(NcSessionAppBasic, AsyncNcSessionBasic):
|
|
531
559
|
cfg: AppConfig
|
|
532
560
|
|
|
533
|
-
def _create_adapter(self, dav: bool = False) ->
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
"EX-APP-ID": self.cfg.app_name,
|
|
544
|
-
"EX-APP-VERSION": self.cfg.app_version,
|
|
545
|
-
"User-Agent": f"ExApp/{self.cfg.app_name}/{self.cfg.app_version} (httpx/{httpx_version})",
|
|
546
|
-
},
|
|
561
|
+
def _create_adapter(self, dav: bool = False) -> AsyncSession | Session:
|
|
562
|
+
session_kwargs = self._get_adapter_kwargs(dav)
|
|
563
|
+
session_kwargs["event_hooks"]["pre_request"].append(self._add_auth)
|
|
564
|
+
|
|
565
|
+
hooks = session_kwargs.pop("event_hooks")
|
|
566
|
+
|
|
567
|
+
session = AsyncSession(
|
|
568
|
+
keepalive_delay=self.limits.keepalive_expiry,
|
|
569
|
+
pool_maxsize=self.limits.max_connections,
|
|
570
|
+
**session_kwargs,
|
|
547
571
|
)
|
|
572
|
+
session.verify = self.cfg.options.nc_cert
|
|
573
|
+
session.headers = {
|
|
574
|
+
"AA-VERSION": self.cfg.aa_version,
|
|
575
|
+
"EX-APP-ID": self.cfg.app_name,
|
|
576
|
+
"EX-APP-VERSION": self.cfg.app_version,
|
|
577
|
+
"User-Agent": f"ExApp/{self.cfg.app_name}/{self.cfg.app_version} (niquests/{niquests_version})",
|
|
578
|
+
}
|
|
579
|
+
session.hooks.update(hooks)
|
|
580
|
+
|
|
581
|
+
return session
|
|
548
582
|
|
|
549
583
|
async def _add_auth(self, request: Request):
|
|
550
584
|
request.headers.update(
|
|
551
585
|
{"AUTHORIZATION-APP-API": b64encode(f"{self._user}:{self.cfg.app_secret}".encode("UTF=8"))}
|
|
552
586
|
)
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def patch_param(url: str, key: str, value: str) -> str:
|
|
590
|
+
parts = urlsplit(url)
|
|
591
|
+
query = dict(parse_qsl(parts.query, keep_blank_values=True))
|
|
592
|
+
query[key] = value
|
|
593
|
+
|
|
594
|
+
new_query = urlencode(query, doseq=True)
|
|
595
|
+
|
|
596
|
+
return urlunsplit((parts.scheme, parts.netloc, parts.path, new_query, parts.fragment))
|
|
@@ -23,7 +23,7 @@ try:
|
|
|
23
23
|
if body:
|
|
24
24
|
body = body.replace(b"\n", b"\r\n").replace(b"\r\r\n", b"\r\n")
|
|
25
25
|
r = self._session.adapter_dav.request(
|
|
26
|
-
method, url if isinstance(url, str) else str(url),
|
|
26
|
+
method, url if isinstance(url, str) else str(url), data=body, headers=headers
|
|
27
27
|
)
|
|
28
28
|
return DAVResponse(r)
|
|
29
29
|
|
|
@@ -7,9 +7,10 @@ import hashlib
|
|
|
7
7
|
import json
|
|
8
8
|
import os
|
|
9
9
|
import typing
|
|
10
|
+
from traceback import format_exc
|
|
10
11
|
from urllib.parse import urlparse
|
|
11
12
|
|
|
12
|
-
import
|
|
13
|
+
import niquests
|
|
13
14
|
from fastapi import (
|
|
14
15
|
BackgroundTasks,
|
|
15
16
|
Depends,
|
|
@@ -22,10 +23,10 @@ from fastapi.responses import JSONResponse, PlainTextResponse
|
|
|
22
23
|
from starlette.requests import HTTPConnection, Request
|
|
23
24
|
from starlette.types import ASGIApp, Receive, Scope, Send
|
|
24
25
|
|
|
26
|
+
from .._exceptions import ModelFetchError
|
|
25
27
|
from .._misc import get_username_secret_from_headers
|
|
26
28
|
from ..nextcloud import AsyncNextcloudApp, NextcloudApp
|
|
27
29
|
from ..talk_bot import TalkBotMessage
|
|
28
|
-
from .defs import LogLvl
|
|
29
30
|
from .misc import persistent_storage
|
|
30
31
|
|
|
31
32
|
|
|
@@ -70,9 +71,24 @@ def set_handlers(
|
|
|
70
71
|
|
|
71
72
|
.. note:: When this parameter is ``False``, the provision of ``models_to_fetch`` is not allowed.
|
|
72
73
|
|
|
73
|
-
:param models_to_fetch: Dictionary describing which models should be downloaded during `init
|
|
74
|
+
:param models_to_fetch: Dictionary describing which models should be downloaded during `init` of the form:
|
|
75
|
+
.. code-block:: python
|
|
76
|
+
{
|
|
77
|
+
"model_url_1": {
|
|
78
|
+
"save_path": "path_or_filename_to_save_the_model_to",
|
|
79
|
+
},
|
|
80
|
+
"huggingface_model_name_1": {
|
|
81
|
+
"max_workers": 4,
|
|
82
|
+
"cache_dir": "path_to_cache_dir",
|
|
83
|
+
"revision": "revision_to_fetch",
|
|
84
|
+
...
|
|
85
|
+
},
|
|
86
|
+
...
|
|
87
|
+
}
|
|
88
|
+
|
|
74
89
|
|
|
75
90
|
.. note:: ``huggingface_hub`` package should be present for automatic models fetching.
|
|
91
|
+
All model options are optional and can be left empty.
|
|
76
92
|
|
|
77
93
|
:param map_app_static: Should be folders ``js``, ``css``, ``l10n``, ``img`` automatically mounted in FastAPI or not.
|
|
78
94
|
|
|
@@ -121,73 +137,98 @@ def __map_app_static_folders(fast_api_app: FastAPI):
|
|
|
121
137
|
|
|
122
138
|
|
|
123
139
|
def fetch_models_task(nc: NextcloudApp, models: dict[str, dict], progress_init_start_value: int) -> None:
|
|
124
|
-
"""Use for cases when you want to define custom `/init` but still need to easy download models.
|
|
140
|
+
"""Use for cases when you want to define custom `/init` but still need to easy download models.
|
|
141
|
+
|
|
142
|
+
:param nc: NextcloudApp instance.
|
|
143
|
+
:param models_to_fetch: Dictionary describing which models should be downloaded of the form:
|
|
144
|
+
.. code-block:: python
|
|
145
|
+
{
|
|
146
|
+
"model_url_1": {
|
|
147
|
+
"save_path": "path_or_filename_to_save_the_model_to",
|
|
148
|
+
},
|
|
149
|
+
"huggingface_model_name_1": {
|
|
150
|
+
"max_workers": 4,
|
|
151
|
+
"cache_dir": "path_to_cache_dir",
|
|
152
|
+
"revision": "revision_to_fetch",
|
|
153
|
+
...
|
|
154
|
+
},
|
|
155
|
+
...
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
.. note:: ``huggingface_hub`` package should be present for automatic models fetching.
|
|
159
|
+
All model options are optional and can be left empty.
|
|
160
|
+
|
|
161
|
+
:param progress_init_start_value: Integer value defining from which percent the progress should start.
|
|
162
|
+
|
|
163
|
+
:raises ModelFetchError: in case of a model download error.
|
|
164
|
+
:raises NextcloudException: in case of a network error reaching the Nextcloud server.
|
|
165
|
+
"""
|
|
125
166
|
if models:
|
|
126
167
|
current_progress = progress_init_start_value
|
|
127
168
|
percent_for_each = min(int((100 - progress_init_start_value) / len(models)), 99)
|
|
128
169
|
for model in models:
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
170
|
+
try:
|
|
171
|
+
if model.startswith(("http://", "https://")):
|
|
172
|
+
models[model]["path"] = __fetch_model_as_file(
|
|
173
|
+
current_progress, percent_for_each, nc, model, models[model]
|
|
174
|
+
)
|
|
175
|
+
else:
|
|
176
|
+
models[model]["path"] = __fetch_model_as_snapshot(
|
|
177
|
+
current_progress, percent_for_each, nc, model, models[model]
|
|
178
|
+
)
|
|
179
|
+
current_progress += percent_for_each
|
|
180
|
+
except BaseException as e: # noqa pylint: disable=broad-exception-caught
|
|
181
|
+
nc.set_init_status(current_progress, f"Downloading of '{model}' failed: {e}: {format_exc()}")
|
|
182
|
+
raise ModelFetchError(f"Downloading of '{model}' failed.") from e
|
|
138
183
|
nc.set_init_status(100)
|
|
139
184
|
|
|
140
185
|
|
|
141
186
|
def __fetch_model_as_file(
|
|
142
187
|
current_progress: int, progress_for_task: int, nc: NextcloudApp, model_path: str, download_options: dict
|
|
143
|
-
) -> str
|
|
188
|
+
) -> str:
|
|
144
189
|
result_path = download_options.pop("save_path", urlparse(model_path).path.split("/")[-1])
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
except Exception as e: # noqa pylint: disable=broad-exception-caught
|
|
184
|
-
nc.log(LogLvl.ERROR, f"Downloading of '{model_path}' raised an exception: {e}")
|
|
185
|
-
|
|
186
|
-
return None
|
|
190
|
+
with niquests.get(model_path, stream=True) as response:
|
|
191
|
+
if not response.ok:
|
|
192
|
+
raise ModelFetchError(
|
|
193
|
+
f"Downloading of '{model_path}' failed, returned ({response.status_code}) {response.text}"
|
|
194
|
+
)
|
|
195
|
+
downloaded_size = 0
|
|
196
|
+
linked_etag = ""
|
|
197
|
+
for each_history in response.history:
|
|
198
|
+
linked_etag = each_history.headers.get("X-Linked-ETag", "")
|
|
199
|
+
if linked_etag:
|
|
200
|
+
break
|
|
201
|
+
if not linked_etag:
|
|
202
|
+
linked_etag = response.headers.get("X-Linked-ETag", response.headers.get("ETag", ""))
|
|
203
|
+
total_size = int(response.headers.get("Content-Length"))
|
|
204
|
+
try:
|
|
205
|
+
existing_size = os.path.getsize(result_path)
|
|
206
|
+
except OSError:
|
|
207
|
+
existing_size = 0
|
|
208
|
+
if linked_etag and total_size == existing_size:
|
|
209
|
+
with builtins.open(result_path, "rb") as file:
|
|
210
|
+
sha256_hash = hashlib.sha256()
|
|
211
|
+
for byte_block in iter(lambda: file.read(4096), b""):
|
|
212
|
+
sha256_hash.update(byte_block)
|
|
213
|
+
if f'"{sha256_hash.hexdigest()}"' == linked_etag:
|
|
214
|
+
nc.set_init_status(min(current_progress + progress_for_task, 99))
|
|
215
|
+
return result_path
|
|
216
|
+
|
|
217
|
+
with builtins.open(result_path, "wb") as file:
|
|
218
|
+
last_progress = current_progress
|
|
219
|
+
for chunk in response.iter_raw(-1):
|
|
220
|
+
downloaded_size += file.write(chunk)
|
|
221
|
+
if total_size:
|
|
222
|
+
new_progress = min(current_progress + int(progress_for_task * downloaded_size / total_size), 99)
|
|
223
|
+
if new_progress != last_progress:
|
|
224
|
+
nc.set_init_status(new_progress)
|
|
225
|
+
last_progress = new_progress
|
|
226
|
+
|
|
227
|
+
return result_path
|
|
187
228
|
|
|
188
229
|
|
|
189
230
|
def __fetch_model_as_snapshot(
|
|
190
|
-
current_progress: int, progress_for_task, nc: NextcloudApp,
|
|
231
|
+
current_progress: int, progress_for_task, nc: NextcloudApp, model_name: str, download_options: dict
|
|
191
232
|
) -> str:
|
|
192
233
|
from huggingface_hub import snapshot_download # noqa isort:skip pylint: disable=C0415 disable=E0401
|
|
193
234
|
from tqdm import tqdm # noqa isort:skip pylint: disable=C0415 disable=E0401
|
|
@@ -200,7 +241,7 @@ def __fetch_model_as_snapshot(
|
|
|
200
241
|
workers = download_options.pop("max_workers", 2)
|
|
201
242
|
cache = download_options.pop("cache_dir", persistent_storage())
|
|
202
243
|
return snapshot_download(
|
|
203
|
-
|
|
244
|
+
model_name, tqdm_class=TqdmProgress, **download_options, max_workers=workers, cache_dir=cache
|
|
204
245
|
)
|
|
205
246
|
|
|
206
247
|
|
|
@@ -9,7 +9,7 @@ from urllib.parse import unquote
|
|
|
9
9
|
from xml.etree import ElementTree
|
|
10
10
|
|
|
11
11
|
import xmltodict
|
|
12
|
-
from
|
|
12
|
+
from niquests import Response
|
|
13
13
|
|
|
14
14
|
from .._exceptions import NextcloudException, check_error
|
|
15
15
|
from .._misc import check_capabilities, clear_from_params_empty
|