omlish 0.0.0.dev102__py3-none-any.whl → 0.0.0.dev104__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omlish/__about__.py +2 -2
- omlish/concurrent/threadlets.py +3 -0
- omlish/diag/pycharm/pycharm.py +33 -0
- omlish/fnpairs.py +22 -12
- omlish/formats/json/cli/cli.py +35 -1
- omlish/formats/json/cli/formats.py +7 -0
- omlish/formats/json/cli/io.py +74 -0
- omlish/http/consts.py +2 -0
- omlish/http/jwt.py +179 -0
- omlish/io/__init__.py +3 -0
- omlish/io/pyio.py +2757 -0
- omlish/io/trampoline.py +293 -0
- omlish/lite/cached.py +9 -1
- omlish/lite/contextmanagers.py +34 -0
- omlish/lite/marshal.py +72 -29
- omlish/lite/pidfile.py +1 -1
- omlish/lite/subprocesses.py +18 -0
- omlish/specs/__init__.py +2 -0
- omlish/sync.py +55 -0
- {omlish-0.0.0.dev102.dist-info → omlish-0.0.0.dev104.dist-info}/METADATA +1 -1
- {omlish-0.0.0.dev102.dist-info → omlish-0.0.0.dev104.dist-info}/RECORD +25 -20
- {omlish-0.0.0.dev102.dist-info → omlish-0.0.0.dev104.dist-info}/LICENSE +0 -0
- {omlish-0.0.0.dev102.dist-info → omlish-0.0.0.dev104.dist-info}/WHEEL +0 -0
- {omlish-0.0.0.dev102.dist-info → omlish-0.0.0.dev104.dist-info}/entry_points.txt +0 -0
- {omlish-0.0.0.dev102.dist-info → omlish-0.0.0.dev104.dist-info}/top_level.txt +0 -0
omlish/__about__.py
CHANGED
omlish/concurrent/threadlets.py
CHANGED
omlish/diag/pycharm/pycharm.py
CHANGED
@@ -1,7 +1,9 @@
|
|
1
1
|
import dataclasses as dc
|
2
|
+
import json
|
2
3
|
import os.path
|
3
4
|
import plistlib
|
4
5
|
import re
|
6
|
+
import shutil
|
5
7
|
import subprocess
|
6
8
|
import sys
|
7
9
|
import typing as ta
|
@@ -43,6 +45,26 @@ def read_darwin_pycharm_info_plist() -> ta.Mapping[str, ta.Any] | None:
|
|
43
45
|
return root
|
44
46
|
|
45
47
|
|
48
|
+
#
|
49
|
+
|
50
|
+
|
51
|
+
UBUNTU_PYCHARM_HOME = '/snap/pycharm-professional/current'
|
52
|
+
|
53
|
+
|
54
|
+
def read_ubuntu_pycharm_product_info() -> ta.Mapping[str, ta.Any] | None:
|
55
|
+
json_file = os.path.join(UBUNTU_PYCHARM_HOME, 'product-info.json')
|
56
|
+
if not os.path.isfile(json_file):
|
57
|
+
return None
|
58
|
+
|
59
|
+
with open(json_file) as f:
|
60
|
+
root = json.load(f)
|
61
|
+
|
62
|
+
return root
|
63
|
+
|
64
|
+
|
65
|
+
#
|
66
|
+
|
67
|
+
|
46
68
|
@lang.cached_function
|
47
69
|
def get_pycharm_version() -> str | None:
|
48
70
|
if sys.platform == 'darwin':
|
@@ -54,6 +76,17 @@ def get_pycharm_version() -> str | None:
|
|
54
76
|
check.state(ver.startswith('PY-'))
|
55
77
|
return ver[3:]
|
56
78
|
|
79
|
+
elif sys.platform == 'linux':
|
80
|
+
if shutil.which('lsb_release') is not None:
|
81
|
+
lsb_id = subprocess.check_output(['lsb_release', '-is']).decode().strip()
|
82
|
+
if lsb_id == 'Ubuntu':
|
83
|
+
pi = read_ubuntu_pycharm_product_info()
|
84
|
+
if pi is not None:
|
85
|
+
ver = check.non_empty_str(pi['buildNumber'])
|
86
|
+
return ver
|
87
|
+
|
88
|
+
return None
|
89
|
+
|
57
90
|
else:
|
58
91
|
return None
|
59
92
|
|
omlish/fnpairs.py
CHANGED
@@ -26,7 +26,6 @@ from . import lang
|
|
26
26
|
if ta.TYPE_CHECKING:
|
27
27
|
import bz2 as _bz2
|
28
28
|
import gzip as _gzip
|
29
|
-
import json as _json
|
30
29
|
import lzma as _lzma
|
31
30
|
import pickle as _pickle
|
32
31
|
import struct as _struct
|
@@ -40,10 +39,11 @@ if ta.TYPE_CHECKING:
|
|
40
39
|
import yaml as _yaml
|
41
40
|
import zstandard as _zstandard
|
42
41
|
|
42
|
+
from .formats import json as _json
|
43
|
+
|
43
44
|
else:
|
44
45
|
_bz2 = lang.proxy_import('bz2')
|
45
46
|
_gzip = lang.proxy_import('gzip')
|
46
|
-
_json = lang.proxy_import('json')
|
47
47
|
_lzma = lang.proxy_import('lzma')
|
48
48
|
_pickle = lang.proxy_import('pickle')
|
49
49
|
_struct = lang.proxy_import('struct')
|
@@ -57,6 +57,8 @@ else:
|
|
57
57
|
_yaml = lang.proxy_import('yaml')
|
58
58
|
_zstandard = lang.proxy_import('zstandard')
|
59
59
|
|
60
|
+
_json = lang.proxy_import('.formats.json', __package__)
|
61
|
+
|
60
62
|
|
61
63
|
##
|
62
64
|
|
@@ -277,7 +279,7 @@ def _register_extension(*ss):
|
|
277
279
|
def inner(cls):
|
278
280
|
for s in ss:
|
279
281
|
if s in _EXTENSION_REGISTRY:
|
280
|
-
raise
|
282
|
+
raise KeyError(s)
|
281
283
|
_EXTENSION_REGISTRY[s] = cls
|
282
284
|
return cls
|
283
285
|
return inner
|
@@ -409,22 +411,30 @@ class Pickle(ObjectBytes_):
|
|
409
411
|
return _pickle.loads(t)
|
410
412
|
|
411
413
|
|
414
|
+
class _Json(ObjectStr_, lang.Abstract): # noqa
|
415
|
+
def backward(self, t: str) -> ta.Any:
|
416
|
+
return _json.loads(t)
|
417
|
+
|
418
|
+
|
412
419
|
@_register_extension('json')
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
420
|
+
class Json(_Json):
|
421
|
+
def forward(self, f: ta.Any) -> str:
|
422
|
+
return _json.dumps(f)
|
423
|
+
|
417
424
|
|
425
|
+
class JsonPretty(_Json):
|
418
426
|
def forward(self, f: ta.Any) -> str:
|
419
|
-
return _json.
|
427
|
+
return _json.dumps_pretty(f)
|
420
428
|
|
421
|
-
|
422
|
-
|
429
|
+
|
430
|
+
class JsonCompact(_Json):
|
431
|
+
def forward(self, f: ta.Any) -> str:
|
432
|
+
return _json.dumps_compact(f)
|
423
433
|
|
424
434
|
|
425
435
|
JSON = Json()
|
426
|
-
PRETTY_JSON =
|
427
|
-
COMPACT_JSON =
|
436
|
+
PRETTY_JSON = JsonPretty()
|
437
|
+
COMPACT_JSON = JsonCompact()
|
428
438
|
|
429
439
|
|
430
440
|
@_register_extension('jsonl')
|
omlish/formats/json/cli/cli.py
CHANGED
@@ -1,7 +1,41 @@
|
|
1
1
|
"""
|
2
2
|
TODO:
|
3
|
-
- -I/-Ogz, lz4, etc - fnpairs
|
4
3
|
- read from http
|
4
|
+
- jmespath output flat, unquoted strs like jq '.[]'
|
5
|
+
|
6
|
+
==
|
7
|
+
|
8
|
+
jq Command options:
|
9
|
+
-n, --null-input use `null` as the single input value;
|
10
|
+
-R, --raw-input read each line as string instead of JSON;
|
11
|
+
-s, --slurp read all inputs into an array and use it as the single input value;
|
12
|
+
-c, --compact-output compact instead of pretty-printed output;
|
13
|
+
-r, --raw-output output strings without escapes and quotes;
|
14
|
+
--raw-output0 implies -r and output NUL after each output;
|
15
|
+
-j, --join-output implies -r and output without newline after each output;
|
16
|
+
-a, --ascii-output output strings by only ASCII characters using escape sequences;
|
17
|
+
-S, --sort-keys sort keys of each object on output;
|
18
|
+
-C, --color-output colorize JSON output;
|
19
|
+
-M, --monochrome-output disable colored output;
|
20
|
+
--tab use tabs for indentation;
|
21
|
+
--indent n use n spaces for indentation (max 7 spaces);
|
22
|
+
--unbuffered flush output stream after each output;
|
23
|
+
--stream parse the input value in streaming fashion;
|
24
|
+
--stream-errors implies --stream and report parse error as an array;
|
25
|
+
--seq parse input/output as application/json-seq;
|
26
|
+
-f, --from-file file load filter from the file;
|
27
|
+
-L directory search modules from the directory;
|
28
|
+
--arg name value set $name to the string value;
|
29
|
+
--argjson name value set $name to the JSON value;
|
30
|
+
--slurpfile name file set $name to an array of JSON values read from the file;
|
31
|
+
--rawfile name file set $name to string contents of file;
|
32
|
+
--args consume remaining arguments as positional string values;
|
33
|
+
--jsonargs consume remaining arguments as positional JSON values;
|
34
|
+
-e, --exit-status set exit status code based on the output;
|
35
|
+
-V, --version show the version;
|
36
|
+
--build-configuration show jq's build configuration;
|
37
|
+
-h, --help show the help;
|
38
|
+
-- terminates argument processing;
|
5
39
|
"""
|
6
40
|
import argparse
|
7
41
|
import codecs
|
@@ -44,12 +44,19 @@ class Format:
|
|
44
44
|
|
45
45
|
class Formats(enum.Enum):
|
46
46
|
JSON = Format(['json'], json.load)
|
47
|
+
|
47
48
|
YAML = Format(['yaml', 'yml'], lambda f: yaml.safe_load(f))
|
49
|
+
|
48
50
|
TOML = Format(['toml'], lambda f: tomllib.loads(f.read()))
|
51
|
+
|
49
52
|
ENV = Format(['env', 'dotenv'], lambda f: dotenv.dotenv_values(stream=f))
|
53
|
+
|
50
54
|
PROPS = Format(['properties', 'props'], lambda f: dict(props.Properties().load(f.read())))
|
55
|
+
|
51
56
|
PY = Format(['py', 'python', 'repr'], lambda f: ast.literal_eval(f.read()))
|
57
|
+
|
52
58
|
XML = Format(['xml'], lambda f: xml.build_simple_element(xml.parse_tree(f.read()).getroot()).as_dict())
|
59
|
+
|
53
60
|
CSV = Format(['csv'], lambda f: list(csv.DictReader(f)))
|
54
61
|
TSV = Format(['tsv'], lambda f: list(csv.DictReader(f, delimiter='\t')))
|
55
62
|
FLAT_CSV = Format(['fcsv'], lambda f: list(csv.reader(f)))
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# """
|
2
|
+
# TODO:
|
3
|
+
# - -I/-Ogz, lz4, etc
|
4
|
+
# - fnpairs? or not yet just do it
|
5
|
+
# """
|
6
|
+
# import abc
|
7
|
+
# import contextlib
|
8
|
+
# import dataclasses as dc
|
9
|
+
# import gzip
|
10
|
+
# import os
|
11
|
+
# import typing as ta
|
12
|
+
#
|
13
|
+
# from .... import lang
|
14
|
+
#
|
15
|
+
#
|
16
|
+
# if ta.TYPE_CHECKING:
|
17
|
+
# import bz2 as _bz2
|
18
|
+
# import gzip as _gzip
|
19
|
+
# import lzma as _lzma
|
20
|
+
# else:
|
21
|
+
# _bz2 = lang.proxy_import('bz2')
|
22
|
+
# _gzip = lang.proxy_import('gzip')
|
23
|
+
# _lzma = lang.proxy_import('lzma')
|
24
|
+
#
|
25
|
+
#
|
26
|
+
# ##
|
27
|
+
#
|
28
|
+
#
|
29
|
+
# class BaseIo(lang.Abstract):
|
30
|
+
# def close(self) -> None:
|
31
|
+
# raise NotImplementedError
|
32
|
+
#
|
33
|
+
# def fileno(self) -> int | None:
|
34
|
+
# return None
|
35
|
+
#
|
36
|
+
#
|
37
|
+
# class Input(BaseIo):
|
38
|
+
# @abc.abstractmethod
|
39
|
+
# def read(self, sz: int | None = None) -> bytes:
|
40
|
+
# raise NotImplementedError
|
41
|
+
#
|
42
|
+
#
|
43
|
+
# class Output(BaseIo):
|
44
|
+
# @abc.abstractmethod
|
45
|
+
# def write(self, data: bytes) -> int:
|
46
|
+
# raise NotImplementedError
|
47
|
+
#
|
48
|
+
#
|
49
|
+
# #
|
50
|
+
#
|
51
|
+
#
|
52
|
+
# DEFAULT_READ_SZ = 0x4000
|
53
|
+
#
|
54
|
+
#
|
55
|
+
# @dc.dataclass(frozen=True)
|
56
|
+
# class FdIo(Input, Output):
|
57
|
+
# fd: int
|
58
|
+
#
|
59
|
+
# default_read_sz: int = DEFAULT_READ_SZ
|
60
|
+
#
|
61
|
+
# def read(self, sz: int | None = None) -> bytes:
|
62
|
+
# return os.read(self.fd, sz or self.default_read_sz)
|
63
|
+
#
|
64
|
+
# def write(self, data: bytes) -> int:
|
65
|
+
# return os.write(self.fd, data)
|
66
|
+
#
|
67
|
+
#
|
68
|
+
# ##
|
69
|
+
#
|
70
|
+
#
|
71
|
+
# @contextlib.contextmanager
|
72
|
+
# def gzip_io_codec(f: ta.IO, mode: str) -> ta.ContextManager[ta.IO]:
|
73
|
+
# with gzip.open(f, mode) as o:
|
74
|
+
# yield o
|
omlish/http/consts.py
CHANGED
@@ -42,6 +42,8 @@ CONTENT_CHARSET_UTF8 = b'charset=utf-8'
|
|
42
42
|
|
43
43
|
CONTENT_TYPE_BYTES = b'application/octet-stream'
|
44
44
|
|
45
|
+
CONTENT_TYPE_FORM_URLENCODED = b'application/x-www-form-urlencoded'
|
46
|
+
|
45
47
|
CONTENT_TYPE_HTML = b'text/html'
|
46
48
|
CONTENT_TYPE_HTML_UTF8 = b'; '.join([CONTENT_TYPE_HTML, CONTENT_CHARSET_UTF8])
|
47
49
|
|
omlish/http/jwt.py
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
import abc
|
2
|
+
import base64
|
3
|
+
import hashlib
|
4
|
+
import hmac
|
5
|
+
import json
|
6
|
+
import typing as ta
|
7
|
+
|
8
|
+
from .. import lang
|
9
|
+
|
10
|
+
|
11
|
+
if ta.TYPE_CHECKING:
|
12
|
+
import cryptography.hazmat.primitives.asymmetric.padding
|
13
|
+
import cryptography.hazmat.primitives.hashes
|
14
|
+
import cryptography.hazmat.primitives.serialization
|
15
|
+
else:
|
16
|
+
cryptography = lang.proxy_import('cryptography', extras=[
|
17
|
+
'hazmat.primitives.asymmetric.padding',
|
18
|
+
'hazmat.primitives.hashes',
|
19
|
+
'hazmat.primitives.serialization',
|
20
|
+
])
|
21
|
+
|
22
|
+
|
23
|
+
##
|
24
|
+
|
25
|
+
|
26
|
+
def as_bytes(v: str | bytes, encoding: str = 'utf-8') -> bytes:
|
27
|
+
return v.encode(encoding) if isinstance(v, str) else v
|
28
|
+
|
29
|
+
|
30
|
+
def base64url_encode(b: bytes) -> bytes:
|
31
|
+
return base64.urlsafe_b64encode(b).replace(b'=', b'')
|
32
|
+
|
33
|
+
|
34
|
+
##
|
35
|
+
|
36
|
+
|
37
|
+
class Algorithm(abc.ABC):
|
38
|
+
@property
|
39
|
+
@abc.abstractmethod
|
40
|
+
def name(self) -> str:
|
41
|
+
raise NotImplementedError
|
42
|
+
|
43
|
+
@abc.abstractmethod
|
44
|
+
def prepare_key(self, key: str | bytes) -> ta.Any:
|
45
|
+
raise NotImplementedError
|
46
|
+
|
47
|
+
@abc.abstractmethod
|
48
|
+
def sign(self, msg: str | bytes, key: ta.Any) -> bytes:
|
49
|
+
raise NotImplementedError
|
50
|
+
|
51
|
+
|
52
|
+
class HmacAlgorithm(Algorithm):
|
53
|
+
def __init__(self, name: str, digest: ta.Any) -> None:
|
54
|
+
super().__init__()
|
55
|
+
self._name = name
|
56
|
+
self._digest = digest
|
57
|
+
|
58
|
+
@property
|
59
|
+
def name(self) -> str:
|
60
|
+
return self._name
|
61
|
+
|
62
|
+
def prepare_key(self, key: str | bytes) -> ta.Any:
|
63
|
+
return as_bytes(key)
|
64
|
+
|
65
|
+
def sign(self, msg: str | bytes, key: ta.Any) -> bytes:
|
66
|
+
return hmac.new(key, as_bytes(msg), self._digest).digest()
|
67
|
+
|
68
|
+
|
69
|
+
class RsaAlgorithm(Algorithm):
|
70
|
+
def __init__(self, name: str, digest: str) -> None:
|
71
|
+
super().__init__()
|
72
|
+
self._name = name
|
73
|
+
self._digest = digest
|
74
|
+
|
75
|
+
@property
|
76
|
+
def name(self) -> str:
|
77
|
+
return self._name
|
78
|
+
|
79
|
+
def prepare_key(self, key: str | bytes) -> ta.Any:
|
80
|
+
if (key_bytes := as_bytes(key)).startswith(b'ssh-rsa'):
|
81
|
+
return cryptography.hazmat.primitives.serialization.load_ssh_public_key(key_bytes)
|
82
|
+
else:
|
83
|
+
return cryptography.hazmat.primitives.serialization.load_pem_private_key(key_bytes, password=None)
|
84
|
+
|
85
|
+
def sign(self, msg: str | bytes, key: ta.Any) -> bytes:
|
86
|
+
return key.sign(
|
87
|
+
as_bytes(msg),
|
88
|
+
cryptography.hazmat.primitives.asymmetric.padding.PKCS1v15(),
|
89
|
+
getattr(cryptography.hazmat.primitives.hashes, self._digest)(),
|
90
|
+
)
|
91
|
+
|
92
|
+
|
93
|
+
ALGORITHMS_BY_NAME = {
|
94
|
+
a.name: a for a in [
|
95
|
+
HmacAlgorithm('HS256', hashlib.sha256),
|
96
|
+
HmacAlgorithm('HS384', hashlib.sha384),
|
97
|
+
HmacAlgorithm('HS512', hashlib.sha512),
|
98
|
+
|
99
|
+
RsaAlgorithm('RS256', 'SHA256'),
|
100
|
+
RsaAlgorithm('RS384', 'SHA384'),
|
101
|
+
RsaAlgorithm('RS512', 'SHA512'),
|
102
|
+
]
|
103
|
+
}
|
104
|
+
|
105
|
+
|
106
|
+
##
|
107
|
+
|
108
|
+
|
109
|
+
def jwt_encode(
|
110
|
+
payload: ta.Mapping[str, ta.Any],
|
111
|
+
key: str | bytes,
|
112
|
+
*,
|
113
|
+
algorithm: str = 'HS256',
|
114
|
+
) -> str:
|
115
|
+
alg = ALGORITHMS_BY_NAME[algorithm]
|
116
|
+
|
117
|
+
segments: list[bytes] = []
|
118
|
+
|
119
|
+
header: dict[str, ta.Any] = {
|
120
|
+
'typ': 'jwt',
|
121
|
+
'alg': alg.name,
|
122
|
+
}
|
123
|
+
json_header = json.dumps(header, separators=(',', ':'), sort_keys=True).encode('utf-8')
|
124
|
+
segments.append(base64url_encode(json_header))
|
125
|
+
|
126
|
+
json_payload = json.dumps(payload, separators=(',', ':')).encode('utf-8')
|
127
|
+
msg_payload = base64url_encode(json_payload)
|
128
|
+
segments.append(msg_payload)
|
129
|
+
|
130
|
+
signing_input = b'.'.join(segments)
|
131
|
+
|
132
|
+
key = alg.prepare_key(key)
|
133
|
+
signature = alg.sign(signing_input, key)
|
134
|
+
|
135
|
+
segments.append(base64url_encode(signature))
|
136
|
+
|
137
|
+
encoded_string = b'.'.join(segments)
|
138
|
+
return encoded_string.decode('utf-8')
|
139
|
+
|
140
|
+
|
141
|
+
def generate_jwt(
|
142
|
+
*,
|
143
|
+
issuer: str,
|
144
|
+
subject: str,
|
145
|
+
audience: str,
|
146
|
+
issued_at: int,
|
147
|
+
expires_at: int,
|
148
|
+
scope: str,
|
149
|
+
key: str | bytes,
|
150
|
+
**kwargs: ta.Any,
|
151
|
+
) -> str:
|
152
|
+
payload = {
|
153
|
+
'iss': issuer,
|
154
|
+
'sub': subject,
|
155
|
+
'aud': audience,
|
156
|
+
'iat': issued_at,
|
157
|
+
'exp': expires_at,
|
158
|
+
'scope': scope,
|
159
|
+
}
|
160
|
+
|
161
|
+
return jwt_encode(
|
162
|
+
payload,
|
163
|
+
key,
|
164
|
+
**kwargs,
|
165
|
+
)
|
166
|
+
|
167
|
+
|
168
|
+
##
|
169
|
+
|
170
|
+
|
171
|
+
SERVICE_APPLICATION_GRANT_TYPE = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
172
|
+
|
173
|
+
|
174
|
+
def build_get_token_body(
|
175
|
+
signed_jwt: str,
|
176
|
+
*,
|
177
|
+
grant_type: str = SERVICE_APPLICATION_GRANT_TYPE,
|
178
|
+
) -> str:
|
179
|
+
return f'grant_type={grant_type}&assertion={signed_jwt}'
|
omlish/io/__init__.py
ADDED