django-nativemojo 0.1.10__py3-none-any.whl → 0.1.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_nativemojo-0.1.15.dist-info/METADATA +136 -0
- {django_nativemojo-0.1.10.dist-info → django_nativemojo-0.1.15.dist-info}/RECORD +105 -65
- mojo/__init__.py +1 -1
- mojo/apps/account/management/__init__.py +5 -0
- mojo/apps/account/management/commands/__init__.py +6 -0
- mojo/apps/account/management/commands/serializer_admin.py +531 -0
- mojo/apps/account/migrations/0004_user_avatar.py +20 -0
- mojo/apps/account/migrations/0005_group_last_activity.py +18 -0
- mojo/apps/account/models/group.py +25 -7
- mojo/apps/account/models/member.py +15 -4
- mojo/apps/account/models/user.py +197 -20
- mojo/apps/account/rest/group.py +1 -0
- mojo/apps/account/rest/user.py +6 -2
- mojo/apps/aws/rest/__init__.py +1 -0
- mojo/apps/aws/rest/s3.py +64 -0
- mojo/apps/fileman/README.md +8 -8
- mojo/apps/fileman/backends/base.py +76 -70
- mojo/apps/fileman/backends/filesystem.py +86 -86
- mojo/apps/fileman/backends/s3.py +200 -108
- mojo/apps/fileman/migrations/0001_initial.py +106 -0
- mojo/apps/fileman/migrations/0002_filemanager_parent_alter_filemanager_max_file_size.py +24 -0
- mojo/apps/fileman/migrations/0003_remove_file_fileman_fil_upload__c4bc35_idx_and_more.py +25 -0
- mojo/apps/fileman/migrations/0004_remove_file_original_filename_and_more.py +39 -0
- mojo/apps/fileman/migrations/0005_alter_file_upload_token.py +18 -0
- mojo/apps/fileman/migrations/0006_file_download_url_filemanager_forever_urls.py +23 -0
- mojo/apps/fileman/migrations/0007_remove_filemanager_forever_urls_and_more.py +22 -0
- mojo/apps/fileman/migrations/0008_file_category.py +18 -0
- mojo/apps/fileman/migrations/0009_rename_file_path_file_storage_file_path.py +18 -0
- mojo/apps/fileman/migrations/0010_filerendition.py +33 -0
- mojo/apps/fileman/migrations/0011_alter_filerendition_original_file.py +19 -0
- mojo/apps/fileman/models/__init__.py +1 -5
- mojo/apps/fileman/models/file.py +204 -58
- mojo/apps/fileman/models/manager.py +161 -31
- mojo/apps/fileman/models/rendition.py +118 -0
- mojo/apps/fileman/renderer/__init__.py +111 -0
- mojo/apps/fileman/renderer/audio.py +403 -0
- mojo/apps/fileman/renderer/base.py +205 -0
- mojo/apps/fileman/renderer/document.py +404 -0
- mojo/apps/fileman/renderer/image.py +222 -0
- mojo/apps/fileman/renderer/utils.py +297 -0
- mojo/apps/fileman/renderer/video.py +304 -0
- mojo/apps/fileman/rest/__init__.py +1 -18
- mojo/apps/fileman/rest/upload.py +22 -32
- mojo/apps/fileman/signals.py +58 -0
- mojo/apps/fileman/tasks.py +254 -0
- mojo/apps/fileman/utils/__init__.py +40 -16
- mojo/apps/incident/migrations/0005_incidenthistory.py +39 -0
- mojo/apps/incident/migrations/0006_alter_incident_state.py +18 -0
- mojo/apps/incident/models/__init__.py +1 -0
- mojo/apps/incident/models/history.py +36 -0
- mojo/apps/incident/models/incident.py +1 -1
- mojo/apps/incident/reporter.py +3 -1
- mojo/apps/incident/rest/event.py +7 -1
- mojo/apps/logit/migrations/0004_alter_log_level.py +18 -0
- mojo/apps/logit/models/log.py +4 -1
- mojo/apps/metrics/utils.py +2 -2
- mojo/apps/notify/handlers/ses/message.py +1 -1
- mojo/apps/notify/providers/aws.py +2 -2
- mojo/apps/tasks/__init__.py +34 -1
- mojo/apps/tasks/manager.py +200 -45
- mojo/apps/tasks/rest/tasks.py +24 -10
- mojo/apps/tasks/runner.py +283 -18
- mojo/apps/tasks/task.py +99 -0
- mojo/apps/tasks/tq_handlers.py +118 -0
- mojo/decorators/auth.py +6 -1
- mojo/decorators/http.py +7 -2
- mojo/helpers/aws/__init__.py +41 -0
- mojo/helpers/aws/ec2.py +804 -0
- mojo/helpers/aws/iam.py +748 -0
- mojo/helpers/aws/s3.py +451 -11
- mojo/helpers/aws/ses.py +483 -0
- mojo/helpers/aws/sns.py +461 -0
- mojo/helpers/crypto/__pycache__/hash.cpython-310.pyc +0 -0
- mojo/helpers/crypto/__pycache__/sign.cpython-310.pyc +0 -0
- mojo/helpers/crypto/__pycache__/utils.cpython-310.pyc +0 -0
- mojo/helpers/dates.py +18 -0
- mojo/helpers/response.py +6 -2
- mojo/helpers/settings/__init__.py +2 -0
- mojo/helpers/{settings.py → settings/helper.py} +1 -37
- mojo/helpers/settings/parser.py +132 -0
- mojo/middleware/logging.py +1 -1
- mojo/middleware/mojo.py +5 -0
- mojo/models/rest.py +261 -46
- mojo/models/secrets.py +13 -4
- mojo/serializers/__init__.py +100 -0
- mojo/serializers/advanced/README.md +363 -0
- mojo/serializers/advanced/__init__.py +247 -0
- mojo/serializers/advanced/formats/__init__.py +28 -0
- mojo/serializers/advanced/formats/csv.py +416 -0
- mojo/serializers/advanced/formats/excel.py +516 -0
- mojo/serializers/advanced/formats/json.py +239 -0
- mojo/serializers/advanced/formats/localizers.py +509 -0
- mojo/serializers/advanced/formats/response.py +485 -0
- mojo/serializers/advanced/serializer.py +568 -0
- mojo/serializers/manager.py +501 -0
- mojo/serializers/optimized.py +618 -0
- mojo/serializers/settings_example.py +322 -0
- mojo/serializers/{models.py → simple.py} +38 -15
- testit/helpers.py +21 -4
- django_nativemojo-0.1.10.dist-info/METADATA +0 -96
- mojo/apps/metrics/rest/db.py +0 -0
- mojo/helpers/aws/setup_email.py +0 -0
- mojo/ws4redis/README.md +0 -174
- mojo/ws4redis/__init__.py +0 -2
- mojo/ws4redis/client.py +0 -283
- mojo/ws4redis/connection.py +0 -327
- mojo/ws4redis/exceptions.py +0 -32
- mojo/ws4redis/redis.py +0 -183
- mojo/ws4redis/servers/base.py +0 -86
- mojo/ws4redis/servers/django.py +0 -171
- mojo/ws4redis/servers/uwsgi.py +0 -63
- mojo/ws4redis/settings.py +0 -45
- mojo/ws4redis/utf8validator.py +0 -128
- mojo/ws4redis/websocket.py +0 -403
- {django_nativemojo-0.1.10.dist-info → django_nativemojo-0.1.15.dist-info}/LICENSE +0 -0
- {django_nativemojo-0.1.10.dist-info → django_nativemojo-0.1.15.dist-info}/NOTICE +0 -0
- {django_nativemojo-0.1.10.dist-info → django_nativemojo-0.1.15.dist-info}/WHEEL +0 -0
- /mojo/{ws4redis/servers → apps/aws}/__init__.py +0 -0
- /mojo/apps/{fileman/models/render.py → aws/models/__init__.py} +0 -0
- /mojo/apps/fileman/{rest/__init__ → migrations/__init__.py} +0 -0
mojo/ws4redis/servers/django.py
DELETED
@@ -1,171 +0,0 @@
|
|
1
|
-
#-*- coding: utf-8 -*-
|
2
|
-
import base64
|
3
|
-
import select
|
4
|
-
from hashlib import sha1
|
5
|
-
from wsgiref import util
|
6
|
-
from django.core.handlers import wsgi as django_wsgi
|
7
|
-
from django.core.wsgi import get_wsgi_application
|
8
|
-
from django.core.servers.basehttp import WSGIServer, WSGIRequestHandler, ServerHandler
|
9
|
-
|
10
|
-
from django.conf import settings
|
11
|
-
from django.core.management.commands import runserver
|
12
|
-
import socketserver
|
13
|
-
from django.utils.encoding import force_str
|
14
|
-
from mojo.ws4redis.websocket import WebSocket
|
15
|
-
from mojo.ws4redis.servers.base import WebsocketServerBase, HandshakeError, UpgradeRequiredError
|
16
|
-
|
17
|
-
from io import IOBase
|
18
|
-
|
19
|
-
|
20
|
-
from mojo.helpers.logit import get_logger
|
21
|
-
logger = get_logger("async", filename="async.log")
|
22
|
-
|
23
|
-
logger.info("YES")
|
24
|
-
|
25
|
-
util._hoppish = {}.__contains__
|
26
|
-
|
27
|
-
|
28
|
-
class LimitedStreamPatched(IOBase):
|
29
|
-
"""
|
30
|
-
Wrap another stream to disallow reading it past a number of bytes.
|
31
|
-
|
32
|
-
Based on the implementation from werkzeug.wsgi.LimitedStream
|
33
|
-
See https://github.com/pallets/werkzeug/blob/dbf78f67/src/werkzeug/wsgi.py#L828
|
34
|
-
"""
|
35
|
-
|
36
|
-
def __init__(self, stream, limit):
|
37
|
-
self.stream = stream
|
38
|
-
self._read = stream.read
|
39
|
-
self._readline = stream.readline
|
40
|
-
self._pos = 0
|
41
|
-
self.limit = limit
|
42
|
-
|
43
|
-
def read(self, size=-1, /):
|
44
|
-
_pos = self._pos
|
45
|
-
limit = self.limit
|
46
|
-
if _pos >= limit:
|
47
|
-
return b""
|
48
|
-
if size == -1 or size is None:
|
49
|
-
size = limit - _pos
|
50
|
-
else:
|
51
|
-
size = min(size, limit - _pos)
|
52
|
-
data = self._read(size)
|
53
|
-
self._pos += len(data)
|
54
|
-
return data
|
55
|
-
|
56
|
-
def readline(self, size=-1, /):
|
57
|
-
_pos = self._pos
|
58
|
-
limit = self.limit
|
59
|
-
if _pos >= limit:
|
60
|
-
return b""
|
61
|
-
if size == -1 or size is None:
|
62
|
-
size = limit - _pos
|
63
|
-
else:
|
64
|
-
size = min(size, limit - _pos)
|
65
|
-
line = self._readline(size)
|
66
|
-
self._pos += len(line)
|
67
|
-
return line
|
68
|
-
|
69
|
-
|
70
|
-
def patchLimitedStream():
|
71
|
-
django_wsgi.LimitedStream = LimitedStreamPatched
|
72
|
-
|
73
|
-
|
74
|
-
class WSGIRequestHandlerRunServer(WSGIRequestHandler):
|
75
|
-
def handle(self):
|
76
|
-
self.raw_requestline = self.rfile.readline(65537)
|
77
|
-
if len(self.raw_requestline) > 65536:
|
78
|
-
self.requestline = ''
|
79
|
-
self.request_version = ''
|
80
|
-
self.command = ''
|
81
|
-
self.send_error(414)
|
82
|
-
return
|
83
|
-
|
84
|
-
if not self.parse_request(): # An error code has been sent, just exit
|
85
|
-
return
|
86
|
-
|
87
|
-
handler = ServerHandler(
|
88
|
-
self.rfile, self.wfile, self.get_stderr(), self.get_environ()
|
89
|
-
)
|
90
|
-
handler.request_handler = self # backpointer for logging
|
91
|
-
handler.http_version = '1.1'
|
92
|
-
handler.run(self.server.get_app())
|
93
|
-
|
94
|
-
|
95
|
-
class WebsocketRunServer(WebsocketServerBase):
|
96
|
-
WS_GUID = b'258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
|
97
|
-
WS_VERSIONS = ('13', '8', '7')
|
98
|
-
|
99
|
-
def upgrade_websocket(self, environ, start_response):
|
100
|
-
"""
|
101
|
-
Attempt to upgrade the socket environ['wsgi.input'] into a websocket enabled connection.
|
102
|
-
"""
|
103
|
-
websocket_version = environ.get('HTTP_SEC_WEBSOCKET_VERSION', '')
|
104
|
-
if not websocket_version:
|
105
|
-
raise UpgradeRequiredError
|
106
|
-
elif websocket_version not in self.WS_VERSIONS:
|
107
|
-
raise HandshakeError('Unsupported WebSocket Version: {0}'.format(websocket_version))
|
108
|
-
|
109
|
-
key = environ.get('HTTP_SEC_WEBSOCKET_KEY', '').strip()
|
110
|
-
if not key:
|
111
|
-
raise HandshakeError('Sec-WebSocket-Key header is missing/empty')
|
112
|
-
try:
|
113
|
-
key_len = len(base64.b64decode(key))
|
114
|
-
except TypeError:
|
115
|
-
raise HandshakeError('Invalid key: {0}'.format(key))
|
116
|
-
if key_len != 16:
|
117
|
-
# 5.2.1 (3)
|
118
|
-
raise HandshakeError('Invalid key: {0}'.format(key))
|
119
|
-
|
120
|
-
sec_ws_accept = base64.b64encode(sha1(key.encode('utf-8') + self.WS_GUID).digest())
|
121
|
-
sec_ws_accept = sec_ws_accept.decode('ascii')
|
122
|
-
headers = [
|
123
|
-
('Upgrade', 'websocket'),
|
124
|
-
('Connection', 'Upgrade'),
|
125
|
-
('Sec-WebSocket-Accept', sec_ws_accept),
|
126
|
-
('Sec-WebSocket-Version', str(websocket_version)),
|
127
|
-
]
|
128
|
-
if environ.get('HTTP_SEC_WEBSOCKET_PROTOCOL') is not None:
|
129
|
-
headers.append(('Sec-WebSocket-Protocol', environ.get('HTTP_SEC_WEBSOCKET_PROTOCOL')))
|
130
|
-
|
131
|
-
logger.debug('WebSocket request accepted, switching protocols')
|
132
|
-
start_response(force_str('101 Switching Protocols'), headers)
|
133
|
-
start_response.__self__.finish_content()
|
134
|
-
winput = environ['wsgi.input']
|
135
|
-
if not hasattr(winput, "stream"):
|
136
|
-
# hack to get the stream back in later DJANGO
|
137
|
-
winput.stream = winput._read.__self__
|
138
|
-
return WebSocket(winput.stream)
|
139
|
-
|
140
|
-
def select(self, rlist, wlist, xlist, timeout=None):
|
141
|
-
return select.select(rlist, wlist, xlist, timeout)
|
142
|
-
|
143
|
-
|
144
|
-
def run(addr, port, wsgi_handler, ipv6=False, threading=False, server_cls=None, **kwargs):
|
145
|
-
"""
|
146
|
-
Function to monkey patch the internal Django command: manage.py runserver
|
147
|
-
"""
|
148
|
-
server_address = (addr, port)
|
149
|
-
logger.info('Websocket support is enabled', server_address)
|
150
|
-
if not threading:
|
151
|
-
raise Exception("Django's Websocket server must run with threading enabled")
|
152
|
-
httpd_cls = type('WSGIServer', (socketserver.ThreadingMixIn, WSGIServer), {'daemon_threads': True})
|
153
|
-
httpd = httpd_cls(server_address, WSGIRequestHandlerRunServer, ipv6=ipv6)
|
154
|
-
httpd.set_app(wsgi_handler)
|
155
|
-
httpd.serve_forever()
|
156
|
-
|
157
|
-
|
158
|
-
patchLimitedStream()
|
159
|
-
|
160
|
-
runserver.run = run
|
161
|
-
|
162
|
-
_django_app = get_wsgi_application()
|
163
|
-
_websocket_app = WebsocketRunServer()
|
164
|
-
_websocket_url = getattr(settings, 'WEBSOCKET_URL')
|
165
|
-
|
166
|
-
|
167
|
-
def application(environ, start_response):
|
168
|
-
if _websocket_url and environ.get('PATH_INFO').startswith(_websocket_url):
|
169
|
-
# logger.info("environ", environ)
|
170
|
-
return _websocket_app(environ, start_response)
|
171
|
-
return _django_app(environ, start_response)
|
mojo/ws4redis/servers/uwsgi.py
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
import uwsgi
|
3
|
-
import gevent.select
|
4
|
-
|
5
|
-
import django
|
6
|
-
django.setup()
|
7
|
-
|
8
|
-
from mojo.ws4redis.exceptions import WebSocketError, SSLRequiredError
|
9
|
-
from mojo.ws4redis.servers.base import WebsocketServerBase
|
10
|
-
|
11
|
-
|
12
|
-
class uWSGIWebsocket(object):
|
13
|
-
def __init__(self):
|
14
|
-
self._closed = False
|
15
|
-
|
16
|
-
def get_file_descriptor(self):
|
17
|
-
"""Return the file descriptor for the given websocket"""
|
18
|
-
try:
|
19
|
-
return uwsgi.connection_fd()
|
20
|
-
except IOError as e:
|
21
|
-
self.close()
|
22
|
-
raise WebSocketError(e)
|
23
|
-
|
24
|
-
@property
|
25
|
-
def closed(self):
|
26
|
-
return self._closed
|
27
|
-
|
28
|
-
def receive(self):
|
29
|
-
if self._closed:
|
30
|
-
raise WebSocketError("Connection is already closed")
|
31
|
-
try:
|
32
|
-
return uwsgi.websocket_recv_nb()
|
33
|
-
except IOError as e:
|
34
|
-
self.close()
|
35
|
-
raise WebSocketError(e)
|
36
|
-
|
37
|
-
def flush(self):
|
38
|
-
try:
|
39
|
-
uwsgi.websocket_recv_nb()
|
40
|
-
except IOError:
|
41
|
-
self.close()
|
42
|
-
|
43
|
-
def send(self, message, binary=None):
|
44
|
-
try:
|
45
|
-
uwsgi.websocket_send(message)
|
46
|
-
except IOError as e:
|
47
|
-
self.close()
|
48
|
-
raise WebSocketError(e)
|
49
|
-
|
50
|
-
def close(self, code=1000, message=''):
|
51
|
-
self._closed = True
|
52
|
-
|
53
|
-
|
54
|
-
class uWSGIWebsocketServer(WebsocketServerBase):
|
55
|
-
def upgrade_websocket(self, environ, start_response):
|
56
|
-
try:
|
57
|
-
uwsgi.websocket_handshake(environ['HTTP_SEC_WEBSOCKET_KEY'], environ.get('HTTP_ORIGIN', ''))
|
58
|
-
except Exception:
|
59
|
-
raise SSLRequiredError("UWSGI REQUIRES SSL SUPPORT!")
|
60
|
-
return uWSGIWebsocket()
|
61
|
-
|
62
|
-
def select(self, rlist, wlist, xlist, timeout=None):
|
63
|
-
return gevent.select.select(rlist, wlist, xlist, timeout)
|
mojo/ws4redis/settings.py
DELETED
@@ -1,45 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
from django.conf import settings
|
3
|
-
|
4
|
-
WEBSOCKET_URL = getattr(settings, 'WEBSOCKET_URL', '/ws/')
|
5
|
-
|
6
|
-
WS4REDIS_CONNECTION = getattr(settings, 'WS4REDIS_CONNECTION', {
|
7
|
-
'host': 'localhost',
|
8
|
-
'port': 6379,
|
9
|
-
'db': 0,
|
10
|
-
'password': None,
|
11
|
-
})
|
12
|
-
|
13
|
-
"""
|
14
|
-
A string to prefix elements in the Redis datastore, to avoid naming conflicts with other services.
|
15
|
-
"""
|
16
|
-
WS4REDIS_PREFIX = getattr(settings, 'WS4REDIS_PREFIX', None)
|
17
|
-
|
18
|
-
"""
|
19
|
-
The time in seconds, items shall be persisted by the Redis datastore.
|
20
|
-
"""
|
21
|
-
WS4REDIS_EXPIRE = getattr(settings, 'WS4REDIS_EXPIRE', 3600)
|
22
|
-
|
23
|
-
"""
|
24
|
-
Replace the subscriber class by a customized version.
|
25
|
-
"""
|
26
|
-
WS4REDIS_SUBSCRIBER = getattr(settings, 'WS4REDIS_SUBSCRIBER', 'ws4redis.subscriber.RedisSubscriber')
|
27
|
-
|
28
|
-
"""
|
29
|
-
This set the magic string to recognize heartbeat messages. If set, this message string is ignored
|
30
|
-
by the server and also shall be ignored on the client.
|
31
|
-
|
32
|
-
If WS4REDIS_HEARTBEAT is not None, the server sends at least every 4 seconds a heartbeat message.
|
33
|
-
It is then up to the client to decide, what to do with these messages.
|
34
|
-
"""
|
35
|
-
WS4REDIS_HEARTBEAT = getattr(settings, 'WS4REDIS_HEARTBEAT', None)
|
36
|
-
|
37
|
-
# by default we only allow the "events" facillity
|
38
|
-
WS4REDIS_FACILITIES = getattr(settings, "WS4REDIS_FACILITIES", ["events"])
|
39
|
-
WS4REDIS_CHANNELS = getattr(settings, "WS4REDIS_CHANNELS", {})
|
40
|
-
WS4REDIS_AUTHENTICATORS = getattr(settings, "WS4REDIS_AUTHENTICATORS", {})
|
41
|
-
|
42
|
-
URL_AUTHENTICATOR = getattr(settings, "URL_AUTHENTICATOR", None)
|
43
|
-
|
44
|
-
WS4REDIS_LOG_DEBUG = getattr(settings, 'WS4REDIS_LOG_DEBUG', False)
|
45
|
-
WS4REDIS_NOAUTH_CLOSE = getattr(settings, 'WS4REDIS_NOAUTH_CLOSE', False)
|
mojo/ws4redis/utf8validator.py
DELETED
@@ -1,128 +0,0 @@
|
|
1
|
-
###############################################################################
|
2
|
-
##
|
3
|
-
## Copyright 2011-2013 Tavendo GmbH
|
4
|
-
##
|
5
|
-
## Note:
|
6
|
-
##
|
7
|
-
## This code is a Python implementation of the algorithm
|
8
|
-
##
|
9
|
-
## "Flexible and Economical UTF-8 Decoder"
|
10
|
-
##
|
11
|
-
## by Bjoern Hoehrmann
|
12
|
-
##
|
13
|
-
## bjoern@hoehrmann.de
|
14
|
-
## http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
|
15
|
-
##
|
16
|
-
## Licensed under the Apache License, Version 2.0 (the "License");
|
17
|
-
## you may not use this file except in compliance with the License.
|
18
|
-
## You may obtain a copy of the License at
|
19
|
-
##
|
20
|
-
## http://www.apache.org/licenses/LICENSE-2.0
|
21
|
-
##
|
22
|
-
## Unless required by applicable law or agreed to in writing, software
|
23
|
-
## distributed under the License is distributed on an "AS IS" BASIS,
|
24
|
-
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
25
|
-
## See the License for the specific language governing permissions and
|
26
|
-
## limitations under the License.
|
27
|
-
##
|
28
|
-
###############################################################################
|
29
|
-
|
30
|
-
|
31
|
-
## use Cython implementation of UTF8 validator if available
|
32
|
-
##
|
33
|
-
try:
|
34
|
-
from wsaccel.utf8validator import Utf8Validator
|
35
|
-
except:
|
36
|
-
## fallback to pure Python implementation
|
37
|
-
|
38
|
-
class Utf8Validator:
|
39
|
-
"""
|
40
|
-
Incremental UTF-8 validator with constant memory consumption (minimal
|
41
|
-
state).
|
42
|
-
|
43
|
-
Implements the algorithm "Flexible and Economical UTF-8 Decoder" by
|
44
|
-
Bjoern Hoehrmann (http://bjoern.hoehrmann.de/utf-8/decoder/dfa/).
|
45
|
-
"""
|
46
|
-
|
47
|
-
## DFA transitions
|
48
|
-
UTF8VALIDATOR_DFA = [
|
49
|
-
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, # 00..1f
|
50
|
-
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, # 20..3f
|
51
|
-
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, # 40..5f
|
52
|
-
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, # 60..7f
|
53
|
-
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, # 80..9f
|
54
|
-
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, # a0..bf
|
55
|
-
8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, # c0..df
|
56
|
-
0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, # e0..ef
|
57
|
-
0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, # f0..ff
|
58
|
-
0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, # s0..s0
|
59
|
-
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, # s1..s2
|
60
|
-
1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, # s3..s4
|
61
|
-
1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, # s5..s6
|
62
|
-
1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, # s7..s8
|
63
|
-
]
|
64
|
-
|
65
|
-
UTF8_ACCEPT = 0
|
66
|
-
UTF8_REJECT = 1
|
67
|
-
|
68
|
-
def __init__(self):
|
69
|
-
self.reset()
|
70
|
-
|
71
|
-
def decode(self, b):
|
72
|
-
"""
|
73
|
-
Eat one UTF-8 octet, and validate on the fly.
|
74
|
-
|
75
|
-
Returns UTF8_ACCEPT when enough octets have been consumed, in which case
|
76
|
-
self.codepoint contains the decoded Unicode code point.
|
77
|
-
|
78
|
-
Returns UTF8_REJECT when invalid UTF-8 was encountered.
|
79
|
-
|
80
|
-
Returns some other positive integer when more octets need to be eaten.
|
81
|
-
"""
|
82
|
-
type = Utf8Validator.UTF8VALIDATOR_DFA[b]
|
83
|
-
|
84
|
-
if self.state != Utf8Validator.UTF8_ACCEPT:
|
85
|
-
self.codepoint = (b & 0x3f) | (self.codepoint << 6)
|
86
|
-
else:
|
87
|
-
self.codepoint = (0xff >> type) & b
|
88
|
-
|
89
|
-
self.state = Utf8Validator.UTF8VALIDATOR_DFA[256 + self.state * 16 + type]
|
90
|
-
|
91
|
-
return self.state
|
92
|
-
|
93
|
-
def reset(self):
|
94
|
-
"""
|
95
|
-
Reset validator to start new incremental UTF-8 decode/validation.
|
96
|
-
"""
|
97
|
-
self.state = Utf8Validator.UTF8_ACCEPT
|
98
|
-
self.codepoint = 0
|
99
|
-
self.i = 0
|
100
|
-
|
101
|
-
def validate(self, ba):
|
102
|
-
"""
|
103
|
-
Incrementally validate a chunk of bytes provided as string.
|
104
|
-
|
105
|
-
Will return a quad (valid?, endsOnCodePoint?, currentIndex, totalIndex).
|
106
|
-
|
107
|
-
As soon as an octet is encountered which renders the octet sequence
|
108
|
-
invalid, a quad with valid? == False is returned. currentIndex returns
|
109
|
-
the index within the currently consumed chunk, and totalIndex the
|
110
|
-
index within the total consumed sequence that was the point of bail out.
|
111
|
-
When valid? == True, currentIndex will be len(ba) and totalIndex the
|
112
|
-
total amount of consumed bytes.
|
113
|
-
"""
|
114
|
-
|
115
|
-
l = len(ba)
|
116
|
-
|
117
|
-
for i in range(l):
|
118
|
-
## optimized version of decode(), since we are not interested in actual code points
|
119
|
-
|
120
|
-
self.state = Utf8Validator.UTF8VALIDATOR_DFA[256 + (self.state << 4) + Utf8Validator.UTF8VALIDATOR_DFA[ord(ba[i])]]
|
121
|
-
|
122
|
-
if self.state == Utf8Validator.UTF8_REJECT:
|
123
|
-
self.i += i
|
124
|
-
return False, False, i, self.i
|
125
|
-
|
126
|
-
self.i += l
|
127
|
-
|
128
|
-
return True, self.state == Utf8Validator.UTF8_ACCEPT, l, self.i
|