mkit-cli 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mkit/__init__.py +5 -0
- mkit/art.py +1220 -0
- mkit/aso.py +359 -0
- mkit/cli.py +176 -0
- mkit/cloud/__init__.py +1 -0
- mkit/cloud/auth.py +130 -0
- mkit/cloud/fbproject.py +180 -0
- mkit/cloud/firebase.py +164 -0
- mkit/cloud/insight.py +286 -0
- mkit/cloud/money.py +676 -0
- mkit/cloud/play.py +497 -0
- mkit/cloud/store.py +390 -0
- mkit/commands/__init__.py +2 -0
- mkit/commands/art.py +116 -0
- mkit/commands/aso.py +130 -0
- mkit/commands/dev.py +256 -0
- mkit/commands/doctor.py +191 -0
- mkit/commands/firebase_cmds.py +194 -0
- mkit/commands/init.py +213 -0
- mkit/commands/insight.py +110 -0
- mkit/commands/meta.py +84 -0
- mkit/commands/money.py +152 -0
- mkit/commands/release.py +430 -0
- mkit/commands/selfcmd.py +167 -0
- mkit/commands/store.py +263 -0
- mkit/config.py +146 -0
- mkit/data/art/art.toml +119 -0
- mkit/data/art/en.json +38 -0
- mkit/data/datasafety-questions.csv +783 -0
- mkit/datasafety.py +302 -0
- mkit/device.py +246 -0
- mkit/docgen.py +100 -0
- mkit/errors.py +78 -0
- mkit/git.py +95 -0
- mkit/hooks.py +32 -0
- mkit/kinds.py +350 -0
- mkit/notify.py +49 -0
- mkit/output.py +377 -0
- mkit/proc.py +154 -0
- mkit/registry.py +99 -0
- mkit/safety.py +60 -0
- mkit/secrets.py +148 -0
- mkit/toolchain.py +210 -0
- mkit_cli-0.3.1.dist-info/METADATA +179 -0
- mkit_cli-0.3.1.dist-info/RECORD +48 -0
- mkit_cli-0.3.1.dist-info/WHEEL +4 -0
- mkit_cli-0.3.1.dist-info/entry_points.txt +2 -0
- mkit_cli-0.3.1.dist-info/licenses/LICENSE +21 -0
mkit/cloud/fbproject.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Remote Config templates and Firebase app management (SHA fingerprints,
|
|
2
|
+
google-services.json), with this app's Firebase service account.
|
|
3
|
+
|
|
4
|
+
Remote Config is published whole: the template is read with its ETag and
|
|
5
|
+
written back with If-Match, so a change someone made in the console between
|
|
6
|
+
plan and apply is a conflict (exit 5), never silently overwritten.
|
|
7
|
+
"""
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
from ..errors import Conflict, MkitError, NotFound, UsageError
|
|
13
|
+
from . import auth
|
|
14
|
+
from .firebase import project_id
|
|
15
|
+
|
|
16
|
+
RC = 'https://firebaseremoteconfig.googleapis.com/v1/projects/{p}/remoteConfig'
|
|
17
|
+
MGMT = 'https://firebase.googleapis.com/v1beta1/projects/{p}'
|
|
18
|
+
TEMPLATE_KEYS = ('conditions', 'parameters', 'parameterGroups')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _session(ctx, scopes):
|
|
22
|
+
import google.auth.transport.requests
|
|
23
|
+
return google.auth.transport.requests.AuthorizedSession(auth.credentials(ctx, auth.FIREBASE, scopes))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _check(r, what):
|
|
27
|
+
if r.status_code in (401, 403):
|
|
28
|
+
raise MkitError(f'{what} refused ({r.status_code}): {r.text[:200]}', code='PERMISSION_DENIED',
|
|
29
|
+
hint='Give the Firebase service account the matching role (Remote Config Admin, '
|
|
30
|
+
'Firebase Admin) in the Cloud console IAM page.')
|
|
31
|
+
if r.status_code == 409 or r.status_code == 412:
|
|
32
|
+
raise Conflict(f'{what}: the template changed since it was read; run plan again')
|
|
33
|
+
if r.status_code == 404:
|
|
34
|
+
raise NotFound(f'{what}: not found')
|
|
35
|
+
if r.status_code >= 400:
|
|
36
|
+
raise MkitError(f'{what} failed ({r.status_code}): {r.text[:300]}', code=f'HTTP_{r.status_code}')
|
|
37
|
+
return r
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# ------------------------------------------------------------------ Remote Config
|
|
41
|
+
|
|
42
|
+
def template_path(ctx):
|
|
43
|
+
return ctx.project.path('firebase.remote_config', 'remoteconfig.template.json')
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def rc_get(ctx):
|
|
47
|
+
s = _session(ctx, [auth.SCOPE_REMOTE_CONFIG])
|
|
48
|
+
r = _check(s.get(RC.format(p=project_id(ctx)), headers={'Accept-Encoding': 'gzip'}, timeout=60),
|
|
49
|
+
'Remote Config')
|
|
50
|
+
return r.json(), r.headers.get('ETag')
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def local_template(ctx):
|
|
54
|
+
path = template_path(ctx)
|
|
55
|
+
if not path.is_file():
|
|
56
|
+
raise UsageError(f'{path} not found', hint='`mkit remoteconfig pull` writes it from Firebase.',
|
|
57
|
+
code='NO_TEMPLATE')
|
|
58
|
+
data = json.loads(path.read_text(encoding='utf-8'))
|
|
59
|
+
return {k: data[k] for k in TEMPLATE_KEYS if k in data}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def rc_diff(remote, local):
|
|
63
|
+
changes = []
|
|
64
|
+
rp, lp = remote.get('parameters', {}), local.get('parameters', {})
|
|
65
|
+
for key in sorted(set(rp) | set(lp)):
|
|
66
|
+
if key not in lp:
|
|
67
|
+
changes.append({'parameter': key, 'change': 'remove'})
|
|
68
|
+
elif key not in rp:
|
|
69
|
+
changes.append({'parameter': key, 'change': 'add'})
|
|
70
|
+
elif rp[key] != lp[key]:
|
|
71
|
+
changes.append({'parameter': key, 'change': 'update'})
|
|
72
|
+
for key in ('conditions', 'parameterGroups'):
|
|
73
|
+
if remote.get(key, [] if key == 'conditions' else {}) != local.get(key, [] if key == 'conditions' else {}):
|
|
74
|
+
changes.append({'parameter': f'({key})', 'change': 'update'})
|
|
75
|
+
return changes
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def rc_publish(ctx, template, etag, validate_only=False):
|
|
79
|
+
s = _session(ctx, [auth.SCOPE_REMOTE_CONFIG])
|
|
80
|
+
r = _check(s.put(RC.format(p=project_id(ctx)), params={'validate_only': 'true'} if validate_only else None,
|
|
81
|
+
headers={'If-Match': etag, 'Content-Type': 'application/json; UTF-8'},
|
|
82
|
+
data=json.dumps(template), timeout=60), 'Remote Config publish')
|
|
83
|
+
return r.json().get('version', {})
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def rc_versions(ctx, limit=10):
|
|
87
|
+
s = _session(ctx, [auth.SCOPE_REMOTE_CONFIG])
|
|
88
|
+
r = _check(s.get(RC.format(p=project_id(ctx)) + ':listVersions', params={'pageSize': limit}, timeout=60),
|
|
89
|
+
'Remote Config versions')
|
|
90
|
+
return r.json().get('versions', [])
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def rc_rollback(ctx, version):
|
|
94
|
+
s = _session(ctx, [auth.SCOPE_REMOTE_CONFIG])
|
|
95
|
+
r = _check(s.post(RC.format(p=project_id(ctx)) + ':rollback', json={'versionNumber': str(version)},
|
|
96
|
+
timeout=60), 'Remote Config rollback')
|
|
97
|
+
return r.json().get('version', {})
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ------------------------------------------------------------------ Firebase apps
|
|
101
|
+
|
|
102
|
+
def android_apps(ctx):
|
|
103
|
+
s = _session(ctx, [auth.SCOPE_FIREBASE])
|
|
104
|
+
r = _check(s.get(MGMT.format(p=project_id(ctx)) + '/androidApps', params={'pageSize': 100},
|
|
105
|
+
timeout=60), 'Firebase apps')
|
|
106
|
+
return s, r.json().get('apps', [])
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def android_app(ctx, package=None):
|
|
110
|
+
package = package or ctx.project.package
|
|
111
|
+
s, apps = android_apps(ctx)
|
|
112
|
+
app = next((a for a in apps if a.get('packageName') == package), None)
|
|
113
|
+
if not app:
|
|
114
|
+
raise NotFound(f'no Android app {package} in Firebase project {project_id(ctx)}',
|
|
115
|
+
hint=f'It has: {", ".join(a.get("packageName", "?") for a in apps) or "none"}')
|
|
116
|
+
return s, app
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def ensure_android_app(ctx, package, display_name, wait=120):
|
|
120
|
+
"""The project's Android app for `package`, registering it when missing.
|
|
121
|
+
Returns (app, created). Creation is a long-running operation; it is waited
|
|
122
|
+
for, so the config pulled right after already carries the new client."""
|
|
123
|
+
s, apps = android_apps(ctx)
|
|
124
|
+
app = next((a for a in apps if a.get('packageName') == package), None)
|
|
125
|
+
if app:
|
|
126
|
+
return app, False
|
|
127
|
+
op = _check(s.post(MGMT.format(p=project_id(ctx)) + '/androidApps',
|
|
128
|
+
json={'packageName': package, 'displayName': display_name}, timeout=60),
|
|
129
|
+
f'Registering {package}').json()
|
|
130
|
+
deadline = time.monotonic() + wait
|
|
131
|
+
while not op.get('done'):
|
|
132
|
+
if time.monotonic() > deadline:
|
|
133
|
+
raise MkitError(f'Firebase is still registering {package}',
|
|
134
|
+
hint='Run the same command again in a minute; it picks up where this left off.',
|
|
135
|
+
code='FIREBASE_PENDING')
|
|
136
|
+
time.sleep(2)
|
|
137
|
+
op = _check(s.get(f'https://firebase.googleapis.com/v1beta1/{op["name"]}', timeout=60),
|
|
138
|
+
f'Registering {package}').json()
|
|
139
|
+
if op.get('error'):
|
|
140
|
+
raise MkitError(f'Firebase could not register {package}: {op["error"].get("message", op["error"])}',
|
|
141
|
+
code='FIREBASE_CREATE_FAILED')
|
|
142
|
+
return op.get('response', {}), True
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def parse_fingerprints(keytool_text):
|
|
146
|
+
"""SHA-1 and SHA-256 of a certificate, from `keytool -list -v` output."""
|
|
147
|
+
found = {}
|
|
148
|
+
for label, key in (('SHA1', 'SHA_1'), ('SHA256', 'SHA_256')):
|
|
149
|
+
m = re.search(rf'{label}:\s*([0-9A-Fa-f:]+)', keytool_text)
|
|
150
|
+
if m:
|
|
151
|
+
found[key] = m.group(1)
|
|
152
|
+
return found
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def sha_list(ctx):
|
|
156
|
+
s, app = android_app(ctx)
|
|
157
|
+
r = _check(s.get(f'https://firebase.googleapis.com/v1beta1/{app["name"]}/sha', timeout=60), 'SHA list')
|
|
158
|
+
return app, r.json().get('certificates', [])
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def sha_add(ctx, sha, package=None):
|
|
162
|
+
clean = sha.replace(':', '').lower()
|
|
163
|
+
cert_type = {40: 'SHA_1', 64: 'SHA_256'}.get(len(clean))
|
|
164
|
+
if not cert_type:
|
|
165
|
+
raise UsageError('a SHA-1 has 40 hex digits and a SHA-256 has 64 (colons are fine)')
|
|
166
|
+
s, app = android_app(ctx, package)
|
|
167
|
+
r = s.post(f'https://firebase.googleapis.com/v1beta1/{app["name"]}/sha',
|
|
168
|
+
json={'shaHash': clean, 'certType': cert_type}, timeout=60)
|
|
169
|
+
if r.status_code == 409:
|
|
170
|
+
return {'shaHash': clean, 'certType': cert_type, 'changed': False}
|
|
171
|
+
_check(r, 'SHA add')
|
|
172
|
+
return {'shaHash': clean, 'certType': cert_type, 'changed': True}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def config_file(ctx):
|
|
176
|
+
import base64
|
|
177
|
+
s, app = android_app(ctx)
|
|
178
|
+
r = _check(s.get(f'https://firebase.googleapis.com/v1beta1/{app["name"]}/config', timeout=60),
|
|
179
|
+
'google-services.json')
|
|
180
|
+
return base64.b64decode(r.json()['configFileContents']).decode('utf-8')
|
mkit/cloud/firebase.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Firebase: the in-app update document, and Firestore rules/indexes deploy.
|
|
2
|
+
|
|
3
|
+
The Firebase project is the one named under [firebase] project in mkit.toml, or
|
|
4
|
+
else the one the service account belongs to. When both are set they must
|
|
5
|
+
agree: a key for the wrong project is the mistake this catches before a rules
|
|
6
|
+
deploy lands somewhere it should not.
|
|
7
|
+
"""
|
|
8
|
+
from urllib.parse import urlsplit
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import tempfile
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
|
|
14
|
+
from .. import output, proc
|
|
15
|
+
from ..errors import MkitError, UsageError
|
|
16
|
+
from ..toolchain import tool
|
|
17
|
+
from . import auth
|
|
18
|
+
|
|
19
|
+
FIRESTORE = 'https://firestore.googleapis.com/v1'
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def project_id(ctx):
|
|
23
|
+
configured = ctx.project.get('firebase.project')
|
|
24
|
+
from_key = auth.service_account_project(ctx) if ctx.secrets.has(auth.FIREBASE) else None
|
|
25
|
+
if configured and from_key and configured != from_key:
|
|
26
|
+
raise UsageError(f'[firebase] project is {configured} but the service account belongs to {from_key}',
|
|
27
|
+
hint='Use the key for this app\'s Firebase project, or fix mkit.toml.',
|
|
28
|
+
code='PROJECT_MISMATCH')
|
|
29
|
+
project = configured or from_key
|
|
30
|
+
if not project:
|
|
31
|
+
raise UsageError('no Firebase project: set project under [firebase] or add the service account',
|
|
32
|
+
code='NO_FIREBASE_PROJECT')
|
|
33
|
+
return project
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _value(v):
|
|
37
|
+
if isinstance(v, bool):
|
|
38
|
+
return {'booleanValue': v}
|
|
39
|
+
if isinstance(v, int):
|
|
40
|
+
return {'integerValue': str(v)}
|
|
41
|
+
if isinstance(v, datetime):
|
|
42
|
+
return {'timestampValue': v.strftime('%Y-%m-%dT%H:%M:%SZ')}
|
|
43
|
+
return {'stringValue': str(v)}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _send(method, url, **kw):
|
|
47
|
+
"""One request, with a transport failure turned into an MkitError.
|
|
48
|
+
|
|
49
|
+
Timeouts, refused connections and DNS failures arrive as requests'
|
|
50
|
+
own exceptions, which are not MkitErrors - so they used to travel all the
|
|
51
|
+
way out of a command as an "internal error" with a traceback, and past the
|
|
52
|
+
handlers that exist to degrade gracefully. A dropped connection is the most
|
|
53
|
+
ordinary failure there is and says nothing about a bug.
|
|
54
|
+
"""
|
|
55
|
+
import requests
|
|
56
|
+
try:
|
|
57
|
+
return requests.request(method, url, **kw)
|
|
58
|
+
except requests.exceptions.RequestException as error:
|
|
59
|
+
raise MkitError(f'could not reach {urlsplit(url).netloc}: '
|
|
60
|
+
f'{type(error).__name__.replace("Error", "").lower() or "failed"}',
|
|
61
|
+
hint='Check the connection and run it again; nothing was written.',
|
|
62
|
+
code='NETWORK', retryable=True) from None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def patch_document(ctx, path, values):
|
|
66
|
+
"""Write only the given fields of one document (updateMask), creating it if
|
|
67
|
+
needed. Fields not named are left exactly as they are."""
|
|
68
|
+
project = project_id(ctx)
|
|
69
|
+
token = auth.token(ctx, auth.FIREBASE, [auth.SCOPE_DATASTORE])
|
|
70
|
+
fields = {k: _value(v) for k, v in values.items()}
|
|
71
|
+
response = _send(
|
|
72
|
+
'PATCH', f'{FIRESTORE}/projects/{project}/databases/(default)/documents/{path}',
|
|
73
|
+
params=[('updateMask.fieldPaths', k) for k in fields],
|
|
74
|
+
json={'fields': fields}, headers={'Authorization': f'Bearer {token}'}, timeout=30)
|
|
75
|
+
if response.status_code in (401, 403):
|
|
76
|
+
raise MkitError(f'Firestore refused the write ({response.status_code})', code='PERMISSION_DENIED',
|
|
77
|
+
hint='Give the service account the Cloud Datastore User role on the project.')
|
|
78
|
+
if response.status_code >= 400:
|
|
79
|
+
raise MkitError(f'Firestore write failed ({response.status_code}): {response.text[:300]}',
|
|
80
|
+
code=f'HTTP_{response.status_code}')
|
|
81
|
+
return project
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def get_document(ctx, path):
|
|
85
|
+
project = project_id(ctx)
|
|
86
|
+
token = auth.token(ctx, auth.FIREBASE, [auth.SCOPE_DATASTORE])
|
|
87
|
+
response = _send('GET', f'{FIRESTORE}/projects/{project}/databases/(default)/documents/{path}',
|
|
88
|
+
headers={'Authorization': f'Bearer {token}'}, timeout=30)
|
|
89
|
+
if response.status_code == 404:
|
|
90
|
+
return {}
|
|
91
|
+
if response.status_code >= 400:
|
|
92
|
+
raise MkitError(f'Firestore read failed ({response.status_code}): {response.text[:300]}',
|
|
93
|
+
code=f'HTTP_{response.status_code}')
|
|
94
|
+
out = {}
|
|
95
|
+
for key, value in response.json().get('fields', {}).items():
|
|
96
|
+
(kind, raw), = value.items()
|
|
97
|
+
out[key] = int(raw) if kind == 'integerValue' else raw
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def update_doc_path(ctx):
|
|
102
|
+
return ctx.project.get('firebase.update_doc', 'config/app')
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def publish_version(ctx, version_name, version_code):
|
|
106
|
+
"""The document the app reads to decide whether to offer an update."""
|
|
107
|
+
path = update_doc_path(ctx)
|
|
108
|
+
project = patch_document(ctx, path, {'latestBuild': int(version_code),
|
|
109
|
+
'latestVersionName': version_name,
|
|
110
|
+
'updatedAt': datetime.now(timezone.utc)})
|
|
111
|
+
return f'{project}/{path}'
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def deploy_targets(ctx):
|
|
115
|
+
custom = ctx.project.get('firebase.deploy')
|
|
116
|
+
if custom:
|
|
117
|
+
return list(custom)
|
|
118
|
+
targets = []
|
|
119
|
+
if (ctx.project.root / 'firestore.rules').is_file():
|
|
120
|
+
targets.append('firestore:rules')
|
|
121
|
+
if (ctx.project.root / 'firestore.indexes.json').is_file():
|
|
122
|
+
targets.append('firestore:indexes')
|
|
123
|
+
return targets
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def deploy_rules(ctx):
|
|
127
|
+
"""`firebase deploy --only firestore:rules,firestore:indexes` as the
|
|
128
|
+
project's service account, not whoever happens to be logged in to the CLI.
|
|
129
|
+
|
|
130
|
+
The key is written to a private temp file for the one command and deleted
|
|
131
|
+
straight after. Rules go out before a build on purpose: rules live ahead of
|
|
132
|
+
the app that needs them only widen what older clients may do, while an app
|
|
133
|
+
that reaches users before its rules gets PERMISSION_DENIED.
|
|
134
|
+
"""
|
|
135
|
+
targets = deploy_targets(ctx)
|
|
136
|
+
if not targets:
|
|
137
|
+
raise UsageError('nothing to deploy: no firestore.rules or firestore.indexes.json',
|
|
138
|
+
code='NO_RULES')
|
|
139
|
+
if not (ctx.project.root / 'firebase.json').is_file():
|
|
140
|
+
raise UsageError('firebase.json not found in the project root', code='NO_FIREBASE_JSON')
|
|
141
|
+
project = project_id(ctx)
|
|
142
|
+
info = ctx.secrets.json(auth.FIREBASE)
|
|
143
|
+
fd, key_path = tempfile.mkstemp(prefix='mkit-sa-', suffix='.json')
|
|
144
|
+
try:
|
|
145
|
+
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
|
146
|
+
json.dump(info, f)
|
|
147
|
+
env = dict(os.environ, GOOGLE_APPLICATION_CREDENTIALS=key_path)
|
|
148
|
+
env.pop('FIREBASE_TOKEN', None)
|
|
149
|
+
# "Deploy complete!" is the CLI's own word that the rules and indexes are live. It
|
|
150
|
+
# can still exit non-zero afterwards: on a success it posts a Google Analytics event
|
|
151
|
+
# with a 5s timeout (command.js -> withTimeout(5000, trackGA4)), and where that call
|
|
152
|
+
# cannot reach Google the timeout escapes as "An unexpected error has occurred". The
|
|
153
|
+
# deploy is done by then, and failing here would roll back a version bump over a
|
|
154
|
+
# telemetry ping.
|
|
155
|
+
proc.stream([tool('firebase'), 'deploy', '--only', ','.join(targets), '--project', project,
|
|
156
|
+
'--non-interactive'], 'firebase deploy', proc.firebase_interesting,
|
|
157
|
+
cwd=ctx.project.root, env=env, quiet=True,
|
|
158
|
+
done_marker='Deploy complete!')
|
|
159
|
+
finally:
|
|
160
|
+
try:
|
|
161
|
+
os.remove(key_path)
|
|
162
|
+
except OSError:
|
|
163
|
+
output.out.warn(f'could not delete the temporary key file {key_path}; delete it by hand')
|
|
164
|
+
return project, targets
|
mkit/cloud/insight.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"""How the app is doing: bulk reports, Android vitals, crash clusters, reviews.
|
|
2
|
+
|
|
3
|
+
stats the Play Console bulk reports (Cloud Storage bucket, UTF-16 CSVs,
|
|
4
|
+
3-7 days behind). The service account needs "View app information
|
|
5
|
+
and download bulk reports"; the bucket is MKIT_PLAY_REPORTS_BUCKET.
|
|
6
|
+
vitals Play Developer Reporting API crash and ANR rates.
|
|
7
|
+
crashes Reporting API error issues: crash/ANR clusters with stack location.
|
|
8
|
+
reviews Android Publisher reviews: only the last 7 days with text.
|
|
9
|
+
"""
|
|
10
|
+
import csv
|
|
11
|
+
import io
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from datetime import date, timedelta
|
|
14
|
+
|
|
15
|
+
from ..errors import CredentialMissing, MkitError, UsageError
|
|
16
|
+
from . import auth
|
|
17
|
+
|
|
18
|
+
REPORTING = 'https://playdeveloperreporting.googleapis.com/v1beta1'
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# ------------------------------------------------------------------ bulk reports
|
|
22
|
+
|
|
23
|
+
def _bucket(ctx):
|
|
24
|
+
bucket = ctx.project.get('play.reports_bucket') or ctx.secrets.value('MKIT_PLAY_REPORTS_BUCKET')
|
|
25
|
+
if not bucket:
|
|
26
|
+
raise UsageError('no reports bucket', code='NO_BUCKET',
|
|
27
|
+
hint='Set MKIT_PLAY_REPORTS_BUCKET in ~/.mkit/.env (Play Console > Download '
|
|
28
|
+
'reports > copy the Cloud Storage URI, the part after gs://).')
|
|
29
|
+
return bucket.removeprefix('gs://').strip('/')
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _decode(raw):
|
|
33
|
+
for encoding in ('utf-16', 'utf-8-sig', 'utf-8'):
|
|
34
|
+
try:
|
|
35
|
+
text = raw.decode(encoding)
|
|
36
|
+
except UnicodeDecodeError:
|
|
37
|
+
continue
|
|
38
|
+
if '\x00' not in text:
|
|
39
|
+
rows = list(csv.reader(io.StringIO(text)))
|
|
40
|
+
return (rows[0], rows[1:]) if rows else ([], [])
|
|
41
|
+
return [], []
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _col(header, *names):
|
|
45
|
+
lowered = [h.strip().lower() for h in header]
|
|
46
|
+
for n in names:
|
|
47
|
+
if n.lower() in lowered:
|
|
48
|
+
return lowered.index(n.lower())
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _int(v):
|
|
53
|
+
try:
|
|
54
|
+
return int(float(v or 0))
|
|
55
|
+
except ValueError:
|
|
56
|
+
return 0
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def store_daily(ctx):
|
|
60
|
+
"""Store listing visitors and acquisitions per day, all countries summed:
|
|
61
|
+
{'YYYY-MM-DD': (visitors, acquisitions)}. Days Play has no row for are
|
|
62
|
+
left out (no visitors that day, or under Play's reporting threshold)."""
|
|
63
|
+
from googleapiclient.errors import HttpError
|
|
64
|
+
bucket, package = _bucket(ctx), ctx.project.package
|
|
65
|
+
st = auth.build(ctx, 'storage', 'v1', auth.PLAY, [auth.SCOPE_STORAGE_READ])
|
|
66
|
+
prefix = f'stats/store_performance/store_performance_{package}_'
|
|
67
|
+
try:
|
|
68
|
+
names, token = [], None
|
|
69
|
+
while True:
|
|
70
|
+
res = st.objects().list(bucket=bucket, prefix=prefix, maxResults=1000, pageToken=token).execute()
|
|
71
|
+
names += [o['name'] for o in res.get('items', [])]
|
|
72
|
+
token = res.get('nextPageToken')
|
|
73
|
+
if not token:
|
|
74
|
+
break
|
|
75
|
+
except HttpError as e:
|
|
76
|
+
raise auth.translate(e, 'Play reports bucket') from None
|
|
77
|
+
daily = defaultdict(lambda: [0, 0])
|
|
78
|
+
for name in sorted(names):
|
|
79
|
+
if not name.endswith('_country.csv'):
|
|
80
|
+
continue
|
|
81
|
+
header, rows = _decode(st.objects().get_media(bucket=bucket, object=name).execute())
|
|
82
|
+
v, a = _col(header, 'Store listing visitors'), _col(header, 'Store listing acquisitions')
|
|
83
|
+
if None in (v, a):
|
|
84
|
+
continue
|
|
85
|
+
for r in rows:
|
|
86
|
+
if len(r) > max(v, a) and r[0]:
|
|
87
|
+
daily[r[0]][0] += _int(r[v])
|
|
88
|
+
daily[r[0]][1] += _int(r[a])
|
|
89
|
+
return {d: tuple(x) for d, x in sorted(daily.items())}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def stats(ctx):
|
|
93
|
+
from googleapiclient.errors import HttpError
|
|
94
|
+
bucket, package = _bucket(ctx), ctx.project.package
|
|
95
|
+
st = auth.build(ctx, 'storage', 'v1', auth.PLAY, [auth.SCOPE_STORAGE_READ])
|
|
96
|
+
|
|
97
|
+
def listing(prefix):
|
|
98
|
+
names, token = [], None
|
|
99
|
+
while True:
|
|
100
|
+
res = st.objects().list(bucket=bucket, prefix=prefix, maxResults=1000, pageToken=token).execute()
|
|
101
|
+
names += [o['name'] for o in res.get('items', [])]
|
|
102
|
+
token = res.get('nextPageToken')
|
|
103
|
+
if not token:
|
|
104
|
+
return sorted(names)
|
|
105
|
+
|
|
106
|
+
def fetch(name):
|
|
107
|
+
return _decode(st.objects().get_media(bucket=bucket, object=name).execute())
|
|
108
|
+
|
|
109
|
+
try:
|
|
110
|
+
installs = listing(f'stats/installs/installs_{package}_')
|
|
111
|
+
store_perf = listing(f'stats/store_performance/store_performance_{package}_')
|
|
112
|
+
reviews = listing(f'reviews/reviews_{package}_')
|
|
113
|
+
except HttpError as e:
|
|
114
|
+
err = auth.translate(e, 'Play reports bucket')
|
|
115
|
+
if isinstance(err, CredentialMissing):
|
|
116
|
+
err.hint = ('In Play Console > Users and permissions, give the service account "View app '
|
|
117
|
+
'information and download bulk reports" (account level).')
|
|
118
|
+
raise err from None
|
|
119
|
+
|
|
120
|
+
daily, by_country = [], defaultdict(int)
|
|
121
|
+
for name in installs:
|
|
122
|
+
header, rows = fetch(name)
|
|
123
|
+
if name.endswith('_overview.csv'):
|
|
124
|
+
i, u, a = (_col(header, 'Daily User Installs'), _col(header, 'Daily User Uninstalls'),
|
|
125
|
+
_col(header, 'Active Device Installs'))
|
|
126
|
+
if None in (i, u, a):
|
|
127
|
+
continue
|
|
128
|
+
daily += [{'date': r[0], 'installs': _int(r[i]), 'uninstalls': _int(r[u]), 'active': _int(r[a])}
|
|
129
|
+
for r in rows if len(r) > max(i, u, a)]
|
|
130
|
+
elif name.endswith('_country.csv'):
|
|
131
|
+
c, i = _col(header, 'Country', 'Country / region'), _col(header, 'Daily User Installs')
|
|
132
|
+
if None in (c, i):
|
|
133
|
+
continue
|
|
134
|
+
for r in rows:
|
|
135
|
+
if len(r) > max(c, i):
|
|
136
|
+
by_country[r[c] or '??'] += _int(r[i])
|
|
137
|
+
daily.sort(key=lambda d: d['date'])
|
|
138
|
+
|
|
139
|
+
visitors = acquisitions = 0
|
|
140
|
+
for name in store_perf:
|
|
141
|
+
if not name.endswith('_country.csv'):
|
|
142
|
+
continue
|
|
143
|
+
header, rows = fetch(name)
|
|
144
|
+
v, a = _col(header, 'Store listing visitors'), _col(header, 'Store listing acquisitions')
|
|
145
|
+
if None in (v, a):
|
|
146
|
+
continue
|
|
147
|
+
for r in rows:
|
|
148
|
+
if len(r) > max(v, a):
|
|
149
|
+
visitors += _int(r[v])
|
|
150
|
+
acquisitions += _int(r[a])
|
|
151
|
+
|
|
152
|
+
stars = []
|
|
153
|
+
for name in reviews:
|
|
154
|
+
header, rows = fetch(name)
|
|
155
|
+
s = _col(header, 'Star Rating')
|
|
156
|
+
if s is not None:
|
|
157
|
+
stars += [_int(r[s]) for r in rows if len(r) > s and (r[s] or '').strip()]
|
|
158
|
+
|
|
159
|
+
active = [d['active'] for d in daily]
|
|
160
|
+
return {
|
|
161
|
+
'window': [daily[0]['date'], daily[-1]['date']] if daily else None,
|
|
162
|
+
'store': {'visitors': visitors, 'acquisitions': acquisitions,
|
|
163
|
+
'conversion': round(acquisitions / visitors * 100, 1) if visitors else None},
|
|
164
|
+
'installs': {'installs': sum(d['installs'] for d in daily),
|
|
165
|
+
'uninstalls': sum(d['uninstalls'] for d in daily),
|
|
166
|
+
'active_peak': max(active, default=0), 'active_latest': active[-1] if active else 0},
|
|
167
|
+
'ratings': {'count': len(stars), 'average': round(sum(stars) / len(stars), 2) if stars else None},
|
|
168
|
+
'countries': dict(sorted(by_country.items(), key=lambda kv: -kv[1])),
|
|
169
|
+
'daily': daily,
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
# ------------------------------------------------------------------ Reporting API
|
|
174
|
+
|
|
175
|
+
def _session(ctx):
|
|
176
|
+
import google.auth.transport.requests
|
|
177
|
+
creds = auth.credentials(ctx, auth.PLAY, [auth.SCOPE_REPORTING])
|
|
178
|
+
return google.auth.transport.requests.AuthorizedSession(creds)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _call(session, method, url, **kw):
|
|
182
|
+
r = session.request(method, url, timeout=60, **kw)
|
|
183
|
+
if r.status_code in (401, 403):
|
|
184
|
+
raise CredentialMissing(f'Play Developer Reporting API refused the request ({r.status_code})',
|
|
185
|
+
hint='Enable the "Google Play Developer Reporting API" in the service '
|
|
186
|
+
"account's Cloud project and give it \"View app information\" in Play Console.",
|
|
187
|
+
code='PERMISSION_DENIED')
|
|
188
|
+
if r.status_code >= 400:
|
|
189
|
+
raise MkitError(f'Reporting API error {r.status_code}: {r.text[:300]}', code=f'HTTP_{r.status_code}')
|
|
190
|
+
return r.json()
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _date(d):
|
|
194
|
+
return {'year': d.year, 'month': d.month, 'day': d.day}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
METRICS = {
|
|
198
|
+
'crash': ('crashRateMetricSet', ['crashRate', 'userPerceivedCrashRate', 'distinctUsers']),
|
|
199
|
+
'anr': ('anrRateMetricSet', ['anrRate', 'userPerceivedAnrRate', 'distinctUsers']),
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def vitals(ctx, days=28):
|
|
204
|
+
"""Crash and ANR rates over the last `days` days Play has data for, per
|
|
205
|
+
version code, plus the overall figure for the window."""
|
|
206
|
+
session = _session(ctx)
|
|
207
|
+
package = ctx.project.package
|
|
208
|
+
out = {}
|
|
209
|
+
for key, (metric_set, metrics) in METRICS.items():
|
|
210
|
+
name = f'apps/{package}/{metric_set}'
|
|
211
|
+
meta = _call(session, 'GET', f'{REPORTING}/{name}')
|
|
212
|
+
latest = next((f['latestEndTime'] for f in meta.get('freshnessInfo', {}).get('freshnesses', [])
|
|
213
|
+
if f.get('aggregationPeriod') == 'DAILY'), None)
|
|
214
|
+
if not latest:
|
|
215
|
+
out[key] = {'rows': [], 'note': 'no data yet'}
|
|
216
|
+
continue
|
|
217
|
+
end = date(latest['year'], latest['month'], latest['day'])
|
|
218
|
+
start = end - timedelta(days=days)
|
|
219
|
+
tz = {'id': 'America/Los_Angeles'}
|
|
220
|
+
body = {'timelineSpec': {'aggregationPeriod': 'DAILY',
|
|
221
|
+
'startTime': dict(_date(start), timeZone=tz),
|
|
222
|
+
'endTime': dict(_date(end), timeZone=tz)},
|
|
223
|
+
'dimensions': ['versionCode'], 'metrics': metrics, 'pageSize': 1000}
|
|
224
|
+
rows = _call(session, 'POST', f'{REPORTING}/{name}:query', json=body).get('rows', [])
|
|
225
|
+
per_version = defaultdict(lambda: {'users': 0, 'weighted': defaultdict(float)})
|
|
226
|
+
for row in rows:
|
|
227
|
+
version = next((d.get('stringValue') or str(d.get('int64Value', '')) for d in row.get('dimensions', [])
|
|
228
|
+
if d.get('dimension') == 'versionCode'), '?')
|
|
229
|
+
values = {m['metric']: float(m.get('decimalValue', {}).get('value', m.get('int64Value', 0)) or 0)
|
|
230
|
+
for m in row.get('metrics', [])}
|
|
231
|
+
users = values.get('distinctUsers', 0)
|
|
232
|
+
entry = per_version[version]
|
|
233
|
+
entry['users'] += users
|
|
234
|
+
for m in metrics:
|
|
235
|
+
if m != 'distinctUsers':
|
|
236
|
+
entry['weighted'][m] += values.get(m, 0) * users
|
|
237
|
+
table = []
|
|
238
|
+
for version, e in sorted(per_version.items(), key=lambda kv: kv[0]):
|
|
239
|
+
users = e['users'] or 1
|
|
240
|
+
table.append({'versionCode': version, 'users': int(e['users']),
|
|
241
|
+
**{m: round(v / users * 100, 3) for m, v in e['weighted'].items()}})
|
|
242
|
+
out[key] = {'window': [str(start), str(end)], 'rows': table}
|
|
243
|
+
return out
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def crashes(ctx, limit=10):
|
|
247
|
+
session = _session(ctx)
|
|
248
|
+
data = _call(session, 'GET', f'{REPORTING}/apps/{ctx.project.package}/errorIssues:search',
|
|
249
|
+
params={'pageSize': max(1, min(limit, 50))})
|
|
250
|
+
issues = []
|
|
251
|
+
for i in data.get('errorIssues', []):
|
|
252
|
+
issues.append({'type': i.get('type'), 'cause': i.get('cause'), 'location': i.get('location'),
|
|
253
|
+
'reports': _int(i.get('errorReportCount')), 'users': _int(i.get('distinctUsers')),
|
|
254
|
+
'firstVersion': (i.get('firstAppVersion') or {}).get('versionCode'),
|
|
255
|
+
'lastVersion': (i.get('lastAppVersion') or {}).get('versionCode'),
|
|
256
|
+
'link': i.get('issueUri')})
|
|
257
|
+
issues.sort(key=lambda i: -i['reports'])
|
|
258
|
+
return issues[:limit]
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
# ------------------------------------------------------------------ reviews
|
|
262
|
+
|
|
263
|
+
def reviews(ctx, limit=50, translate=None):
|
|
264
|
+
svc = auth.play(ctx, timeout=60)
|
|
265
|
+
kw = {'packageName': ctx.project.package, 'maxResults': min(limit, 100)}
|
|
266
|
+
if translate:
|
|
267
|
+
kw['translationLanguage'] = translate
|
|
268
|
+
data = auth.retry(lambda: svc.reviews().list(**kw), 'reviews')
|
|
269
|
+
out = []
|
|
270
|
+
for r in data.get('reviews', []):
|
|
271
|
+
user = next((c['userComment'] for c in r.get('comments', []) if 'userComment' in c), {})
|
|
272
|
+
reply = next((c['developerComment'] for c in r.get('comments', []) if 'developerComment' in c), None)
|
|
273
|
+
out.append({'id': r['reviewId'], 'author': r.get('authorName', ''), 'stars': user.get('starRating'),
|
|
274
|
+
'text': user.get('text', '').strip(), 'language': user.get('reviewerLanguage'),
|
|
275
|
+
'version': user.get('appVersionName'), 'device': (user.get('deviceMetadata') or {}).get('productName'),
|
|
276
|
+
'date': (user.get('lastModified') or {}).get('seconds'),
|
|
277
|
+
'replied': bool(reply), 'reply': (reply or {}).get('text')})
|
|
278
|
+
return out
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def reply(ctx, review_id, text):
|
|
282
|
+
if len(text) > 350:
|
|
283
|
+
raise UsageError(f'reply is {len(text)} characters; Play allows 350', code='REPLY_TOO_LONG')
|
|
284
|
+
svc = auth.play(ctx, timeout=60)
|
|
285
|
+
return auth.retry(lambda: svc.reviews().reply(packageName=ctx.project.package, reviewId=review_id,
|
|
286
|
+
body={'replyText': text}), 'reply')
|