socialvec 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
socialvec/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Top-level package for SocialVec."""
2
+
3
+ __author__ = """Nir Lotan"""
4
+ __email__ = 'nir.lotan@gmail.com'
5
+ __version__ = '0.2.0'
socialvec/_download.py ADDED
@@ -0,0 +1,140 @@
1
+ """Find, download and verify the SocialVec data files (embedding models and metadata)."""
2
+ import hashlib
3
+ import io
4
+ import os
5
+ import warnings
6
+ from pathlib import Path
7
+
8
+ import requests
9
+ from yaspin import yaspin
10
+
11
+ from .exceptions import SocialVecError
12
+
13
+ PACKAGE_DIR = Path(__file__).resolve().parent
14
+
15
+ _CHUNK_SIZE = 1 << 20
16
+ # (connect, read) timeouts in seconds; the read timeout is per chunk, not for the whole file.
17
+ _TIMEOUT = (15, 120)
18
+
19
+
20
+ def cache_dir() -> Path:
21
+ """The folder downloaded files are kept in: $SOCIALVEC_HOME, or ~/.cache/socialvec."""
22
+ return Path(os.environ.get("SOCIALVEC_HOME") or Path.home() / ".cache" / "socialvec").expanduser()
23
+
24
+
25
+ def _existing_locations(filename):
26
+ """Where the file may already be: the cache, then where versions up to 0.1.7.1 saved it."""
27
+ locations = []
28
+ for folder in (cache_dir(), PACKAGE_DIR, Path.cwd()):
29
+ path = folder / filename
30
+ if path not in locations:
31
+ locations.append(path)
32
+ return locations
33
+
34
+
35
+ def _file_sha256(path):
36
+ digest = hashlib.sha256()
37
+ with open(path, "rb") as f:
38
+ for chunk in iter(lambda: f.read(_CHUNK_SIZE), b""):
39
+ digest.update(chunk)
40
+ return digest.hexdigest()
41
+
42
+
43
+ def _download(url, out):
44
+ """Write the content at url to the binary file object out; return its sha256."""
45
+ digest = hashlib.sha256()
46
+ with requests.get(url, stream=True, timeout=_TIMEOUT) as response:
47
+ response.raise_for_status()
48
+ for chunk in response.iter_content(_CHUNK_SIZE):
49
+ out.write(chunk)
50
+ digest.update(chunk)
51
+ return digest.hexdigest()
52
+
53
+
54
+ def _verify(actual, expected, label, url):
55
+ if expected and actual != expected:
56
+ raise SocialVecError(
57
+ f"The {label} downloaded from {url} is corrupted or has changed: "
58
+ f"expected sha256 {expected}, got {actual}.")
59
+
60
+
61
+ def _download_error(label, url, error):
62
+ return SocialVecError(f"Could not download the {label} from {url}: {error}")
63
+
64
+
65
+ def fetch(filename, url, sha256=None, label=None):
66
+ """
67
+ Get a SocialVec data file, downloading it on first use.
68
+
69
+ The file is looked up in the cache folder (see ``cache_dir``), then in the
70
+ package folder and the current folder, where older versions saved it.
71
+ If it isn't found (or its checksum doesn't match), it is downloaded to the
72
+ cache folder. If the cache folder can't be written, the file is downloaded
73
+ into memory instead.
74
+
75
+ Parameters
76
+ ----------
77
+ filename : name of the file, e.g. "SocialVec2020.pkl.gz"
78
+ url : where to download it from
79
+ sha256 : expected sha256 hex digest; the file is not used if it doesn't match
80
+ label : what the file is, for messages
81
+
82
+ Returns
83
+ -------
84
+ A ``pathlib.Path`` to the file, or an ``io.BytesIO`` with its contents.
85
+
86
+ Raises
87
+ ------
88
+ SocialVecError if the file can't be downloaded or doesn't match the checksum.
89
+
90
+ """
91
+ label = label or filename
92
+
93
+ for path in _existing_locations(filename):
94
+ if not path.is_file():
95
+ continue
96
+ if not sha256 or _file_sha256(path) == sha256:
97
+ return path
98
+ warnings.warn(f"Ignoring {path}: it does not match the expected checksum of the {label}.")
99
+
100
+ target = cache_dir() / filename
101
+ partial = target.with_name(target.name + ".part")
102
+ try:
103
+ target.parent.mkdir(parents=True, exist_ok=True)
104
+ with yaspin(text=f"Downloading the {label} to {target}") as spinner:
105
+ try:
106
+ with open(partial, "wb") as f:
107
+ actual = _download(url, f)
108
+ _verify(actual, sha256, label, url)
109
+ os.replace(partial, target)
110
+ except BaseException:
111
+ spinner.fail("❌ ")
112
+ try:
113
+ partial.unlink()
114
+ except OSError:
115
+ pass
116
+ raise
117
+ spinner.ok("✅ ")
118
+ return target
119
+ except requests.RequestException as e:
120
+ # RequestException is an OSError too, so this must come before the OSError handler.
121
+ raise _download_error(label, url, e) from e
122
+ except OSError as e:
123
+ warnings.warn(
124
+ f"Could not save the {label} to {target.parent} ({e}); loading it into memory instead, "
125
+ f"so it will be downloaded again next time. Set SOCIALVEC_HOME to a writable folder to avoid this.")
126
+
127
+ buffer = io.BytesIO()
128
+ with yaspin(text=f"Downloading the {label} into memory") as spinner:
129
+ try:
130
+ actual = _download(url, buffer)
131
+ _verify(actual, sha256, label, url)
132
+ except requests.RequestException as e:
133
+ spinner.fail("❌ ")
134
+ raise _download_error(label, url, e) from e
135
+ except BaseException:
136
+ spinner.fail("❌ ")
137
+ raise
138
+ spinner.ok("✅ ")
139
+ buffer.seek(0)
140
+ return buffer
socialvec/api.py ADDED
@@ -0,0 +1,279 @@
1
+ """HTTP API for SocialVec.
2
+
3
+ Run it with ``python -m socialvec.api``, or with a WSGI server, e.g.
4
+ ``gunicorn "socialvec.api:create_app()"``.
5
+
6
+ Configuration (environment variables):
7
+
8
+ * ``SOCIALVEC_MODEL`` - the model to load (default ``"2020"``). The classifier
9
+ endpoints (``/predict``, ``/predict_proba``) need ``"2020c"``.
10
+ * ``SOCIALVEC_CORS_ORIGINS`` - comma-separated list of origins allowed to call
11
+ the API from a browser (default ``*``, any origin).
12
+ * ``SOCIALVEC_HOST`` - address to listen on (default ``127.0.0.1``; use
13
+ ``0.0.0.0`` to accept connections from other machines).
14
+ * ``PORT`` - port to listen on (default ``8000``).
15
+
16
+ ``SOCIALVEC_HOST`` and ``PORT`` only apply to ``python -m socialvec.api``.
17
+ """
18
+ import logging
19
+ import os
20
+
21
+ import numpy as np
22
+ from flask import Blueprint, Flask, current_app, jsonify, request
23
+ from flask_cors import CORS
24
+ from werkzeug.exceptions import HTTPException
25
+
26
+ from socialvec.exceptions import InvalidUserIdError, SocialVecError, UserNotFoundError
27
+
28
+ logger = logging.getLogger(__name__)
29
+
30
+ DEFAULT_MODEL = "2020"
31
+ DEFAULT_HOST = "127.0.0.1"
32
+ DEFAULT_PORT = 8000
33
+ DEFAULT_CORS_ORIGINS = "*"
34
+
35
+ TOPN_DEFAULT = 10
36
+ TOPN_MIN = 1
37
+ TOPN_MAX = 100
38
+
39
+ _EXTENSION_KEY = "socialvec"
40
+
41
+
42
+ class ApiError(Exception):
43
+ """An error caused by the request (missing or malformed input) or by the server setup."""
44
+
45
+ def __init__(self, message, status=400):
46
+ super().__init__(message)
47
+ self.message = message
48
+ self.status = status
49
+
50
+
51
+ bp = Blueprint("socialvec", __name__)
52
+
53
+
54
+ def _state():
55
+ return current_app.extensions[_EXTENSION_KEY]
56
+
57
+
58
+ def _sv():
59
+ return _state()["sv"]
60
+
61
+
62
+ def _classifier():
63
+ state = _state()
64
+ if state["classifier_error"] is not None:
65
+ raise ApiError(state["classifier_error"], status=501)
66
+ return state["sv"].classifier
67
+
68
+
69
+ def _require_arg(name):
70
+ value = request.args.get(name)
71
+ if value is None or value == "":
72
+ raise ApiError(f"Missing required query parameter '{name}'")
73
+ return value
74
+
75
+
76
+ def _json_body():
77
+ data = request.get_json(silent=True)
78
+ if data is None:
79
+ data = {}
80
+ if not isinstance(data, dict):
81
+ raise ApiError("Request body must be a JSON object")
82
+ return data
83
+
84
+
85
+ def _require_entity(data, name):
86
+ value = data.get(name)
87
+ if value is None or value == "" or isinstance(value, bool) or not isinstance(value, (str, int)):
88
+ raise ApiError(f"'{name}' is required and must be a user ID or a username")
89
+ return value
90
+
91
+
92
+ def _require_classifier_name(data):
93
+ value = data.get("classifier_name")
94
+ if not isinstance(value, str) or not value:
95
+ raise ApiError("'classifier_name' is required and must be a string")
96
+ return value
97
+
98
+
99
+ def _parse_topn():
100
+ raw = request.args.get("topn")
101
+ if raw is None or raw == "":
102
+ return TOPN_DEFAULT
103
+ try:
104
+ topn = int(raw)
105
+ except ValueError:
106
+ raise ApiError(f"'topn' must be an integer, got {raw!r}")
107
+ return max(TOPN_MIN, min(TOPN_MAX, topn))
108
+
109
+
110
+ @bp.route('/validate_userid', methods=['GET'])
111
+ def validate_userid():
112
+ result = _sv().validate_userid(_require_arg('userid'))
113
+ return jsonify({'userid': result})
114
+
115
+
116
+ @bp.route('/validate_username', methods=['GET'])
117
+ def validate_username():
118
+ result = _sv().validate_username(_require_arg('username'))
119
+ return jsonify({'userid': result})
120
+
121
+
122
+ @bp.route('/get_screen_name', methods=['GET'])
123
+ def get_screen_name():
124
+ result = _sv().get_screen_name(_require_arg('userid'))
125
+ return jsonify({'screen_name': result})
126
+
127
+
128
+ @bp.route('/get_userid', methods=['GET'])
129
+ def get_userid():
130
+ result = _sv().get_userid(_require_arg('username'))
131
+ return jsonify({'userid': result})
132
+
133
+
134
+ @bp.route('/get_similar', methods=['GET'])
135
+ def get_similar():
136
+ input_val = _require_arg('input')
137
+ topn = _parse_topn()
138
+ df = _sv().get_similar(input_val, topn)
139
+ return current_app.response_class(df.to_json(orient='records'), mimetype='application/json')
140
+
141
+
142
+ @bp.route('/get_embeddings', methods=['GET'])
143
+ def get_embeddings():
144
+ emb = _sv().get_embeddings(_require_arg('entity'))
145
+ return jsonify({'embeddings': np.asarray(emb).tolist()})
146
+
147
+
148
+ @bp.route('/get_average_embeddings', methods=['POST'])
149
+ def get_average_embeddings():
150
+ data = _json_body()
151
+ entity_list = data.get('entity_list')
152
+ if not isinstance(entity_list, list):
153
+ raise ApiError("'entity_list' is required and must be a list of user IDs")
154
+ avg_emb, count = _sv().get_average_embeddings(entity_list)
155
+ return jsonify({'average_embeddings': np.asarray(avg_emb).tolist(), 'count': int(count)})
156
+
157
+
158
+ @bp.route('/get_similarity', methods=['GET'])
159
+ def get_similarity():
160
+ sim = _sv().get_similarity(_require_arg('entity1'), _require_arg('entity2'))
161
+ return jsonify({'similarity': float(sim)})
162
+
163
+
164
+ @bp.route('/predict', methods=['POST'])
165
+ def predict():
166
+ data = _json_body()
167
+ classifier_name = _require_classifier_name(data)
168
+ entity = _require_entity(data, 'entity')
169
+ classifier = _classifier()
170
+ emb = _sv().get_embeddings(entity)
171
+ affiliation, pred_proba = classifier.predict(classifier_name, np.asarray(emb))
172
+ return jsonify({'affiliation': affiliation, 'confidence': float(pred_proba)})
173
+
174
+
175
+ @bp.route('/predict_proba', methods=['POST'])
176
+ def predict_proba():
177
+ data = _json_body()
178
+ classifier_name = _require_classifier_name(data)
179
+ entity = _require_entity(data, 'entity')
180
+ classifier = _classifier()
181
+ emb = _sv().get_embeddings(entity)
182
+ proba = classifier.predict_proba(classifier_name, np.asarray(emb))
183
+ return jsonify({'probability': float(proba)})
184
+
185
+
186
+ def _error(message, status):
187
+ return jsonify({'error': message}), status
188
+
189
+
190
+ @bp.app_errorhandler(ApiError)
191
+ def _handle_api_error(e):
192
+ return _error(e.message, e.status)
193
+
194
+
195
+ @bp.app_errorhandler(InvalidUserIdError)
196
+ def _handle_invalid_userid(e):
197
+ return _error(str(e), 400)
198
+
199
+
200
+ @bp.app_errorhandler(UserNotFoundError)
201
+ def _handle_user_not_found(e):
202
+ return _error(str(e), 404)
203
+
204
+
205
+ @bp.app_errorhandler(SocialVecError)
206
+ def _handle_socialvec_error(e):
207
+ return _error(str(e), 400)
208
+
209
+
210
+ @bp.app_errorhandler(HTTPException)
211
+ def _handle_http_exception(e):
212
+ return _error(e.description, e.code)
213
+
214
+
215
+ @bp.app_errorhandler(Exception)
216
+ def _handle_unexpected(e):
217
+ logger.exception("Unhandled error in %s %s", request.method, request.path)
218
+ return _error("Internal server error", 500)
219
+
220
+
221
+ def _init_classifier(sv):
222
+ """Load the classifiers once, up front. Returns an error message if they aren't available."""
223
+ try:
224
+ sv.init_classifier()
225
+ except SocialVecError as e:
226
+ logger.info("Classifier endpoints are disabled: %s", e)
227
+ return (f"Classifiers are not available with model '{sv.model_name}': {e} "
228
+ f"Start the API with SOCIALVEC_MODEL=2020c to enable them.")
229
+ return None
230
+
231
+
232
+ def _cors_origins(value):
233
+ origins = [o.strip() for o in value.split(",") if o.strip()]
234
+ if not origins or origins == ["*"]:
235
+ return "*"
236
+ return origins
237
+
238
+
239
+ def create_app(sv=None, model_name=None, cors_origins=None):
240
+ """
241
+ Create the Flask app.
242
+
243
+ Parameters
244
+ ----------
245
+ sv : an already loaded SocialVec instance. If omitted, one is created with
246
+ ``model_name``.
247
+ model_name : the model to load when ``sv`` is not given. Defaults to the
248
+ ``SOCIALVEC_MODEL`` environment variable, or "2020".
249
+ cors_origins : comma-separated allowed origins. Defaults to the
250
+ ``SOCIALVEC_CORS_ORIGINS`` environment variable, or "*".
251
+ """
252
+ if sv is None:
253
+ from socialvec.socialvec import SocialVec
254
+ sv = SocialVec(model_name=model_name or os.environ.get("SOCIALVEC_MODEL", DEFAULT_MODEL))
255
+
256
+ if cors_origins is None:
257
+ cors_origins = os.environ.get("SOCIALVEC_CORS_ORIGINS", DEFAULT_CORS_ORIGINS)
258
+
259
+ app = Flask(__name__)
260
+ CORS(app, origins=_cors_origins(cors_origins))
261
+ app.extensions[_EXTENSION_KEY] = {
262
+ "sv": sv,
263
+ # Loaded here, before any request is served, so requests never race to initialize it.
264
+ "classifier_error": _init_classifier(sv),
265
+ }
266
+ app.register_blueprint(bp)
267
+ return app
268
+
269
+
270
+ def main():
271
+ logging.basicConfig(level=logging.INFO)
272
+ host = os.environ.get("SOCIALVEC_HOST", DEFAULT_HOST)
273
+ port = int(os.environ.get("PORT", DEFAULT_PORT))
274
+ app = create_app()
275
+ app.run(host=host, port=port)
276
+
277
+
278
+ if __name__ == '__main__':
279
+ main()
socialvec/cli.py ADDED
@@ -0,0 +1,88 @@
1
+ """Console script for socialvec."""
2
+ import sys
3
+
4
+ import click
5
+
6
+ from socialvec import __version__
7
+ from socialvec.exceptions import SocialVecError
8
+
9
+ DEFAULT_MODEL = "2020"
10
+ CLASSIFIER_MODEL = "2020c"
11
+
12
+
13
+ def _load(model_name):
14
+ # Imported here so `socialvec --help` stays fast.
15
+ from socialvec.socialvec import SocialVec
16
+ return SocialVec(model_name=model_name)
17
+
18
+
19
+ def _model_option(default):
20
+ return click.option("--model", "model_name", default=default, show_default=True,
21
+ help="SocialVec model to use.")
22
+
23
+
24
+ @click.group(context_settings={"help_option_names": ["-h", "--help"]})
25
+ @click.version_option(__version__, prog_name="socialvec")
26
+ def main():
27
+ """SocialVec: social embeddings of popular Twitter accounts."""
28
+
29
+
30
+ @main.command()
31
+ @click.argument("user")
32
+ @click.option("--topn", "-n", default=10, show_default=True, type=click.IntRange(1, 1000),
33
+ help="Number of similar accounts to show.")
34
+ @_model_option(DEFAULT_MODEL)
35
+ def similar(user, topn, model_name):
36
+ """Show the accounts most similar to USER (a user ID or username)."""
37
+ sv = _load(model_name)
38
+ df = sv.get_similar(user, topn)
39
+ click.echo(df[["twitter_id", "similarity", "screen_name", "name"]].to_string(index=False))
40
+
41
+
42
+ @main.command()
43
+ @click.argument("user1")
44
+ @click.argument("user2")
45
+ @_model_option(DEFAULT_MODEL)
46
+ def similarity(user1, user2, model_name):
47
+ """Show the cosine similarity between USER1 and USER2."""
48
+ sv = _load(model_name)
49
+ click.echo(f"{sv.get_similarity(user1, user2):.4f}")
50
+
51
+
52
+ @main.command()
53
+ @click.argument("classifier")
54
+ @click.argument("user")
55
+ @_model_option(CLASSIFIER_MODEL)
56
+ def predict(classifier, user, model_name):
57
+ """Predict a trait of USER with CLASSIFIER (e.g. political, gender, age)."""
58
+ sv = _load(model_name)
59
+ sv.init_classifier()
60
+ label, confidence = sv.classifier.predict(classifier, sv[user])
61
+ click.echo(f"{label}\t(confidence {confidence:.3f})")
62
+
63
+
64
+ @main.command()
65
+ @_model_option(DEFAULT_MODEL)
66
+ def download(model_name):
67
+ """Download a model and the account metadata ahead of time."""
68
+ _load(model_name)
69
+ click.echo(f"Model {model_name} is ready.")
70
+
71
+
72
+ def run():
73
+ """Entry point: show SocialVec errors as a message instead of a traceback."""
74
+ try:
75
+ return main(standalone_mode=False)
76
+ except click.exceptions.Abort:
77
+ click.echo("Aborted!", err=True)
78
+ return 1
79
+ except click.ClickException as e:
80
+ e.show()
81
+ return e.exit_code
82
+ except SocialVecError as e:
83
+ click.echo(f"Error: {e}", err=True)
84
+ return 1
85
+
86
+
87
+ if __name__ == "__main__":
88
+ sys.exit(run()) # pragma: no cover
socialvec/config.yaml ADDED
@@ -0,0 +1,48 @@
1
+ models:
2
+ - name: "2020"
3
+ filename: SocialVec2020.pkl.gz
4
+ download_url: https://www.dropbox.com/s/lti8npc8r2l4wch/SocialVec2020.pkl.gz?dl=1
5
+ sha256: 7a820239ced548010c4da0354f662feca1b1388a224ff3a672834d85dc2167ad
6
+ - name: "2020c"
7
+ filename: SocialVec2020c.pkl.gz
8
+ download_url: https://www.dropbox.com/scl/fi/1s1kay2z4a430lkz1erjn/SocialVec2020c.pkl.gz?rlkey=fcfbb83gkdb80ghivzt62ljmk&dl=1
9
+ sha256: 979c5b55fc2197a001a1cc538bbd185ab40cf9257b68f6170f4a145caa12f645
10
+ - name: "2020_2022"
11
+ filename: SocialVec2020_2022.pkl.gz
12
+ download_url: https://www.dropbox.com/s/5s9mykbwe0nxir6/SocialVec2020_2022.pkl.gz?dl=1
13
+ sha256: 7719b179778feb1d9c781ac0608413fb07638a9e95a2beea9bdabf560032181a
14
+ metadata:
15
+ filename: metadata.parquet.gz
16
+ download_url: https://www.dropbox.com/s/060xprg4emblqwu/metadata.parquet.gz?dl=1
17
+ sha256: ffd6d9143b3665072d9862ca232f05ef21a2198f58522cfb37b918e77fc91902
18
+ classification_models:
19
+ - name: "2020c"
20
+ classifiers:
21
+ - name: "political"
22
+ filename: "models/SocialVec_political_2022.h5"
23
+ class0: Republican
24
+ class1: Democrat
25
+ - name: "gender"
26
+ filename: "models/SocialVec_gender_2022.h5"
27
+ class0: Male
28
+ class1: Female
29
+ - name: "children"
30
+ filename: "models/SocialVec_children_2022.h5"
31
+ class0: "Yes"
32
+ class1: "No"
33
+ - name: "age"
34
+ filename: "models/SocialVec_age_2022.h5"
35
+ class0: Under25
36
+ class1: Over25
37
+ - name: "education"
38
+ filename: "models/SocialVec_education_2022.h5"
39
+ class0: Degree
40
+ class1: HighSchool
41
+ - name: "income"
42
+ filename: "models/SocialVec_income_2022.h5"
43
+ class0: High
44
+ class1: Low
45
+ - name: "ethnicity"
46
+ filename: "models/SocialVec_ethnicity_2022.h5"
47
+ class0: Caucasian
48
+ class1: AfrAmerican
@@ -0,0 +1,21 @@
1
+ """Exceptions raised by SocialVec."""
2
+
3
+
4
+ class SocialVecError(Exception):
5
+ """Base class for all SocialVec errors."""
6
+
7
+
8
+ class InvalidUserIdError(SocialVecError, ValueError):
9
+ """The value given is not a valid Twitter user ID or username."""
10
+
11
+
12
+ class UserNotFoundError(SocialVecError, KeyError):
13
+ """The user is not in the loaded SocialVec model (or its metadata)."""
14
+
15
+ def __str__(self):
16
+ # KeyError wraps its message in quotes; show it like a normal error.
17
+ return str(self.args[0]) if self.args else ""
18
+
19
+
20
+ class UnknownModelError(SocialVecError, ValueError):
21
+ """The requested SocialVec model name is not in the configuration."""
Binary file
socialvec/socialvec.py ADDED
@@ -0,0 +1,409 @@
1
+ """Main module."""
2
+ import gzip
3
+ import numbers
4
+ import os
5
+ import pickle
6
+
7
+ import numpy as np
8
+ import pandas as pd
9
+ import yaml
10
+ from sklearn.metrics.pairwise import cosine_similarity
11
+ from yaspin import yaspin
12
+
13
+ from ._download import fetch
14
+ from .exceptions import SocialVecError, InvalidUserIdError, UnknownModelError, UserNotFoundError
15
+
16
+ # TensorFlow is only needed by SocialVecClassifier, which imports it when it is created,
17
+ # so the embeddings can be used without TensorFlow installed.
18
+
19
+
20
+ def _read_config():
21
+ with open(os.path.join(os.path.dirname(__file__), "config.yaml"), 'r') as f:
22
+ return yaml.safe_load(f)
23
+
24
+
25
+ class SocialVec():
26
+ def __init__(self, model_name="2020"):
27
+
28
+ self.config = _read_config()
29
+
30
+ entry = next((m for m in self.config['models'] if m['name'] == model_name), None)
31
+ if entry is None:
32
+ available = [m['name'] for m in self.config['models']]
33
+ raise UnknownModelError(f"Unknown SocialVec model {model_name!r}; available models: {available}")
34
+ self.model_name = model_name
35
+
36
+ model_file = fetch(entry['filename'], entry['download_url'], entry.get('sha256'),
37
+ label=f"SocialVec {model_name} model")
38
+ with yaspin(text="Initializing Model") as spinner:
39
+ with gzip.open(model_file, 'rb') as pickle_file:
40
+ self.sv = pickle.load(pickle_file)
41
+ spinner.ok("✅ ")
42
+
43
+ metadata = self.config['metadata']
44
+ metadata_file = fetch(metadata['filename'], metadata['download_url'], metadata.get('sha256'),
45
+ label="SocialVec metadata")
46
+ with yaspin(text="Load Metadata") as spinner:
47
+ self.entities = pd.read_parquet(metadata_file, engine="fastparquet")
48
+ spinner.ok("✅ ")
49
+
50
+ self._build_lookup_tables()
51
+
52
+ def _build_lookup_tables(self):
53
+ """
54
+ Build the username -> ID and ID -> screen name lookups once, at load time.
55
+
56
+ A screen name can appear on more than one account (case-insensitively).
57
+ Accounts that are in the loaded model are preferred, so a lookup by name
58
+ finds the account that actually has an embedding.
59
+ """
60
+ ids = self.entities['twitter_id'].astype(str)
61
+ names = self.entities['screen_name'].astype(str)
62
+ in_model = ids.isin(self.sv.wv.key_to_index)
63
+
64
+ names_df = pd.DataFrame({'name': names.str.lower(), 'twitter_id': ids, 'in_model': in_model})
65
+ names_df = names_df.sort_values('in_model', ascending=False, kind='stable')
66
+ names_df = names_df.drop_duplicates('name', keep='first')
67
+ self._name_to_id = dict(zip(names_df['name'], names_df['twitter_id']))
68
+
69
+ id_df = pd.DataFrame({'twitter_id': ids, 'screen_name': self.entities['screen_name']})
70
+ id_df = id_df.drop_duplicates('twitter_id', keep='first')
71
+ self._id_to_screen_name = dict(zip(id_df['twitter_id'], id_df['screen_name']))
72
+
73
+ @staticmethod
74
+ def _is_userid_like(value) -> bool:
75
+ """True for ints (including numpy ints) and strings made only of digits."""
76
+ if isinstance(value, bool):
77
+ return False
78
+ return isinstance(value, numbers.Integral) or (isinstance(value, str) and value.isdigit())
79
+
80
+ @staticmethod
81
+ def _normalize_userid(userid) -> str:
82
+ """Convert an int / numpy int / digit string to the string ID used by the model."""
83
+ if isinstance(userid, bool) or userid is None:
84
+ raise InvalidUserIdError(f"Invalid user id: {userid!r}")
85
+ if isinstance(userid, numbers.Integral):
86
+ return str(int(userid))
87
+ if isinstance(userid, str) and userid.isdigit():
88
+ return userid
89
+ raise InvalidUserIdError(
90
+ f"User id must be an integer or a string with integer value, got {userid!r}")
91
+
92
+ def _resolve(self, entity) -> str:
93
+ """Resolve a user ID or a username to a user ID that is in the model."""
94
+ if self._is_userid_like(entity):
95
+ return self.validate_userid(entity)
96
+ if isinstance(entity, str):
97
+ return self.validate_username(entity)
98
+ raise InvalidUserIdError(f"Expected a user ID or a username, got {entity!r}")
99
+
100
+ def validate_userid(self, userid) -> str:
101
+ """
102
+ Validate the requested user ID and convert it to a string if needed.
103
+
104
+ Parameters
105
+ ----------
106
+ userid : Twitter user ID, as an int (including numpy ints) or a string of digits
107
+
108
+ Returns
109
+ -------
110
+ The user ID as a string.
111
+
112
+ Raises
113
+ ------
114
+ InvalidUserIdError if the value is not a user ID,
115
+ UserNotFoundError if the user is not in the loaded model.
116
+
117
+ """
118
+ userid = self._normalize_userid(userid)
119
+
120
+ if userid not in self.sv.wv.key_to_index:
121
+ raise UserNotFoundError(f"User {userid} not in SocialVec model {self.model_name}")
122
+
123
+ return userid
124
+
125
+ def validate_username(self, username) -> str:
126
+ """
127
+ Look up a username (case-insensitive) and return its user ID.
128
+
129
+ Parameters
130
+ ----------
131
+ username : Twitter screen name, e.g. "jack"
132
+
133
+ Returns
134
+ -------
135
+ The user ID (string) of an account with that name that is in the loaded model.
136
+
137
+ Raises
138
+ ------
139
+ InvalidUserIdError if username is not a string,
140
+ UserNotFoundError if no account with that name is in the loaded model.
141
+
142
+ """
143
+ userid = self.get_userid(username)
144
+ if userid not in self.sv.wv.key_to_index:
145
+ raise UserNotFoundError(
146
+ f"User '{username}' is in the SocialVec metadata but not in model {self.model_name}")
147
+ return userid
148
+
149
+ def get_screen_name(self, userid):
150
+ """
151
+ Get the screen name for a given user ID.
152
+
153
+ Parameters
154
+ ----------
155
+ userid : string or int representing the Twitter user ID
156
+
157
+ Returns
158
+ -------
159
+ The Twitter screen name, or None if the user is in the model but has no metadata.
160
+
161
+ Raises
162
+ ------
163
+ InvalidUserIdError / UserNotFoundError, as in validate_userid.
164
+
165
+ """
166
+ userid = self.validate_userid(userid)
167
+ return self._id_to_screen_name.get(userid)
168
+
169
+ def get_userid(self, username: str) -> str:
170
+ """
171
+ Get the user ID for a given screen name (case-insensitive).
172
+
173
+ If several accounts share the name, an account that is in the loaded
174
+ model is preferred. The returned account may still be one that only
175
+ exists in the metadata; use validate_username to require an embedding.
176
+
177
+ Parameters
178
+ ----------
179
+ username : Twitter screen name
180
+
181
+ Returns
182
+ -------
183
+ The Twitter user ID as a string.
184
+
185
+ Raises
186
+ ------
187
+ InvalidUserIdError if username is not a string,
188
+ UserNotFoundError if the name is not in the SocialVec metadata.
189
+
190
+ """
191
+ if not isinstance(username, str):
192
+ raise InvalidUserIdError(f"Username must be a string, got {username!r}")
193
+ userid = self._name_to_id.get(username.lower())
194
+ if userid is None:
195
+ raise UserNotFoundError(f"User '{username}' not in SocialVec metadata")
196
+ return userid
197
+
198
+ def get_similar(self, input, topn: int = 10):
199
+ """
200
+ This function returns the topn similar entities for a given entity
201
+
202
+ Parameters
203
+ ----------
204
+ input : a user ID, a username, an embedding vector, or a list of user IDs / usernames (whose embeddings are averaged)
205
+ topn : requested number of similar entities
206
+
207
+ Returns
208
+ -------
209
+ Pandas dataframe with the top n similar entities details
210
+
211
+ """
212
+
213
+ if isinstance(input, (list, tuple)):
214
+ input = [item if isinstance(item, np.ndarray) else self._resolve(item) for item in input]
215
+ elif not isinstance(input, np.ndarray):
216
+ input = self._resolve(input)
217
+
218
+ sim = self.sv.wv.most_similar(input, topn=topn)
219
+ similar = pd.DataFrame(sim, columns=['twitter_id', 'similarity'])
220
+ return pd.merge(similar, self.entities, on='twitter_id', how='left')
221
+
222
+ def __getitem__(self, key):
223
+ """
224
+ Overload [] operator to return the entity embeddings
225
+
226
+ Parameters
227
+ ----------
228
+ key - integer, string with user ID or string with username
229
+
230
+ Returns
231
+ -------
232
+ SocialVec embeddings vector of the popular entity
233
+
234
+ """
235
+ return self.sv.wv[self._resolve(key)]
236
+
237
+ def get_embeddings(self, entity):
238
+ """
239
+ This function is a different way to trigger the [] operator
240
+ """
241
+ return self[entity]
242
+
243
+ def get_average_embeddings(self, entity_list, type=""):
244
+ """
245
+ This function returns the average embeddings for a given list of Twitter user IDs
246
+
247
+ Parameters
248
+ ----------
249
+ entity_list - list of entities IDs
250
+ type - will be extended for future use with usernames
251
+
252
+ Returns
253
+ -------
254
+ A tuple of the (1) average vector (2) the number of popular entities on which it was based.
255
+ IDs that are not in the model are skipped; if none are left, the vector is all zeros.
256
+
257
+ """
258
+
259
+ popular_entities = []
260
+ for entity in entity_list:
261
+ try:
262
+ popular_entities.append(self.validate_userid(entity))
263
+ except SocialVecError:
264
+ continue
265
+
266
+ if len(popular_entities) != 0:
267
+ sv = np.mean(self.sv.wv[popular_entities], axis=0)
268
+ else:
269
+ sv = np.zeros(self.sv.wv.vector_size, dtype=np.float32)
270
+
271
+ return sv, len(popular_entities)
272
+
273
+ def get_similarity(self, entity1, entity2):
274
+ """
275
+ This function returns the similarity between two entities
276
+
277
+ Parameters
278
+ ----------
279
+ entity1 - first entity ID
280
+ entity2 - second entity ID
281
+
282
+ Returns
283
+ -------
284
+ The cosine similarity score between the two entities
285
+
286
+ """
287
+
288
+ if type(entity1)==np.ndarray:
289
+ v1 = entity1
290
+ else:
291
+ v1 = self[entity1]
292
+ if type(entity2) == np.ndarray:
293
+ v2 = entity2
294
+ else:
295
+ v2 = self[entity2]
296
+
297
+ return cosine_similarity(v1.reshape(1, -1),v2.reshape(1, -1))[0][0]
298
+
299
+ def init_classifier(self):
300
+ self.classifier = self.SocialVecClassifier(self.model_name)
301
+
302
+ class SocialVecClassifier():
303
+ def __init__(self, model_name):
304
+
305
+ current_folder = os.path.dirname(__file__)
306
+ self.config = _read_config()
307
+
308
+ # Classifiers are trained on one model's embedding space and can't be used with another
309
+ entry = next((m for m in self.config['classification_models'] if m['name'] == model_name), None)
310
+ if entry is None:
311
+ available = [m['name'] for m in self.config['classification_models']]
312
+ raise SocialVecError(
313
+ f"No classifiers are available for model '{model_name}'. "
314
+ f"Classifiers exist for: {available}, e.g. SocialVec(model_name='{available[0]}')")
315
+
316
+ try:
317
+ import tensorflow as tf
318
+ from tensorflow import keras
319
+ except ImportError as e:
320
+ raise SocialVecError(
321
+ "The SocialVec classifiers need TensorFlow, which could not be imported "
322
+ f"({e}). Install it with: pip install 'socialvec[classifier]'") from e
323
+ self._tf = tf
324
+
325
+ keras.utils.get_custom_objects().update({'recall_m': self.recall_m,
326
+ 'precision_m': self.precision_m,
327
+ 'f1_m': self.f1_m})
328
+
329
+ self.models_dict = {}
330
+ self.models_classes = {}
331
+
332
+ for classifier in entry['classifiers']:
333
+ classifier_name = classifier['name']
334
+ classifier_path = os.path.join(current_folder, classifier['filename'])
335
+ if not os.path.exists(classifier_path):
336
+ raise SocialVecError(
337
+ f"Configuration error: classifier {classifier_name} not found at {classifier_path}")
338
+ self.models_dict[classifier_name] = keras.models.load_model(classifier_path)
339
+ self.models_classes[classifier_name] = {'class0': classifier['class0'],
340
+ 'class1': classifier['class1']}
341
+
342
+ @property
343
+ def classifier_names(self):
344
+ return list(self.models_dict)
345
+
346
+ def _raw_predict(self, classifier_name, v):
347
+ if classifier_name not in self.models_dict:
348
+ raise SocialVecError(
349
+ f"Unknown classifier {classifier_name!r}; available: {self.classifier_names}")
350
+ model = self.models_dict[classifier_name]
351
+ param_to_predict = self._tf.convert_to_tensor(np.asarray(v, dtype=np.float32).reshape(1, -1))
352
+ return model.predict(param_to_predict, verbose=False)[0][0]
353
+
354
+ def predict_proba(self, classifier_name, v):
355
+ """
356
+ Return the confidence of the prediction, between 0 and 1.
357
+
358
+ Despite the name, this is not the probability of a specific class:
359
+ it is abs(p - 0.5) * 2 for the classifier's raw output p, i.e. 1 when the
360
+ classifier is certain of either class and 0 when it can't tell them apart.
361
+ Use predict() to also get which class was predicted.
362
+ """
363
+ return abs(self._raw_predict(classifier_name, v) - 0.5)*2
364
+
365
+ def predict(self, classifier_name, v):
366
+ """
367
+ Classify an embedding vector.
368
+
369
+ Returns
370
+ -------
371
+ A tuple (class label, confidence between 0 and 1); see predict_proba for the confidence.
372
+ """
373
+ prediction = self._raw_predict(classifier_name, v)
374
+
375
+ # Original prediction is 0 for Democrat, 1 for Republican.
376
+ # We convert it here to a confidence interval between 0 to 1 for either of the classes
377
+ pred_proba = abs(prediction - 0.5)*2
378
+
379
+ if round(prediction):
380
+ affiliation = self.models_classes[classifier_name]['class0']
381
+ else:
382
+ affiliation = self.models_classes[classifier_name]['class1']
383
+
384
+ return (affiliation, pred_proba)
385
+
386
+ # Custom metrics the classifiers were saved with; Keras needs them to load the .h5 files.
387
+
388
+ @staticmethod
389
+ def recall_m(y_true, y_pred):
390
+ from tensorflow.keras import backend as K
391
+ true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
392
+ possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
393
+ recall = true_positives / (possible_positives + K.epsilon())
394
+ return recall
395
+
396
+ @staticmethod
397
+ def precision_m(y_true, y_pred):
398
+ from tensorflow.keras import backend as K
399
+ true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
400
+ predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
401
+ precision = true_positives / (predicted_positives + K.epsilon())
402
+ return precision
403
+
404
+ @staticmethod
405
+ def f1_m(y_true, y_pred):
406
+ from tensorflow.keras import backend as K
407
+ precision = SocialVec.SocialVecClassifier.precision_m(y_true, y_pred)
408
+ recall = SocialVec.SocialVecClassifier.recall_m(y_true, y_pred)
409
+ return 2*((precision*recall)/(precision+recall+K.epsilon()))
@@ -0,0 +1,275 @@
1
+ Metadata-Version: 2.4
2
+ Name: socialvec
3
+ Version: 0.2.0
4
+ Summary: Pre-trained social embeddings of popular Twitter accounts, for eliciting social world knowledge.
5
+ Home-page: https://github.com/nirlotan/SocialVec
6
+ Author: Nir Lotan
7
+ Author-email: nir.lotan@gmail.com
8
+ License: MIT
9
+ Project-URL: Paper, https://doi.org/10.1371/journal.pone.0283700
10
+ Project-URL: Source, https://github.com/nirlotan/SocialVec
11
+ Project-URL: Issues, https://github.com/nirlotan/SocialVec/issues
12
+ Project-URL: Changelog, https://github.com/nirlotan/SocialVec/blob/master/HISTORY.rst
13
+ Keywords: socialvec,embeddings,social network,twitter,knowledge,word2vec
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Intended Audience :: Science/Research
17
+ Classifier: Natural Language :: English
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Topic :: Sociology
25
+ Requires-Python: >=3.10
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: click>=8.1.3
29
+ Requires-Dist: pandas>=1.5.0
30
+ Requires-Dist: numpy>=1.23.3
31
+ Requires-Dist: fastparquet>=0.8.3
32
+ Requires-Dist: pyarrow
33
+ Requires-Dist: requests
34
+ Requires-Dist: yaspin
35
+ Requires-Dist: PyYAML
36
+ Requires-Dist: gensim>4
37
+ Requires-Dist: scikit-learn
38
+ Provides-Extra: classifier
39
+ Requires-Dist: tensorflow; extra == "classifier"
40
+ Provides-Extra: api
41
+ Requires-Dist: flask; extra == "api"
42
+ Requires-Dist: flask-cors; extra == "api"
43
+ Dynamic: author
44
+ Dynamic: author-email
45
+ Dynamic: classifier
46
+ Dynamic: description
47
+ Dynamic: description-content-type
48
+ Dynamic: home-page
49
+ Dynamic: keywords
50
+ Dynamic: license
51
+ Dynamic: license-file
52
+ Dynamic: project-url
53
+ Dynamic: provides-extra
54
+ Dynamic: requires-dist
55
+ Dynamic: requires-python
56
+ Dynamic: summary
57
+
58
+ # SocialVec
59
+
60
+ [![PyPI](https://img.shields.io/pypi/v/socialvec.svg)](https://pypi.org/project/socialvec/)
61
+ [![Python versions](https://img.shields.io/pypi/pyversions/socialvec.svg)](https://pypi.org/project/socialvec/)
62
+ [![Tests](https://github.com/nirlotan/SocialVec/actions/workflows/tests.yml/badge.svg)](https://github.com/nirlotan/SocialVec/actions/workflows/tests.yml)
63
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/nirlotan/SocialVec/blob/master/LICENSE)
64
+
65
+ **SocialVec** provides pre-trained *social embeddings* for about 200,000 popular Twitter accounts, learned from the follow graph of 1.3 million users. Accounts that are often followed by the same people end up close together, the same way words that appear in similar contexts do in word embeddings. The result captures *social world knowledge*: how people relate to entities, rather than facts about them.
66
+
67
+ SocialVec was developed by Nir Lotan and Einat Minkov. See the paper: [Social world knowledge: Modeling and applications](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0283700) (PLOS ONE, 2023).
68
+
69
+ With this package you can:
70
+
71
+ - get the 100-dimensional embedding of a popular account, by user ID or username;
72
+ - find similar accounts, for one account, a group of accounts, or any vector;
73
+ - represent any user as the average embedding of the accounts they follow;
74
+ - predict personal traits (political leaning, gender, age, and more) from an embedding;
75
+ - serve all of the above over a small HTTP API, or use it from the command line.
76
+
77
+ ## Installation
78
+
79
+ ```bash
80
+ pip install socialvec
81
+ ```
82
+
83
+ The classifiers need TensorFlow and the HTTP API needs Flask. Both are optional extras:
84
+
85
+ ```bash
86
+ pip install "socialvec[classifier]" # adds TensorFlow
87
+ pip install "socialvec[api]" # adds Flask and flask-cors
88
+ pip install "socialvec[classifier,api]" # both
89
+ ```
90
+
91
+ SocialVec supports Python 3.10 to 3.13.
92
+
93
+ ## Quick start
94
+
95
+ ```python
96
+ from socialvec.socialvec import SocialVec
97
+
98
+ sv = SocialVec() # downloads the model on first use, see "Models and data"
99
+ sv.get_similar("jack", topn=5)
100
+ ```
101
+
102
+ ```
103
+ twitter_id similarity screen_name name description
104
+ 0 6385432 0.841613 dickc dick costolo ...
105
+ 1 989 0.831723 om OM ...
106
+ 2 5746452 0.827466 waltmossberg Walt Mossberg ...
107
+ 3 20536157 0.826462 Google Google ...
108
+ 4 6708952 0.819312 SteveCase Steve Case ...
109
+ ```
110
+
111
+ ## Usage
112
+
113
+ ### Embeddings
114
+
115
+ An account can be given as a user ID (an `int`, including numpy integers, or a string of digits) or as a username (case-insensitive):
116
+
117
+ ```python
118
+ sv[12] # the same 100-dimensional vector, three ways
119
+ sv["12"]
120
+ sv["jack"]
121
+
122
+ sv.get_userid("NASA") # '11348282'
123
+ sv.get_screen_name(813286) # 'BarackObama'
124
+ ```
125
+
126
+ ### Similar accounts
127
+
128
+ `get_similar` takes a user ID, a username, a vector, or a list of IDs and usernames (whose embeddings are averaged). It returns a pandas DataFrame with the `topn` most similar accounts and their metadata.
129
+
130
+ ```python
131
+ sv.get_similar(["Harvard", "MIT", "UCLA"], topn=5)
132
+ ```
133
+
134
+ ```
135
+ twitter_id similarity screen_name name
136
+ 0 5695032 0.867065 Yale Yale University
137
+ 1 5694822 0.861724 Princeton Princeton University
138
+ 2 248795646 0.850461 Columbia Columbia University
139
+ 3 14884486 0.845595 BrownUniversity Brown University
140
+ 4 33474655 0.840983 Cambridge_Uni Cambridge University
141
+ ```
142
+
143
+ ```python
144
+ sv.get_similar(["FCBarcelona", "ManUtd", "realmadrid"], topn=5)
145
+ ```
146
+
147
+ ```
148
+ twitter_id similarity screen_name name
149
+ 0 740336334 0.931517 GarethBale11 Gareth Bale
150
+ 1 344801362 0.917337 DavidLuiz_4 David Luiz
151
+ 2 140750163 0.915364 juanmata8 Juan Mata García
152
+ 3 112764971 0.913976 FCBarcelona_es FC Barcelona
153
+ 4 533085085 0.912526 M10 Mesut Özil
154
+ ```
155
+
156
+ ### Similarity between two accounts
157
+
158
+ ```python
159
+ sv.get_similarity("BarackObama", "HillaryClinton") # 0.755
160
+ sv.get_similarity("BarackObama", "realDonaldTrump") # 0.613
161
+ ```
162
+
163
+ Either argument can also be a vector.
164
+
165
+ ### Embedding any user
166
+
167
+ Most users aren't popular enough to have their own embedding. You can represent them by the average embedding of the accounts they follow. `get_average_embeddings` takes a list of user IDs and returns the average vector and the number of accounts it was based on. IDs that aren't in the model are skipped; if none are left, the vector is all zeros and the count is 0.
168
+
169
+ ```python
170
+ following = [12, 989, 783214] # e.g. the IDs a user follows, from the Twitter/X API
171
+ vector, count = sv.get_average_embeddings(following)
172
+ sv.get_similar(vector, topn=5)
173
+ ```
174
+
175
+ ### Classifiers
176
+
177
+ The classifiers predict personal traits from an embedding. They need the `classifier` extra (TensorFlow) and the `"2020c"` model they were trained on; calling `init_classifier()` on another model raises an error.
178
+
179
+ ```python
180
+ sv = SocialVec(model_name="2020c")
181
+ sv.init_classifier()
182
+
183
+ sv.classifier.predict("political", sv["JoeBiden"]) # ('Democrat', 0.989)
184
+ sv.classifier.predict("political", sv["realDonaldTrump"]) # ('Republican', 0.866)
185
+ ```
186
+
187
+ `predict` returns the predicted class and a confidence between 0 and 1, where 1 is high confidence and 0 is no confidence (which may be expected for accounts with no affiliation, e.g. a non-political account and the `political` classifier). To classify an ordinary user, pass the vector from `get_average_embeddings`.
188
+
189
+ | Classifier | Classes |
190
+ |---|---|
191
+ | `political` | Republican / Democrat |
192
+ | `gender` | Male / Female |
193
+ | `children` | Yes / No |
194
+ | `age` | Under25 / Over25 |
195
+ | `education` | Degree / HighSchool |
196
+ | `income` | High / Low |
197
+ | `ethnicity` | Caucasian / AfrAmerican |
198
+
199
+ `sv.classifier.classifier_names` lists them.
200
+
201
+ ### Errors
202
+
203
+ Lookups raise `socialvec.exceptions.InvalidUserIdError` (a `ValueError`) for values that aren't a user ID or username, and `socialvec.exceptions.UserNotFoundError` (a `KeyError`) for accounts that aren't in the loaded model. An unknown `model_name` raises `UnknownModelError` (a `ValueError`), and a failed download raises `SocialVecError`. All of them subclass `socialvec.exceptions.SocialVecError`.
204
+
205
+ ## Models and data
206
+
207
+ | `model_name` | Notes |
208
+ |---|---|
209
+ | `"2020"` | Default. |
210
+ | `"2020c"` | The model the classifiers were trained on. |
211
+ | `"2020_2022"` | |
212
+
213
+ On first use, SocialVec downloads the model (about 150 MB, or 300 MB for `"2020_2022"`) and the account metadata (about 30 MB). Later runs load them from disk.
214
+
215
+ - Files are kept in `~/.cache/socialvec`. To keep them somewhere else, set the `SOCIALVEC_HOME` environment variable to a folder.
216
+ - Files that versions up to 0.1.7.1 saved in the package folder or the working folder are still found and used.
217
+ - Every file is checked against a SHA-256 checksum before it is loaded.
218
+ - If the cache folder can't be written, the model is loaded into memory instead (and downloaded again next time).
219
+
220
+ ## Command line
221
+
222
+ ```bash
223
+ socialvec similar jack --topn 5
224
+ socialvec similarity BarackObama HillaryClinton
225
+ socialvec predict political JoeBiden # uses the 2020c model; needs the classifier extra
226
+ socialvec download --model 2020c # fetch a model ahead of time
227
+ ```
228
+
229
+ `socialvec --help` lists all options. Every command takes `--model` to choose the model.
230
+
231
+ ## HTTP API
232
+
233
+ SocialVec includes a small Flask API (install the `api` extra):
234
+
235
+ ```bash
236
+ SOCIALVEC_MODEL=2020c python -m socialvec.api
237
+ ```
238
+
239
+ It can also run under a WSGI server, e.g. `gunicorn "socialvec.api:create_app()"`. Importing `socialvec.api` does not load a model; `create_app()` does.
240
+
241
+ | Variable | Default | Meaning |
242
+ |---|---|---|
243
+ | `SOCIALVEC_MODEL` | `2020` | Model to load. `/predict` and `/predict_proba` need `2020c`; with other models they return 501. |
244
+ | `SOCIALVEC_CORS_ORIGINS` | `*` | Comma-separated origins allowed to call the API from a browser. |
245
+ | `SOCIALVEC_HOST` | `127.0.0.1` | Address to listen on. Use `0.0.0.0` to accept connections from other machines. |
246
+ | `PORT` | `8000` | Port to listen on. |
247
+
248
+ Endpoints:
249
+
250
+ - `GET /validate_userid?userid=`, `/validate_username?username=`, `/get_screen_name?userid=`, `/get_userid?username=`
251
+ - `GET /get_similar?input=&topn=` (`topn` is clamped to 1–100), `/get_embeddings?entity=`, `/get_similarity?entity1=&entity2=`
252
+ - `POST` with a JSON body: `/get_average_embeddings` `{"entity_list": [...]}`, `/predict` and `/predict_proba` `{"classifier_name": "political", "entity": "jack"}`
253
+
254
+ Errors are returned as `{"error": "..."}` with status 400 for bad input, 404 for accounts not in the model, and 500 (with no details) for unexpected errors, which are logged.
255
+
256
+ ## Citation
257
+
258
+ If you use SocialVec in your research, please cite:
259
+
260
+ ```bibtex
261
+ @article{lotan2023social,
262
+ title = {Social world knowledge: Modeling and applications},
263
+ author = {Lotan, Nir and Minkov, Einat},
264
+ journal = {PLOS ONE},
265
+ volume = {18},
266
+ number = {7},
267
+ pages = {e0283700},
268
+ year = {2023},
269
+ doi = {10.1371/journal.pone.0283700}
270
+ }
271
+ ```
272
+
273
+ ## License
274
+
275
+ MIT. See [LICENSE](https://github.com/nirlotan/SocialVec/blob/master/LICENSE). The changelog is in [HISTORY.rst](https://github.com/nirlotan/SocialVec/blob/master/HISTORY.rst).
@@ -0,0 +1,20 @@
1
+ socialvec/__init__.py,sha256=7WlH3O7yS8uBKs9Th9duhPQQgxpGKi7C5oefIVE_cCs,125
2
+ socialvec/_download.py,sha256=5ScGUX6OI8gXaxPeJI5oUmtwTDMUhHXG5F0c_MEcs9Y,4844
3
+ socialvec/api.py,sha256=qbEXXheZgKws5c4LJeFYsQ9nTrLZl7Jmt6lEz9KQ8k8,8311
4
+ socialvec/cli.py,sha256=IjWqdWuNTbjT-pPrtQvbS8LuIEz4b-NLWf6dywuLipA,2624
5
+ socialvec/config.yaml,sha256=RDscPseGIfXzsYp3znMsofIcbdUgbdXHn1pfAIWyOsA,1873
6
+ socialvec/exceptions.py,sha256=MjXirsjVzQz6eQKxhjs3RNEJofT-KRw3AAr0355DmC0,658
7
+ socialvec/socialvec.py,sha256=5RSd89mCZqMp-YIl10vAbge2SLGzng3EVppV5dufAno,15549
8
+ socialvec/models/SocialVec_age_2022.h5,sha256=Q9nmHTiSbN0kGCGlNE8nan0wmYbFBrmJnVJ4fMwR1M8,22576
9
+ socialvec/models/SocialVec_children_2022.h5,sha256=Og6sYI1PqEa6q_WyZkrMgouKOmhelFq0ypagOKGilkE,22576
10
+ socialvec/models/SocialVec_education_2022.h5,sha256=CmV3L6dVTMICAE5ebCY04D_iBecmksp2PyZv1Hltpy8,22576
11
+ socialvec/models/SocialVec_ethnicity_2022.h5,sha256=BfwOlLFiUgyZa4vaqfCKZstKvRnyXh8HRcA4jF8Jpn4,22576
12
+ socialvec/models/SocialVec_gender_2022.h5,sha256=v-d8tDQ5aJ3uhRFxTmv4wJdtNt4xH5hemiWGkY6iNEs,22576
13
+ socialvec/models/SocialVec_income_2022.h5,sha256=IbFbacYa-pNfyG_i_dhZF_5GlhC05Xp96tzESiq8qZw,22576
14
+ socialvec/models/SocialVec_political_2022.h5,sha256=a61UoZNQtvA2MXbMoAFmWwejh7MjpZ5tdNpUXcFfaLw,22576
15
+ socialvec-0.2.0.dist-info/licenses/LICENSE,sha256=pSpIOtPcWehxx0T4p-Y_1aukVbqwC7M6ElF5akGXl5o,1068
16
+ socialvec-0.2.0.dist-info/METADATA,sha256=Ck87Aa7QcG-z8imlriZGt3X-MlNI-UUCfLEt6fil9cQ,11283
17
+ socialvec-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
18
+ socialvec-0.2.0.dist-info/entry_points.txt,sha256=wo0aeBs8HWfqw-ecAOzOu3PVHG2-qfmA4A3u6M2LWO4,48
19
+ socialvec-0.2.0.dist-info/top_level.txt,sha256=s06aCPWn0lMnLKdU0vCbA7pQhD0DY69ScXnj7TYzJ7A,10
20
+ socialvec-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ socialvec = socialvec.cli:run
@@ -0,0 +1,22 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2022, Nir Lotan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
@@ -0,0 +1 @@
1
+ socialvec