fastcat 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fastcat/__init__.py ADDED
@@ -0,0 +1,12 @@
1
+ #!/usr/bin/env python
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ from fastcat.interface import FastCat
6
+
7
+ try:
8
+ __version__ = version("fastcat")
9
+ except PackageNotFoundError: # running straight from a source checkout
10
+ __version__ = "0.0.0.dev0"
11
+
12
+ __all__ = ["FastCat", "__version__"]
fastcat/interface.py ADDED
@@ -0,0 +1,263 @@
1
+ #!/usr/bin/env python
2
+
3
+ import os
4
+ import sys
5
+ import re
6
+ import bz2
7
+ from fastcat.utils import normalize_language, print_progress_bar, get_wikipedia_mapping
8
+ from urllib import request, parse
9
+ import redis
10
+ import fastcat.store as store
11
+ import fastcat.lang as languages
12
+
13
+
14
+ try:
15
+ p = __file__
16
+ except NameError:
17
+ p = sys.argv[0]
18
+
19
+
20
+ data_location = os.path.join(os.path.dirname(os.path.realpath(p)), 'data')
21
+ if not os.path.isdir(data_location):
22
+ os.makedirs(data_location)
23
+ settings_location = os.path.join(os.path.dirname(os.path.realpath(p)), 'settings')
24
+ if not os.path.isdir(settings_location):
25
+ os.makedirs(settings_location)
26
+
27
+ skos_file_pattern = os.path.join(os.path.dirname(os.path.realpath(p)), 'data', 'skos-%lang%.nt.bz2')
28
+
29
+ # Where to look for Redis unless the caller says otherwise. The environment
30
+ # variables make a containerised Redis (see docker-compose.yml) usable without
31
+ # passing connection arguments around.
32
+ DEFAULT_REDIS_HOST = os.environ.get('FASTCAT_REDIS_HOST', 'localhost')
33
+ DEFAULT_REDIS_PORT = int(os.environ.get('FASTCAT_REDIS_PORT', 6379))
34
+
35
+ ntriple_pattern = re.compile(r'^<(.+)> <(.+)> <(.+)> \.\n$')
36
+ ntriple_pattern_wide = re.compile(r'^<(.+)> <(.+)> <(.+)> <(.+)> \.\n$')
37
+
38
+
39
+ class FastCatBase(object):
40
+
41
+ def _download(self, language, verbose):
42
+ if verbose:
43
+ print("Downloading Wikipedia SKOS file from DBpedia")
44
+
45
+ normalized_language = normalize_language(language)
46
+ wikipedia_mapping = get_wikipedia_mapping(normalized_language)
47
+
48
+ if normalized_language == languages.available_languages['English'].id:
49
+ url = 'http://downloads.dbpedia.org/current/core/skos_categories_en.ttl.bz2'
50
+ else:
51
+ url = 'http://downloads.dbpedia.org/current/core-i18n/{}/skos_categories_{}.tql.bz2'.format(
52
+ wikipedia_mapping, wikipedia_mapping)
53
+
54
+ skos_file = skos_file_pattern.replace('%lang%', normalized_language)
55
+
56
+ if verbose:
57
+ print('-- request.urlretrieve for file {}'.format(skos_file))
58
+
59
+ request.urlretrieve(url, filename=skos_file)
60
+
61
+ if verbose:
62
+ print("Finished downloading {} file".format(skos_file))
63
+
64
+ def _name(self, url_pattern, language):
65
+ if language == languages.available_languages['English'].id:
66
+ m = re.search("^http://dbpedia.org/resource/Category:(.+)$", url_pattern)
67
+ elif language == languages.available_languages['Czech'].id:
68
+ m = re.search("^http://cs.dbpedia.org/resource/Kategorie:(.+)$", url_pattern)
69
+ elif language == languages.available_languages['Estonian'].id:
70
+ m = re.search("^http://et.dbpedia.org/resource/Kategooria:(.+)$", url_pattern)
71
+ elif language == languages.available_languages['German'].id:
72
+ m = re.search("^http://de.dbpedia.org/resource/Kategorie:(.+)$", url_pattern)
73
+ elif language == languages.available_languages['Japanese'].id:
74
+ m = re.search("^http://ja.dbpedia.org/resource/Category:(.+)$", url_pattern)
75
+ elif language == languages.available_languages['Polish'].id:
76
+ m = re.search("^http://pl.dbpedia.org/resource/Kategoria:(.+)$", url_pattern)
77
+ elif language == languages.available_languages['Portuguese'].id:
78
+ m = re.search("^http://pt.dbpedia.org/resource/Categoria:(.+)$", url_pattern)
79
+ elif language == languages.available_languages['Russian'].id:
80
+ m = re.search("^http://ru.dbpedia.org/resource/Категория:(.+)$", url_pattern)
81
+ elif language == languages.available_languages['Ukrainian'].id:
82
+ m = re.search("^http://uk.dbpedia.org/resource/Категорія:(.+)$", url_pattern)
83
+ else:
84
+ raise NotImplementedError
85
+ return parse.unquote(m.group(1).replace("_", " "))
86
+
87
+
88
+ class FastCat(FastCatBase):
89
+
90
+ def __init__(self, db=None, language=None, **kwargs):
91
+ """Creates a new FastCast object, an interface to Wikipedia categories.
92
+
93
+ The __init__ method creates the FastCat object, which acts as the main and only
94
+ interface in communicating with the Redis server (where the Wikipedia categories are
95
+ first loaded (stored) and then retrieved on demand). Default settings mean that you're connecting
96
+ to a Redis instance on the localhost and 6379 port, and your current language of categories is English.
97
+ It's possible to pass your own Redis client object into the 'db' arg, or,
98
+ alternatively, custom args to the Redis client __init__ method.
99
+
100
+ Note:
101
+ No need to pass any extra arguments if you don't understand what you're doing
102
+
103
+ Args:
104
+ db (:obj:`Redis`, optional): Custom Redis client.
105
+ language (:obj:`str`, optional): Choose the default language.
106
+ kwargs (:obj:`dict`, optional): Any arguments, which you wish to pass to the Redis client.
107
+
108
+ """
109
+
110
+ super(FastCatBase, self).__init__()
111
+ # Load most recent language-redis mapping
112
+ store.load_settings()
113
+
114
+ # Anything not set here is left at the redis client's own default
115
+ options = {'host': DEFAULT_REDIS_HOST, 'port': DEFAULT_REDIS_PORT}
116
+ options.update(kwargs)
117
+
118
+ # Remembered so that every later connection (a different language means
119
+ # a different redis db) reaches the same server
120
+ self._options = options
121
+
122
+ # Initialize redis client object
123
+ if db is None:
124
+
125
+ if language is None:
126
+
127
+ # Check if language-redis mapping is ok
128
+ assert 'en' in store.languages
129
+
130
+ # Initialize connection for English dataset
131
+ db = redis.Redis(**options) # default is db=0
132
+ else:
133
+
134
+ # Initialize connection for any other language dataset
135
+ normalized_language = normalize_language(language)
136
+
137
+ try:
138
+ slot = store.get_slot(normalized_language)
139
+ except ValueError:
140
+ slot = store.save_settings(normalized_language)
141
+
142
+ db = redis.Redis(db=slot, **options)
143
+
144
+ # There must be always only one redis client
145
+ self.db = db
146
+
147
+ def switch_language(self, language):
148
+ """Switch language on an existing FastCat object."""
149
+ try:
150
+
151
+ slot = store.get_slot(language)
152
+ self.db = redis.Redis(db=slot, **self._options)
153
+ except ValueError:
154
+
155
+ slot = store.save_settings(language)
156
+ self.db = redis.Redis(db=slot, **self._options)
157
+ self.load(language)
158
+
159
+ def get_current_language(self):
160
+ """Get current language."""
161
+ return store.get_language(slot=self.db.connection_pool.connection_kwargs['db'])
162
+
163
+ @staticmethod
164
+ def get_supported_languages():
165
+ """Get list of supported languages."""
166
+ return languages.available_languages.keys()
167
+
168
+ def broader(self, cat):
169
+ """Pass in a Wikipedia category and get back a list of broader Wikipedia categories."""
170
+ return [s.decode('utf-8') for s in self.db.smembers("b:%s" % cat)]
171
+
172
+ def narrower(self, cat):
173
+ """Pass in a Wikipedia category and get back a list of narrower Wikipedia categories."""
174
+ return [s.decode('utf-8') for s in self.db.smembers("n:%s" % cat)]
175
+
176
+ def _is_loaded(self, language, verbose=False):
177
+ # TODO: process depending on language
178
+ if self.db.get("loaded-skos"):
179
+ if verbose:
180
+ print('Wikipedia SKOS for {} language is already loaded to Redis!'.format(language))
181
+ return True
182
+ else:
183
+ return False
184
+
185
+ def load(self, language=None, verbose=False, progress_bar=True):
186
+ """Fill Redis with Wikipedia SKOS data."""
187
+ if language is None:
188
+ language = self.get_current_language().alpha_2.lower()
189
+
190
+ if self._is_loaded(language, verbose):
191
+ print('Loading aborted (language already exists)')
192
+ return
193
+
194
+ skos_file = skos_file_pattern.replace('%lang%', language)
195
+
196
+ if not os.path.isfile(skos_file):
197
+ if verbose:
198
+ print('Downloading SKOS .gzip file for langauge: {}'.format(language))
199
+ self._download(language, verbose)
200
+
201
+ if verbose:
202
+ print("Loading {} file".format(skos_file))
203
+
204
+ if verbose:
205
+ print('Unpacking DBpedia .GZ file... this may take some time.')
206
+
207
+ uncompressed = bz2.BZ2File(skos_file).readlines()
208
+
209
+ l = len(uncompressed)
210
+
211
+ if verbose:
212
+ print('Starting process of adding DBpedia data to Redis instance')
213
+
214
+ for i, line in enumerate(uncompressed):
215
+
216
+ if progress_bar:
217
+ print_progress_bar(i, l, prefix='Progress:', suffix='Complete', length=50)
218
+
219
+ if language == languages.available_languages['English'].id:
220
+ m = ntriple_pattern.match(line.decode('utf-8'))
221
+ else:
222
+ # Non-english (i18l) SKOS files have different format
223
+ m = ntriple_pattern_wide.match(line.decode('utf-8'))
224
+
225
+ if not m:
226
+ if verbose > 2:
227
+ print('ntripple pattern failed to match')
228
+ continue
229
+
230
+ groups = m.groups()
231
+
232
+ if len(groups) == 4:
233
+ s, p, o, meta = m.groups()
234
+ elif len(groups) == 3:
235
+ s, p, o = m.groups()
236
+ else:
237
+ raise ValueError
238
+
239
+ if p != "http://www.w3.org/2004/02/skos/core#broader":
240
+ if verbose > 2:
241
+ print('p group is not "broader" - {}'.format(p))
242
+ continue
243
+
244
+ narrower = self._name(s, language)
245
+ broader = self._name(o, language)
246
+
247
+ try:
248
+
249
+ if verbose > 1:
250
+ print('Narrower: {}, broader: {}'.format(narrower, broader))
251
+
252
+ self.db.sadd("b:%s" % narrower, broader)
253
+ self.db.sadd("n:%s" % broader, narrower)
254
+
255
+ except UnicodeEncodeError as uee:
256
+
257
+ print('Narrower: {}, broader: {}'.format(narrower.encode("utf-8"), broader.encode("utf-8")))
258
+ raise uee
259
+
260
+ if verbose > 1:
261
+ print("Added %s -> %s" % (broader, narrower))
262
+
263
+ self.db.set("loaded-skos", "1")
fastcat/lang.py ADDED
@@ -0,0 +1,22 @@
1
+ from collections import namedtuple
2
+
3
+ Language = namedtuple('Language', ['id', 'locales', 'alternate', 'wikipedia_mapping'])
4
+
5
+ available_languages = {'Czech': Language(id='cs', locales=['cs-cs'],
6
+ alternate='cze', wikipedia_mapping='cs'),
7
+ 'English': Language(id='en', locales=['en-gb', 'en-us', 'en-ca', 'en-nz'],
8
+ alternate='eng', wikipedia_mapping='en'),
9
+ 'Estonian': Language(id='et', locales=['et-et'],
10
+ alternate='est', wikipedia_mapping='et'),
11
+ 'German': Language(id='de', locales=['de-de'],
12
+ alternate='ger', wikipedia_mapping='de'),
13
+ 'Japanese': Language(id='ja', locales=['ja-jp'],
14
+ alternate='jpn', wikipedia_mapping='ja'),
15
+ 'Polish': Language(id='pl', locales=['pl-pl'],
16
+ alternate='pol', wikipedia_mapping='pl'),
17
+ 'Portuguese': Language(id='pt', locales=['pt-pt', 'pt-br'],
18
+ alternate='por', wikipedia_mapping='pt'),
19
+ 'Russian': Language(id='ru', locales=['ru-ru'],
20
+ alternate='rus', wikipedia_mapping='ru'),
21
+ 'Ukrainian': Language(id='ua', locales=['ua-ua'],
22
+ alternate='ukr', wikipedia_mapping='uk')}
fastcat/store.py ADDED
@@ -0,0 +1,61 @@
1
+ import pickle
2
+ import os
3
+ import sys
4
+ import pycountry
5
+
6
+
7
+ try:
8
+ p = __file__
9
+ except NameError:
10
+ p = sys.argv[0]
11
+
12
+ settings_filename = os.path.join(os.path.dirname(os.path.realpath(p)), 'settings', 'redis_ids.pickle')
13
+ languages = dict()
14
+
15
+
16
+ # When creating new FastCat instance, try to unpickle configuration object
17
+ def load_settings():
18
+ # Always use the global 'languages' singleton dictionary
19
+ global languages
20
+
21
+ try:
22
+ languages = pickle.load(open(settings_filename, "rb"))
23
+ except FileNotFoundError:
24
+ # Usually when file does not exist, i.e. FastCat ran for the first time
25
+ languages = {'en': 0}
26
+ except Exception as exc:
27
+ print('Unknown exception thrown while unpickling {} file'.format(settings_filename))
28
+ # TODO: investigate why PyPy fails and if there is some workaround
29
+ raise exc
30
+
31
+
32
+ def save_settings(new_key):
33
+ new_value = _get_next_slot()
34
+
35
+ print('Assigning redis id {} to language {}'.format(new_value, new_key))
36
+
37
+ languages[new_key] = new_value
38
+ pickle.dump(languages, open(settings_filename, "wb"))
39
+ return new_value
40
+
41
+
42
+ def _get_next_slot():
43
+ return max(languages.values()) + 1
44
+
45
+
46
+ def get_language(slot):
47
+ for key, value in languages.items():
48
+ if value == slot:
49
+ # normally this line below should work fine
50
+ result = pycountry.languages.get(alpha_2=key)
51
+ if result is None:
52
+ # if not, below is last-chance workaround
53
+ # due to some bugs in pycountry package
54
+ result = [c for c in list(pycountry.countries) if c.alpha_2.lower() == key.lower()][0]
55
+ return result
56
+
57
+
58
+ def get_slot(language):
59
+ if language not in languages:
60
+ raise ValueError
61
+ return languages[language]
fastcat/utils.py ADDED
@@ -0,0 +1,46 @@
1
+ import fastcat.lang as languages
2
+
3
+
4
+ def normalize_language(lang):
5
+ language_normalized = lang.lower().replace('_', '-')
6
+
7
+ for l in languages.available_languages.values():
8
+ if l.id == language_normalized:
9
+ # ISO 639-1
10
+ return l.id
11
+ elif language_normalized in l.locales:
12
+ # id and locale pair
13
+ return l.id
14
+ elif language_normalized == l.alternate:
15
+ # ISO 639-2
16
+ return l.id
17
+
18
+ return None
19
+
20
+
21
+ def get_wikipedia_mapping(normalized_lang):
22
+ for l in languages.available_languages.values():
23
+ if l.id == normalized_lang:
24
+ return l.wikipedia_mapping
25
+
26
+
27
+ # Print iterations progress
28
+ def print_progress_bar(iteration, total, prefix='', suffix='', decimals=1, length=100, fill='#'):
29
+ """
30
+ Call in a loop to create terminal progress bar
31
+ @params:
32
+ iteration - Required : current iteration (Int)
33
+ total - Required : total iterations (Int)
34
+ prefix - Optional : prefix string (Str)
35
+ suffix - Optional : suffix string (Str)
36
+ decimals - Optional : positive number of decimals in percent complete (Int)
37
+ length - Optional : character length of bar (Int)
38
+ fill - Optional : bar fill character (Str)
39
+ """
40
+ percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
41
+ filled_length = int(length * iteration // total)
42
+ bar = fill * filled_length + '-' * (length - filled_length)
43
+ print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end='\r')
44
+ # Print New Line on Complete
45
+ if iteration == total:
46
+ print()
@@ -0,0 +1,229 @@
1
+ Metadata-Version: 2.4
2
+ Name: fastcat
3
+ Version: 0.2.1
4
+ Summary: Navigate Wikipedia categories quickly in a local redis instance
5
+ Author-email: Ed Summers <ehs@pobox.com>
6
+ Maintainer-email: Oskar Jarczyk <oskar.jarczyk@gmail.com>
7
+ License-Expression: CC-BY-SA-3.0
8
+ Project-URL: Homepage, https://github.com/oskar-j/fastcat
9
+ Project-URL: Repository, https://github.com/oskar-j/fastcat
10
+ Project-URL: Changelog, https://github.com/oskar-j/fastcat/blob/master/CHANGELOG.md
11
+ Project-URL: Issues, https://github.com/oskar-j/fastcat/issues
12
+ Keywords: Wikipedia,categories,wiki-api,knowledge engineering
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Intended Audience :: Science/Research
15
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Programming Language :: Python :: 3.14
22
+ Requires-Python: >=3.10
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE.txt
25
+ Requires-Dist: redis<8.0.0,>=5.0.0
26
+ Requires-Dist: pycountry<26.0.0,>=24.6.1
27
+ Provides-Extra: dev
28
+ Requires-Dist: pytest<9.0.0,>=8.0.0; extra == "dev"
29
+ Dynamic: license-file
30
+
31
+ fastcat
32
+ =======
33
+
34
+ [![Tests](https://github.com/oskar-j/fastcat/actions/workflows/tests.yml/badge.svg?branch=master)](https://github.com/oskar-j/fastcat/actions/workflows/tests.yml)
35
+ [![Publish](https://github.com/oskar-j/fastcat/actions/workflows/publish.yml/badge.svg?branch=master)](https://github.com/oskar-j/fastcat/actions/workflows/publish.yml)
36
+ [![PyPI](https://img.shields.io/pypi/v/fastcat)](https://pypi.org/project/fastcat/)
37
+ [![Python Versions](https://img.shields.io/pypi/pyversions/fastcat)](https://pypi.org/project/fastcat/)
38
+ [![Downloads](https://static.pepy.tech/badge/fastcat)](https://pepy.tech/project/fastcat)
39
+ [![License](https://img.shields.io/badge/license-CC%20BY--SA%203.0-blue)](http://creativecommons.org/licenses/by-sa/3.0/)
40
+ [![Pending Pull-Requests](https://img.shields.io/github/issues-pr/oskar-j/fastcat)](https://github.com/oskar-j/fastcat/pulls)
41
+ [![Github Issues](https://img.shields.io/github/issues/oskar-j/fastcat)](https://github.com/oskar-j/fastcat/issues)
42
+ [![Commits Since Release](https://img.shields.io/github/commits-since/oskar-j/fastcat/latest)](https://github.com/oskar-j/fastcat/releases)
43
+
44
+ Fastcat is a little Python library for quickly looking up broader/narrower
45
+ relations in Wikipedia categories locally. The idea is that fastcat can be
46
+ useful in situations where you need to rapidly lookup category relations,
47
+ but don't want to hammer on the [Wikipedia
48
+ API](http://en.wikipedia.org/w/api.php). Fastcat relies on Redis and the
49
+ [SKOS file](http://downloads.dbpedia.org/current/en/skos_categories_en.nt.bz2) that DBpedia makes available basing on
50
+ the Wikipedia [MySQL dumps](http://dumps.wikimedia.org/enwiki/latest/).
51
+
52
+ ![fastcat logo](https://datageek.pl/assets/img/projects/fast_cat.png)
53
+
54
+ Attribution
55
+ -----
56
+
57
+ This software is a fork of [fastcat](https://github.com/edsu/fastcat) tool created by [Ed Summers](https://github.com/edsu).
58
+ Some changes were made under the *Creative Commons Attribution-ShareAlike 3.0* license, and they are described in commit
59
+ messages. Major changes are porting the code to Python 3 as well as adding support for more than one language.
60
+
61
+ Usage
62
+ -----
63
+
64
+ #### Basic usage
65
+
66
+ The first time you import fastcat you'll need to populate your Redis database
67
+ with the category data from DBpedia. To do that instantiate a FastCat object
68
+ and call the `load` method. After that you can use it to do lookups.
69
+
70
+ ```python
71
+ >>> import fastcat
72
+ >>> f = fastcat.FastCat()
73
+ >>> f.load() # brew a pot of coffee while the data is downloaded and loaded into redis
74
+ ...
75
+ >>> print(f.broader("Computer programming"))
76
+ ['Software engineering', 'Computing']
77
+ >>> print(f.narrower("Computer programming"))
78
+ ['Programming idioms', 'Programming languages', 'Concurrent computing', 'Source code', 'Refactoring', 'Data structures', 'Programming games', 'Computer programmers', 'Version control', 'Anti-patterns', 'Programming constructs', 'Algorithms', 'Web Services tools', 'Programming paradigms', 'Software optimization', 'Debugging', 'Computer programming tools', 'Computer libraries', 'Programming contests', 'Archive networks', 'Self-hosting software', 'Educational abstract machines', 'Software design patterns', 'Computer arithmetic']
79
+ ```
80
+
81
+ #### Non-english categories
82
+
83
+ Just fill-in the `language` argument in the `FastCat()` constructor with a language code listed below.
84
+
85
+ ```python
86
+ >>> import fastcat
87
+ >>> f = fastcat.FastCat(language='de')
88
+ >>> f.load() # brew a pot of coffee while the data is downloaded and loaded into redis
89
+ ...
90
+ >>> print(f.broader("Berlin"))
91
+ ['Europa nach Ort', 'Deutschland nach Gemeinde', 'Deutschland nach Bundesland']
92
+ >>> print(f.narrower("Berlin"))
93
+ ['Umwelt- und Naturschutz (Berlin)', 'Veranstaltung (Berlin)', 'Stadtplanung (Berlin)', 'Verwaltung (Berlin)', 'Urbaner Freiraum in Berlin als Thema']
94
+ ```
95
+
96
+ ##### Currently supported languages (and their codes)
97
+
98
+ 1. English (`en`)
99
+ 2. Estonian (`et`)
100
+ 3. German (`de`)
101
+ 4. Japanese (`ja`)
102
+ 5. Polish (`pl`)
103
+ 6. Portuguese (`pt`)
104
+ 7. Russian (`ru`)
105
+ 8. Ukrainian (`ua`)
106
+ 9. Czech (`cs`)
107
+
108
+ Install
109
+ -------
110
+
111
+ ### Redis installation
112
+
113
+ You first need to setup Redis server on your machine as follows.
114
+
115
+ **On Mac:**
116
+
117
+ ```
118
+ $ brew install redis
119
+ ```
120
+
121
+ **On Linux:**
122
+
123
+ ```
124
+ $ sudo apt-get install redis-server
125
+ ```
126
+
127
+ **On Windows:**
128
+
129
+ Please refer to instruction on installing [Vagrant Redis](https://github.com/ServiceStack/redis-windows). You will
130
+ need an Ubuntu installation on your Windows, more information can be found
131
+ here: [Install your Linux Distribution of Choice](https://docs.microsoft.com/pl-pl/windows/wsl/install-win10)
132
+
133
+ **With Docker (any platform):**
134
+
135
+ If you would rather not install Redis at all, the bundled compose file spins one
136
+ up on `localhost:6379`, with the loaded categories kept in a named volume so
137
+ they survive a restart:
138
+
139
+ ```
140
+ $ docker compose up -d redis
141
+ ```
142
+
143
+ The same file also defines a `fastcat` container with the package and its dev
144
+ dependencies installed, which is handy for running the suite in a clean
145
+ environment:
146
+
147
+ ```
148
+ $ docker compose run --rm fastcat pytest
149
+ ```
150
+
151
+ Inside a container Redis is not on localhost, so fastcat reads the
152
+ `FASTCAT_REDIS_HOST` and `FASTCAT_REDIS_PORT` environment variables (already set
153
+ for the `fastcat` service) to find it.
154
+
155
+ ### Installing the module
156
+
157
+ If you are ready, installing Fastcat is pretty straightforward:
158
+
159
+ ```
160
+ $ pip install fastcat
161
+ ```
162
+
163
+ Or if you wish to get the newest dev code:
164
+
165
+ ```
166
+ $ pip install git+https://github.com/oskar-j/fastcat.git
167
+ ```
168
+
169
+ That's it!
170
+
171
+ ### Contributing to the project
172
+
173
+ #### Guidelines
174
+
175
+ See [CONTRIBUTING.md](https://github.com/oskar-j/fastcat/blob/master/CONTRIBUTING.md) for more details
176
+
177
+ #### Running unit tests
178
+
179
+ Install the package with its dev dependencies and run pytest:
180
+
181
+ ```
182
+ $ pip install -e '.[dev]'
183
+ $ pytest
184
+ ```
185
+
186
+ That runs the fast, offline tests. The end-to-end tests need a Redis server and
187
+ download a SKOS dump per language from DBpedia, so they are opt-in:
188
+
189
+ ```
190
+ $ docker compose up -d redis # or your own local Redis
191
+ $ pytest --run-integration
192
+ ```
193
+
194
+ Q&A
195
+ -------
196
+
197
+ #### How much is is tested?
198
+
199
+ It's still in early stage of development, please share some feedback with me (under the [ticket #7](https://github.com/oskar-j/fastcat/issues/7)).
200
+
201
+ #### What are biggest drawbacks of Fastcat?
202
+
203
+ DBpedia SKOS file is prone to constant change, which means that *downloading Wikipedia data* from web can stop working
204
+ in some distant future. Moreover, due to the [infrastructure of Redis](http://www.mikeperham.com/2015/09/24/storing-data-with-redis/), you can have a maximum number of 16 languages (1 slot for a language). Last but not least, it takes around `40 MB` of your web transfer (size depends on the selected language) to download a single SKOS file.
205
+
206
+ #### Which Python versions are supported?
207
+
208
+ Python `3.10` and above (tested on GitHub Actions against `3.10` through `3.14`).
209
+ Releases up to `0.1.2` supported Python `3.5`+; if you are stuck on an older
210
+ interpreter, pin `fastcat==0.1.2`.
211
+
212
+ #### Which languages are supported?
213
+
214
+ There are two ways to check the list of available languages.
215
+
216
+ First, is a manual inspection of the [lang.py](https://github.com/oskar-j/fastcat/blob/master/src/fastcat/lang.py) file.
217
+
218
+ Second way is to call the `get_supported_languages()` method on the `FastCat` object.
219
+
220
+ #### What's coming next?
221
+
222
+ Support for the rest of european languages. Exporting n-size tree of categories to a CSV or GraphML file.
223
+ Moving the downloaded dumps and the language mapping out of the package directory into a proper user cache
224
+ directory.
225
+
226
+ License
227
+ -------
228
+
229
+ [Creative Commons Attribution-ShareAlike 3.0](http://creativecommons.org/licenses/by-sa/3.0/)
@@ -0,0 +1,10 @@
1
+ fastcat/__init__.py,sha256=KleBStt21spxos89ukjCFq-Clf9VCOaPfKewVg9ZRUE,307
2
+ fastcat/interface.py,sha256=shVoC3tV-bLHZXa1SU_B3CtuOZAiBbzJDurHCu_YIxo,10229
3
+ fastcat/lang.py,sha256=3wp7EAT4RBjDYGTUF2HvhnddkiwHdL1uql3wVOlU_00,1570
4
+ fastcat/store.py,sha256=E_4mRZZGApbernmnCbmXRqS_rZYJSYfSOoXyQOGHR5E,1761
5
+ fastcat/utils.py,sha256=p7vBVNTCq7l9wlULdzMi8fTU-BsSdbnVkIajOsBSMI8,1652
6
+ fastcat-0.2.1.dist-info/licenses/LICENSE.txt,sha256=GjwVrAU8d_AI1F42DaVqOJQ_PYTeXjhcVDuC4iu3yyM,18547
7
+ fastcat-0.2.1.dist-info/METADATA,sha256=qP8j8ctA02u5ZSIBwNisabI0Oy6UFu8wk35_5ndgld4,8881
8
+ fastcat-0.2.1.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
9
+ fastcat-0.2.1.dist-info/top_level.txt,sha256=VNPSnDKv8XVlMwL9E1hwejtkQAvAOQepeRRuzjfIqh0,8
10
+ fastcat-0.2.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (83.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,73 @@
1
+ Creative Commons
2
+ Attribution 3.0 Unported
3
+
4
+ CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM ITS USE.
5
+
6
+ License
7
+
8
+ THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
9
+
10
+ BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS.
11
+
12
+ 1. Definitions
13
+
14
+ "Adaptation" means a work based upon the Work, or upon the Work and other pre-existing works, such as a translation, adaptation, derivative work, arrangement of music or other alterations of a literary or artistic work, or phonogram or performance and includes cinematographic adaptations or any other form in which the Work may be recast, transformed, or adapted including in any form recognizably derived from the original, except that a work that constitutes a Collection will not be considered an Adaptation for the purpose of this License. For the avoidance of doubt, where the Work is a musical work, performance or phonogram, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered an Adaptation for the purpose of this License.
15
+ "Collection" means a collection of literary or artistic works, such as encyclopedias and anthologies, or performances, phonograms or broadcasts, or other works or subject matter other than works listed in Section 1(f) below, which, by reason of the selection and arrangement of their contents, constitute intellectual creations, in which the Work is included in its entirety in unmodified form along with one or more other contributions, each constituting separate and independent works in themselves, which together are assembled into a collective whole. A work that constitutes a Collection will not be considered an Adaptation (as defined above) for the purposes of this License.
16
+ "Distribute" means to make available to the public the original and copies of the Work or Adaptation, as appropriate, through sale or other transfer of ownership.
17
+ "Licensor" means the individual, individuals, entity or entities that offer(s) the Work under the terms of this License.
18
+ "Original Author" means, in the case of a literary or artistic work, the individual, individuals, entity or entities who created the Work or if no individual or entity can be identified, the publisher; and in addition (i) in the case of a performance the actors, singers, musicians, dancers, and other persons who act, sing, deliver, declaim, play in, interpret or otherwise perform literary or artistic works or expressions of folklore; (ii) in the case of a phonogram the producer being the person or legal entity who first fixes the sounds of a performance or other sounds; and, (iii) in the case of broadcasts, the organization that transmits the broadcast.
19
+ "Work" means the literary and/or artistic work offered under the terms of this License including without limitation any production in the literary, scientific and artistic domain, whatever may be the mode or form of its expression including digital form, such as a book, pamphlet and other writing; a lecture, address, sermon or other work of the same nature; a dramatic or dramatico-musical work; a choreographic work or entertainment in dumb show; a musical composition with or without words; a cinematographic work to which are assimilated works expressed by a process analogous to cinematography; a work of drawing, painting, architecture, sculpture, engraving or lithography; a photographic work to which are assimilated works expressed by a process analogous to photography; a work of applied art; an illustration, map, plan, sketch or three-dimensional work relative to geography, topography, architecture or science; a performance; a broadcast; a phonogram; a compilation of data to the extent it is protected as a copyrightable work; or a work performed by a variety or circus performer to the extent it is not otherwise considered a literary or artistic work.
20
+ "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation.
21
+ "Publicly Perform" means to perform public recitations of the Work and to communicate to the public those public recitations, by any means or process, including by wire or wireless means or public digital performances; to make available to the public Works in such a way that members of the public may access these Works from a place and at a place individually chosen by them; to perform the Work to the public by any means or process and the communication to the public of the performances of the Work, including by public digital performance; to broadcast and rebroadcast the Work by any means including signs, sounds or images.
22
+ "Reproduce" means to make copies of the Work by any means including without limitation by sound or visual recordings and the right of fixation and reproducing fixations of the Work, including storage of a protected performance or phonogram in digital form or other electronic medium.
23
+
24
+ 2. Fair Dealing Rights. Nothing in this License is intended to reduce, limit, or restrict any uses free from copyright or rights arising from limitations or exceptions that are provided for in connection with the copyright protection under copyright law or other applicable laws.
25
+
26
+ 3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below:
27
+
28
+ to Reproduce the Work, to incorporate the Work into one or more Collections, and to Reproduce the Work as incorporated in the Collections;
29
+ to create and Reproduce Adaptations provided that any such Adaptation, including any translation in any medium, takes reasonable steps to clearly label, demarcate or otherwise identify that changes were made to the original Work. For example, a translation could be marked "The original work was translated from English to Spanish," or a modification could indicate "The original work has been modified.";
30
+ to Distribute and Publicly Perform the Work including as incorporated in Collections; and,
31
+ to Distribute and Publicly Perform Adaptations.
32
+
33
+ For the avoidance of doubt:
34
+ Non-waivable Compulsory License Schemes. In those jurisdictions in which the right to collect royalties through any statutory or compulsory licensing scheme cannot be waived, the Licensor reserves the exclusive right to collect such royalties for any exercise by You of the rights granted under this License;
35
+ Waivable Compulsory License Schemes. In those jurisdictions in which the right to collect royalties through any statutory or compulsory licensing scheme can be waived, the Licensor waives the exclusive right to collect such royalties for any exercise by You of the rights granted under this License; and,
36
+ Voluntary License Schemes. The Licensor waives the right to collect royalties, whether individually or, in the event that the Licensor is a member of a collecting society that administers voluntary licensing schemes, via that society, from any exercise by You of the rights granted under this License.
37
+
38
+ The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. Subject to Section 8(f), all rights not expressly granted by Licensor are hereby reserved.
39
+
40
+ 4. Restrictions. The license granted in Section 3 above is expressly made subject to and limited by the following restrictions:
41
+
42
+ You may Distribute or Publicly Perform the Work only under the terms of this License. You must include a copy of, or the Uniform Resource Identifier (URI) for, this License with every copy of the Work You Distribute or Publicly Perform. You may not offer or impose any terms on the Work that restrict the terms of this License or the ability of the recipient of the Work to exercise the rights granted to that recipient under the terms of the License. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties with every copy of the Work You Distribute or Publicly Perform. When You Distribute or Publicly Perform the Work, You may not impose any effective technological measures on the Work that restrict the ability of a recipient of the Work from You to exercise the rights granted to that recipient under the terms of the License. This Section 4(a) applies to the Work as incorporated in a Collection, but this does not require the Collection apart from the Work itself to be made subject to the terms of this License. If You create a Collection, upon notice from any Licensor You must, to the extent practicable, remove from the Collection any credit as required by Section 4(b), as requested. If You create an Adaptation, upon notice from any Licensor You must, to the extent practicable, remove from the Adaptation any credit as required by Section 4(b), as requested.
43
+ If You Distribute, or Publicly Perform the Work or any Adaptations or Collections, You must, unless a request has been made pursuant to Section 4(a), keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or if the Original Author and/or Licensor designate another party or parties (e.g., a sponsor institute, publishing entity, journal) for attribution ("Attribution Parties") in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; (ii) the title of the Work if supplied; (iii) to the extent reasonably practicable, the URI, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and (iv) , consistent with Section 3(b), in the case of an Adaptation, a credit identifying the use of the Work in the Adaptation (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). The credit required by this Section 4 (b) may be implemented in any reasonable manner; provided, however, that in the case of a Adaptation or Collection, at a minimum such credit will appear, if a credit for all contributing authors of the Adaptation or Collection appears, then as part of these credits and in a manner at least as prominent as the credits for the other contributing authors. For the avoidance of doubt, You may only use the credit required by this Section for the purpose of attribution in the manner set out above and, by exercising Your rights under this License, You may not implicitly or explicitly assert or imply any connection with, sponsorship or endorsement by the Original Author, Licensor and/or Attribution Parties, as appropriate, of You or Your use of the Work, without the separate, express prior written permission of the Original Author, Licensor and/or Attribution Parties.
44
+ Except as otherwise agreed in writing by the Licensor or as may be otherwise permitted by applicable law, if You Reproduce, Distribute or Publicly Perform the Work either by itself or as part of any Adaptations or Collections, You must not distort, mutilate, modify or take other derogatory action in relation to the Work which would be prejudicial to the Original Author's honor or reputation. Licensor agrees that in those jurisdictions (e.g. Japan), in which any exercise of the right granted in Section 3(b) of this License (the right to make Adaptations) would be deemed to be a distortion, mutilation, modification or other derogatory action prejudicial to the Original Author's honor and reputation, the Licensor will waive or not assert, as appropriate, this Section, to the fullest extent permitted by the applicable national law, to enable You to reasonably exercise Your right under Section 3(b) of this License (right to make Adaptations) but not otherwise.
45
+
46
+ 5. Representations, Warranties and Disclaimer
47
+
48
+ UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
49
+
50
+ 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
51
+
52
+ 7. Termination
53
+
54
+ This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Adaptations or Collections from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License.
55
+ Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above.
56
+
57
+ 8. Miscellaneous
58
+
59
+ Each time You Distribute or Publicly Perform the Work or a Collection, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License.
60
+ Each time You Distribute or Publicly Perform an Adaptation, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License.
61
+ If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable.
62
+ No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent.
63
+ This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You.
64
+ The rights granted under, and the subject matter referenced, in this License were drafted utilizing the terminology of the Berne Convention for the Protection of Literary and Artistic Works (as amended on September 28, 1979), the Rome Convention of 1961, the WIPO Copyright Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 and the Universal Copyright Convention (as revised on July 24, 1971). These rights and subject matter take effect in the relevant jurisdiction in which the License terms are sought to be enforced according to the corresponding provisions of the implementation of those treaty provisions in the applicable national law. If the standard suite of rights granted under applicable copyright law includes additional rights not granted under this License, such additional rights are deemed to be included in the License; this License is not intended to restrict the license of any rights under applicable law.
65
+
66
+ Creative Commons Notice
67
+
68
+ Creative Commons is not a party to this License, and makes no warranty whatsoever in connection with the Work. Creative Commons will not be liable to You or any party on any legal theory for any damages whatsoever, including without limitation any general, special, incidental or consequential damages arising in connection to this license. Notwithstanding the foregoing two (2) sentences, if Creative Commons has expressly identified itself as the Licensor hereunder, it shall have all rights and obligations of Licensor.
69
+
70
+ Except for the limited purpose of indicating to the public that the Work is licensed under the CCPL, Creative Commons does not authorize the use by either party of the trademark "Creative Commons" or any related trademark or logo of Creative Commons without the prior written consent of Creative Commons. Any permitted use will be in compliance with Creative Commons' then-current trademark usage guidelines, as may be published on its website or otherwise made available upon request from time to time. For the avoidance of doubt, this trademark restriction does not form part of this License.
71
+
72
+ Creative Commons may be contacted at https://creativecommons.org/.
73
+
@@ -0,0 +1 @@
1
+ fastcat