scout 3.0.4__py3-none-any.whl

scout/__init__.py

@@ -0,0 +1 @@
+__version__ = '3.0.4'

scout/__main__.py

@@ -0,0 +1,5 @@
+from scout.server import main
+
+
+if __name__ == '__main__':
+    main()

scout/client.py

@@ -0,0 +1,269 @@
+import base64
+import json
+try:
+    from email.generator import _make_boundary as choose_boundary
+except ImportError:
+    from mimetools import choose_boundary
+import mimetypes
+import os
+try:
+    from urllib.parse import urlencode
+except ImportError:
+    from urllib import urlencode
+try:
+    from urllib.request import Request
+    from urllib.request import urlopen
+except ImportError:
+    from urllib2 import Request
+    from urllib2 import urlopen
+import zlib
+
+
+ENDPOINT = None
+KEY = None
+
+
+class Scout(object):
+    def __init__(self, endpoint=ENDPOINT, key=KEY):
+        self.endpoint = endpoint.rstrip('/')
+        self.key = key
+
+    def get_full_url(self, url):
+        return self.endpoint + url
+
+    def get_raw(self, url, **kwargs):
+        headers = {'Content-Type': 'application/json'}
+        if self.key:
+            headers['key'] = self.key
+        if kwargs:
+            if '?' not in url:
+                url += '?'
+            url += urlencode(kwargs, True)
+        request = Request(self.get_full_url(url), headers=headers)
+        fh = urlopen(request)
+        return fh.read()
+
+    def get(self, url, **kwargs):
+        return json.loads(self.get_raw(url, **kwargs))
+
+    def post(self, url, data=None, files=None):
+        if files:
+            return self.post_files(url, data, files)
+        else:
+            return self.post_json(url, data)
+
+    def post_json(self, url, data=None):
+        headers = {'Content-Type': 'application/json'}
+        if self.key:
+            headers['key'] = self.key
+        data = json.dumps(data or {})
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
+        request = Request(self.get_full_url(url), data=data, headers=headers)
+        return json.loads(urlopen(request).read().decode('utf8'))
+
+    def post_files(self, url, json_data, files=None):
+        if not files or not isinstance(files, dict):
+            raise ValueError('One or more files is required. Files should be '
+                             'passed as a dictionary of filename: file-like-'
+                             'object.')
+        boundary = choose_boundary()
+        form_files = []
+        for i, (filename, file_obj) in enumerate(files.items()):
+            try:
+                data = file_obj.read()
+            except AttributeError:
+                data = bytes(file_obj)
+            mimetype = mimetypes.guess_type(filename)[0]
+            form_files.append((
+                'file_%s' % i,
+                filename,
+                mimetype or 'application/octet-stream',
+                data))
+
+        part_boundary = '--' + boundary
+        parts = [
+            part_boundary,
+            'Content-Disposition: form-data; name="data"',
+            '',
+            json.dumps(json_data)]
+        for field_name, filename, mimetype, data in form_files:
+            parts.extend((
+                part_boundary,
+                'Content-Disposition: file; name="%s"; filename="%s"' % (
+                    field_name, filename),
+                'Content-Type: %s' % mimetype,
+                '',
+                data))
+        parts.append('--' + boundary + '--')
+        parts.append('')
+
+        headers = {'Content-Type': 'multipart/form-data; boundary=%s' %
+                   boundary}
+        if self.key:
+            headers['key'] = self.key
+
+        data = '\r\n'.join(parts)
+        if not isinstance(data, bytes):
+            data = data.encode('utf-8')
+
+        request = Request(self.get_full_url(url), data=data, headers=headers)
+        return json.loads(urlopen(request).read())
+
+    def delete(self, url):
+        headers = {}
+        if self.key:
+            headers['key'] = self.key
+        request = Request(self.get_full_url(url), headers=headers)
+        request.get_method = lambda: 'DELETE'
+        fh = urlopen(request)
+        return json.loads(fh.read())
+
+    def get_indexes(self, **kwargs):
+        return self.get('/', **kwargs)['indexes']
+
+    def create_index(self, name):
+        return self.post('/', {'name': name})
+
+    def rename_index(self, old_name, new_name):
+        return self.post('/%s/' % old_name, {'name': new_name})
+
+    def delete_index(self, name):
+        return self.delete('/%s/' % name)
+
+    def get_index(self, name, **kwargs):
+        return self.get('/%s/' % name, **kwargs)
+
+    def get_documents(self, **kwargs):
+        return self.get('/documents/', **kwargs)
+
+    def create_document(self, content, indexes, identifier=None,
+                        attachments=None, **metadata):
+        if not isinstance(indexes, (list, tuple)):
+            indexes = [indexes]
+        post_data = {
+            'content': content,
+            'identifier': identifier,
+            'indexes': indexes,
+            'metadata': metadata}
+        return self.post('/documents/', post_data, attachments)
+
+    def update_document(self, document_id=None, content=None, indexes=None,
+                        metadata=None, identifier=None, attachments=None):
+        if not document_id and not identifier:
+            raise ValueError('`document_id` must be provided.')
+
+        data = {}
+        if content is not None:
+            data['content'] = content
+        if indexes is not None:
+            if not isinstance(indexes, (list, tuple)):
+                indexes = [indexes]
+            data['indexes'] = indexes
+        if metadata is not None:
+            data['metadata'] = metadata
+
+        if not data and not attachments:
+            raise ValueError('Nothing to update.')
+
+        return self.post('/documents/%s/' % document_id, data, attachments)
+
+    def delete_document(self, document_id=None):
+        if not document_id:
+            raise ValueError('`document_id` must be provided.')
+
+        return self.delete('/documents/%s/' % document_id)
+
+    def get_document(self, document_id=None):
+        if not document_id:
+            raise ValueError('`document_id` must be provided.')
+
+        return self.get('/documents/%s/' % document_id)
+
+    def attach_files(self, document_id, attachments):
+        return self.post_files('/documents/%s/attachments/' % document_id,
+                               {}, attachments)
+
+    def detach_file(self, document_id, filename):
+        return self.delete('/documents/%s/attachments/%s/' %
+                           (document_id, filename))
+
+    def update_file(self, document_id, filename, file_object):
+        return self.post_files('/documents/%s/attachments/%s/' %
+                               (document_id, filename),
+                               {}, {filename: file_object})
+
+    def get_attachments(self, document_id, **kwargs):
+        return self.get('/documents/%s/attachments/' % document_id, **kwargs)
+
+    def get_attachment(self, document_id, filename):
+        return self.get('/documents/%s/attachments/%s/' %
+                        (document_id, filename))
+
+    def download_attachment(self, document_id, filename):
+        return self.get_raw('/documents/%s/attachments/%s/download/' %
+                            (document_id, filename))
+
+    def search_attachments(self, **kwargs):
+        return self.get('/documents/attachments/', **kwargs)
+
+
+class SearchProvider(object):
+    def content(self, obj):
+        raise NotImplementedError
+
+    def identifier(self, obj):
+        raise NotImplementedError
+
+    def metadata(self, obj):
+        raise NotImplementedError
+
+
+class SearchSite(object):
+    def __init__(self, client, index):
+        self.client = client
+        self.index = index
+        self.registry = {}
+
+    def register(self, model_class, search_provider):
+        self.registry.setdefault(model_class, [])
+        self.registry[model_class].append(search_provider())
+
+    def unregister(self, model_class, search_provider=None):
+        if search_provider is None:
+            self.registry.pop(model_class, None)
+        elif model_class in self.registry:
+            self.registry[model_class] = [
+                sp for sp in self.registry[model_class]
+                if not isinstance(sp, search_provider)]
+
+    def store(self, obj):
+        if type(obj) not in self.registry:
+            return False
+
+        for provider in self.registry[type(obj)]:
+            content = provider.content(obj)
+            try:
+                metadata = provider.metadata(obj)
+            except NotImplementedError:
+                metadata = {}
+
+            try:
+                identifier = provider.identifier(obj)
+            except NotImplementedError:
+                pass
+            else:
+                metadata['identifier'] = identifier
+
+            self.client.create_document(content, self.index, **metadata)
+
+        return True
+
+    def remove(self, obj):
+        if type(obj) not in self.registry:
+            return False
+
+        for provider in self.registry[type(obj)]:
+            self.client.delete_document(provider.identifier(obj))
+
+        return True

scout/constants.py

@@ -0,0 +1,7 @@
+SEARCH_BM25 = 'bm25'
+SEARCH_SIMPLE = 'simple'
+SEARCH_NONE = 'none'
+RANKING_CHOICES = (SEARCH_BM25, SEARCH_SIMPLE, SEARCH_NONE)
+
+PROTECTED_KEYS = set(('page', 'q', 'key', 'ranking', 'identifier', 'index',
+                      'ordering'))

scout/exceptions.py

@@ -0,0 +1,21 @@
+from flask import jsonify
+
+
+class InvalidSearchException(ValueError): pass
+
+
+class InvalidRequestException(Exception):
+    def __init__(self, error_message, code=None):
+        self.error_message = error_message
+        self.code = code or 400
+
+    def response(self):
+        return jsonify({'error': self.error_message}), self.code
+
+
+def error(message, code=None):
+    """
+    Trigger an Exception that will short-circuit the Response cycle and return
+    a 400 "Bad request" with the given error message.
+    """
+    raise InvalidRequestException(message, code=code)

scout/models.py

@@ -0,0 +1,223 @@
+import base64
+import datetime
+import hashlib
+import mimetypes
+import sys
+
+from peewee import *
+from playhouse.fields import CompressedField
+from playhouse.sqlite_ext import *
+try:
+    from playhouse.sqlite_ext import CSqliteExtDatabase as SqliteExtDatabase
+except ImportError:
+    pass
+try:
+    from werkzeug import secure_filename
+except ImportError:
+    from werkzeug.utils import secure_filename
+
+
+if sys.version_info[0] == 2:
+    unicode_type = unicode
+else:
+    unicode_type = str
+
+
+database = SqliteExtDatabase(None, regexp_function=True)
+
+
+class Document(FTSModel):
+    """
+    The :py:class:`Document` class contains content which should be indexed
+    for search. Documents can be associated with any number of indexes via
+    the `IndexDocument` junction table. Because `Document` is implemented
+    as an FTS virtual table, it does not support any secondary indexes, and
+    all columns have *Text* type, regardless of their declared type. For that
+    reason we will utilize the internal SQLite `docid` column to relate
+    documents to indexes.
+    """
+    content = SearchField()
+    identifier = SearchField()
+
+    class Meta:
+        database = database
+        options = {
+            'prefix': [2, 3],
+            'tokenize': 'porter unicode61'}
+        table_name = 'main_document'
+
+    @classmethod
+    def all(cls):
+        return Document.select(Document.docid, Document.content,
+                               Document.identifier)
+
+    def get_metadata(self):
+        return dict(Metadata
+                    .select(Metadata.key, Metadata.value)
+                    .where(Metadata.document == self.docid)
+                    .tuples())
+
+    def set_metadata(self, metadata):
+        (Metadata
+         .replace_many([
+             {'key': key, 'value': value, 'document': self.docid}
+             for key, value in metadata.items()])
+         .execute())
+
+    def delete_metadata(self):
+        Metadata.delete().where(Metadata.document == self.docid).execute()
+
+    metadata = property(get_metadata, set_metadata, delete_metadata)
+
+    def get_indexes(self):
+        return (Index
+                .select()
+                .join(IndexDocument)
+                .where(IndexDocument.document == self.docid))
+
+    def attach(self, filename, data):
+        filename = secure_filename(filename)
+        if isinstance(data, unicode_type):
+            data = data.encode('utf-8')
+        hash_obj = hashlib.sha256(data)
+        data_hash = base64.b64encode(hash_obj.digest())
+        try:
+            with database.atomic():
+                data_obj = BlobData.create(hash=data_hash, data=data)
+        except IntegrityError:
+            pass
+
+        mimetype = mimetypes.guess_type(filename)[0] or 'text/plain'
+        try:
+            with database.atomic():
+                attachment = Attachment.create(
+                    document=self,
+                    filename=filename,
+                    hash=data_hash,
+                    mimetype=mimetype)
+        except IntegrityError:
+            attachment = (Attachment
+                          .get((Attachment.document == self) &
+                               (Attachment.filename == filename)))
+            attachment.hash = data_hash
+            attachment.mimetype = mimetype
+            attachment.save(only=[Attachment.hash, Attachment.mimetype])
+
+        return attachment
+
+    def detach(self, filename):
+        return (Attachment
+                .delete()
+                .where((Attachment.document == self) &
+                       (Attachment.filename == filename))
+                .execute())
+
+
+class BaseModel(Model):
+    class Meta:
+        database = database
+
+
+class Attachment(BaseModel):
+    """
+    A mapping of a BLOB to a Document.
+    """
+    document = ForeignKeyField(Document, backref='attachments')
+    hash = TextField()
+    filename = TextField(index=True)
+    mimetype = TextField()
+    timestamp = DateTimeField(default=datetime.datetime.now, index=True)
+
+    class Meta:
+        indexes = (
+            (('document', 'filename'), True),
+        )
+
+    @property
+    def blob(self):
+        if not hasattr(self, '_blob'):
+            self._blob = BlobData.get(BlobData.hash == self.hash)
+        return self._blob
+
+    @property
+    def length(self):
+        return len(self.blob.data)
+
+
+class BlobData(BaseModel):
+    """Content-addressable BLOB."""
+    hash = TextField(primary_key=True)
+    data = CompressedField(compression_level=6, algorithm='zlib')
+
+
+class Metadata(BaseModel):
+    """
+    Arbitrary key/value pairs associated with an indexed `Document`. The
+    metadata associated with a document can also be used to filter search
+    results.
+    """
+    document = ForeignKeyField(Document, backref='metadata_set')
+    key = TextField()
+    value = TextField()
+
+    class Meta:
+        indexes = (
+            (('document', 'key'), True),
+            (('key', 'value'), False),
+        )
+        table_name = 'main_metadata'
+
+
+class Index(BaseModel):
+    """
+    Indexes contain any number of documents and expose a clean API for
+    searching and storing content.
+    """
+    name = TextField(unique=True)
+
+    class Meta:
+        table_name = 'main_index'
+
+    def add_to_index(self, document):
+        with database.atomic():
+            try:
+                IndexDocument.create(index=self, document=document)
+            except IntegrityError:
+                pass
+
+    def index(self, content, document=None, identifier=None, **metadata):
+        if document is None:
+            document = Document.create(
+                content=content,
+                identifier=identifier)
+        else:
+            del document.metadata
+            nrows = (Document
+                     .update(
+                         content=content,
+                         identifier=identifier)
+                     .where(Document.docid == document.docid)
+                     .execute())
+
+        self.add_to_index(document)
+        if metadata:
+            document.metadata = metadata
+        return document
+
+    @property
+    def documents(self):
+        return (Document
+                .all()
+                .join(IndexDocument)
+                .where(IndexDocument.index == self))
+
+
+class IndexDocument(BaseModel):
+    index = ForeignKeyField(Index)
+    document = ForeignKeyField(Document)
+
+    class Meta:
+        indexes = (
+            (('index', 'document'), True),
+        )
+        table_name = 'main_index_document'

scout/search.py

@@ -0,0 +1,135 @@
+try:
+    from functools import reduce
+except ImportError:
+    pass
+import operator
+
+from peewee import fn
+from peewee import Select
+
+from .constants import PROTECTED_KEYS
+from .constants import SEARCH_BM25
+from .constants import SEARCH_NONE
+from .constants import SEARCH_SIMPLE
+from .exceptions import InvalidSearchException
+from .exceptions import error
+from .models import Document
+from .models import Index
+from .models import IndexDocument
+from .models import Metadata
+
+
+class DocumentSearch(object):
+    def search(self, phrase, index=None, ranking='bm25', ordering=None,
+               **filters):
+        phrase = phrase.strip()
+        if not phrase:
+            raise InvalidSearchException('Must provide a search query.')
+        elif phrase == '*' or ranking == SEARCH_NONE:
+            ranking = None
+
+        query = Document.select()
+        if phrase != '*':
+            query = query.where(Document.match(phrase))
+
+        # Allow filtering by index(es).
+        if index is not None:
+            query = query.join(IndexDocument)
+            if isinstance(index, (list, tuple, Select)):
+                query = query.where(IndexDocument.index << index)
+            else:
+                query = query.where(IndexDocument.index == index)
+
+        # Allow filtering by metadata.
+        metadata_expr = self.get_metadata_filter_expression(filters)
+        if metadata_expr is not None:
+            query = query.where(metadata_expr)
+
+        # Allow sorting and ranking.
+        return self.apply_rank_and_sort(query, ranking, ordering or ())
+
+    def get_metadata_filter_expression(self, filters):
+        valid_keys = [key for key in filters if key not in PROTECTED_KEYS]
+        if valid_keys:
+            return reduce(operator.and_, [
+                self._build_filter_expression(key, values)
+                for key, values in filters.items()])
+
+    @staticmethod
+    def _build_filter_expression(key, values):
+        def in_(lhs, rhs):
+            return lhs << ([i.strip() for i in rhs.split(',')])
+        operations = {
+            'eq': operator.eq,
+            'ne': operator.ne,
+            'ge': operator.ge,
+            'gt': operator.gt,
+            'le': operator.le,
+            'lt': operator.lt,
+            'in': in_,
+            'contains': lambda l, r: operator.pow(l, '%%%s%%' % r),
+            'startswith': lambda l, r: operator.pow(l, '%s%%' % r),
+            'endswith': lambda l, r: operator.pow(l, '%%%s' % r),
+            'regex': lambda l, r: l.regexp(r),
+        }
+        if key.find('__') != -1:
+            key, op = key.rsplit('__', 1)
+            if op not in operations:
+                error('Unrecognized operation: %s. Supported operations are:'
+                      '\n%s' % (op, '\n'.join(sorted(operations.keys()))))
+        else:
+            op = 'eq'
+
+        op = operations[op]
+        if isinstance(values, (list, tuple)):
+            expr = reduce(operator.or_, [
+                ((Metadata.key == key) & op(Metadata.value, value))
+                for value in values])
+        else:
+            expr = ((Metadata.key == key) & op(Metadata.value, values))
+
+        return fn.EXISTS(Metadata.select().where(
+            expr &
+            (Metadata.document == Document.docid)))
+
+    def apply_rank_and_sort(self, query, ranking, ordering, sort_options=None,
+                            sort_default='id'):
+        sort_options = sort_options or {
+            'content': Document.content,
+            'id': Document.docid,
+            'identifier': Document.identifier,
+        }
+        if ranking is not None:
+            rank = self.get_rank_expression(ranking)
+            sort_options['score'] = rank
+            sort_default = 'score'
+
+            # Add score to the selected columns.
+            query = query.select(*query._returning + [rank.alias('score')])
+
+        return self.apply_sorting(query, ordering, sort_options, sort_default)
+
+    def get_rank_expression(self, ranking):
+        if ranking == SEARCH_BM25:
+            # Search only the content field, do not search the identifiers.
+            return Document.bm25(1.0, 0.0)
+        elif ranking == SEARCH_SIMPLE:
+            # Search only the content field, do not search the identifiers.
+            return Document.rank(1.0, 0.0)
+        else:
+            error('Unrecognized ranking: "%s"' % ranking)
+
+    def apply_sorting(self, query, ordering, mapping, default):
+        sortables = [part.strip() for part in ordering]
+        accum = []
+        for identifier in sortables:
+            is_desc = identifier.startswith('-')
+            identifier = identifier.lstrip('-')
+            if identifier in mapping:
+                value = mapping[identifier]
+                accum.append(value.desc() if is_desc else value)
+
+        if not accum:
+            accum = [mapping[default]]
+
+        return query.order_by(*accum)