solrclient 0.4.0__tar.gz

solrclient-0.4.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: solrclient
+Version: 0.4.0
+Summary: Python based client for Solr. 
+Home-page: https://github.com/moonlitesolutions/SolrClient
+Author: Nick Vasilyev
+Author-email: nick.vasilyev1@gmail.com
+License: MIT
+Requires-Dist: requests>=2.2.1
+Requires-Dist: kazoo==2.6.1
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary

solrclient-0.4.0/README.md

@@ -0,0 +1,77 @@
+[![Documentation Status](https://readthedocs.org/projects/solrclient/badge/?version=latest)](http://solrclient.readthedocs.org/en/latest/?badge=latest)
+
+# SolrClient
+SolrClient 0.2.2
+----------
+SolrClient is a simple python library for Solr; built in python3 with support for latest features of Solr 5 and 6. Development is heavily focused on indexing as well as parsing various query responses and returning them in native python data structures. Several helper classes will be built to automate querying and management of Solr clusters.
+
+Enhancements in version 0.2.0:
+* Basic parsing for json.facet output
+* Better support for grouped results (SolrResponse)
+* Other minor enhancements to SolrClient
+* Fixed SolrClient.index method
+
+Planned enhancements in version 0.3.0:
+* Solr node query routing (by @ddorian)
+* Streaming Expressions Support
+*
+
+Requirements
+----------
+* python 3.3+
+* requests library (http://docs.python-requests.org/en/latest/)
+* Solr
+* kazoo for working with zookeeper (optional)
+
+
+Features
+----------
+* Flexible and simple query mechanism
+* Response Object to easily extract data from Solr Response
+* Cursor Mark support
+* Indexing (raw JSON, JSON Files, gzipped JSON)
+* Specify multiple hosts/IPs for SolrCloud for redundancy
+* Basic Managed Schema field management
+* IndexManager for storing indexing documents off-line and batch indexing them
+
+Getting Started
+----------
+Installation:
+
+	pip install SolrClient
+
+Basic usage:
+
+	>>> from SolrClient import SolrClient
+	>>> solr = SolrClient('http://localhost:8983/solr')
+	>>> res = solr.query('SolrClient_unittest',{
+                'q':'product_name:Lorem',
+                'facet':True,
+                'facet.field':'facet_test',
+        })
+	>>> res.get_results_count()
+	4
+	>>> res.get_facets()
+	{'facet_test': {'ipsum': 0, 'sit': 0, 'dolor': 2, 'amet,': 1, 'Lorem': 1}}
+	>>> res.get_facet_keys_as_list('facet_test')
+	['ipsum', 'sit', 'dolor', 'amet,', 'Lorem']
+	>>> res.docs
+	[{'product_name_exact': 'orci. Morbi ipsum
+	..... all the docs ....
+	 'consectetur Mauris dolor Lorem adipiscing'}]
+
+See, easy.... you just need to know the Solr query syntax.
+
+
+Roadmap
+----------
+* Better test coverage
+* Solr Streaming
+
+Contributing
+----------
+I've realized that that there isn't really a well maintained Solr Python library I liked so I put this together. Contributions (code, tests, documentation) are definitely welcome; if you have a question about development please open up an issue on github page. If you have a pull request, please make sure to add tests and that all of them pass before submitting. See tests README for testing resources.
+
+
+Documentation:
+http://solrclient.readthedocs.org/en/latest/

solrclient-0.4.0/SolrClient/__init__.py

@@ -0,0 +1,10 @@
+from .solrclient import SolrClient
+from .solrresp import SolrResponse
+from .schema import Schema
+from .indexq import IndexQ
+from .helpers import Reindexer
+from .collections import Collections
+from .zk import ZK
+#This is the main project version. On new releases, it only needs to be updated here and in the README.md.
+#Documentation and setup.py will pull from here.
+__version__ = '0.4.0'

solrclient-0.4.0/SolrClient/collections.py

@@ -0,0 +1,183 @@
+import json
+import logging
+from .transport import TransportRequests
+from .schema import Schema
+from .exceptions import *
+from .solrresp import SolrResponse
+
+
+class Collections():
+    """
+    Provides an interface to Solr Collections API.
+    """
+
+    def __init__(self, solr, log):
+
+        self.solr = solr
+        self.logger = log
+        self.solr_clients = {}
+
+    def api(self, action, args=None):
+        """
+        Sends a request to Solr Collections API.
+        Documentation is here: https://cwiki.apache.org/confluence/display/solr/Collections+API
+
+        :param string action: Name of the collection for the action
+        :param dict args: Dictionary of specific parameters for action
+        """
+        if args is None:
+            args = {}
+        args['action'] = action.upper()
+
+        try:
+            res, con_info = self.solr.transport.send_request(endpoint='admin/collections', params=args)
+        except Exception as e:
+            self.logger.error("Error querying SolrCloud Collections API. ")
+            self.logger.exception(e)
+            raise e
+
+        if 'responseHeader' in res and res['responseHeader']['status'] == 0:
+            return res, con_info
+        else:
+            raise SolrError("Error Issuing Collections API Call for: {} +".format(con_info, res))
+
+    def clusterstatus(self):
+        """
+        Returns a slightly slimmed down version of the clusterstatus api command. It also gets count of documents in each shard on each replica and returns
+        it as doc_count key for each replica.
+
+        """
+
+        res = self.cluster_status_raw()
+
+        cluster = res['cluster']['collections']
+        out = {}
+        try:
+            for collection in cluster:
+                out[collection] = {}
+                for shard in cluster[collection]['shards']:
+                    out[collection][shard] = {}
+                    for replica in cluster[collection]['shards'][shard]['replicas']:
+                        out[collection][shard][replica] = cluster[collection]['shards'][shard]['replicas'][replica]
+                        if out[collection][shard][replica]['state'] != 'active':
+                            out[collection][shard][replica]['doc_count'] = False
+                        else:
+                            out[collection][shard][replica]['doc_count'] = self._get_collection_counts(
+                                out[collection][shard][replica])
+        except Exception as e:
+            self.logger.error("Couldn't parse response from clusterstatus API call")
+            self.logger.exception(e)
+
+        return out
+
+    def cluster_status_raw(self, **kwargs):
+        """
+        Returns raw output of the clusterstatus api command.
+
+        """
+        res, con_info = self.api('clusterstatus', **kwargs)
+        return res
+
+    def exists(self, collection):
+        """
+        Return True if a collection exists.
+        """
+        all_collections = self.list()
+        if collection in all_collections:
+            return True
+
+    def list(self):
+        """
+        Returns a list[string] of all collection names on the cluster.
+        """
+        res, info = self.api('LIST')
+        collections = res['collections']
+        return collections
+
+    def create(self, name, numShards, params=None):
+        """
+        Create a new collection.
+        """
+        if params is None:
+            params = {}
+        params.update(
+            name=name,
+            numShards=numShards
+        )
+        return self.api('CREATE', params)
+
+    def _get_collection_counts(self, core_data):
+        """
+        Queries each core to get individual counts for each core for each shard.
+        """
+        if core_data['base_url'] not in self.solr_clients:
+            from SolrClient import SolrClient
+            self.solr_clients['base_url'] = SolrClient(core_data['base_url'], log=self.logger)
+        try:
+            return self.solr_clients['base_url'].query(core_data['core'],
+                                                       {'q': '*:*',
+                                                        'rows': 0,
+                                                        'distrib': 'false',
+                                                        }).get_num_found()
+        except Exception as e:
+            self.logger.error("Couldn't get Counts for {}/{}".format(core_data['base_url'], core_data['core']))
+            self.logger.exception(e)
+            return False
+
+    def _for_core(self, cluster_resp=None):
+        if cluster_resp is None:
+            cluster_resp = self.clusterstatus()
+        for collection in cluster_resp:
+            for shard in cluster_resp[collection]:
+                for core in cluster_resp[collection][shard]:
+                    yield collection, shard, core, cluster_resp[collection][shard][core]
+
+    def _for_shard(self, cluster_resp=None):
+        if cluster_resp is None:
+            cluster_resp = self.clusterstatus()
+        for collection in cluster_resp:
+            for shard in cluster_resp[collection]:
+                yield collection, shard, cluster_resp[collection][shard]
+
+    def check_status(self, ignore=(), status=None):
+        """
+        Checks status of each collection and shard to make sure that:
+          a) Cluster state is active
+          b) Number of docs matches across replicas for a given shard.
+        Returns a dict of results for custom alerting.
+        """
+        self.SHARD_CHECKS = [
+            {'check_msg': 'Bad Core Count Check', 'f': self._check_shard_count},
+            {'check_msg': 'Bad Shard Cluster Status', 'f': self._check_shard_status}
+        ]
+        if status is None:
+            status = self.clusterstatus()
+        out = {}
+        for collection in status:
+            out[collection] = {}
+            out[collection]['coll_status'] = True  # Means it's fine
+            out[collection]['coll_messages'] = []
+            for shard in status[collection]:
+                self.logger.debug("Checking {}/{}".format(collection, shard))
+                s_dict = status[collection][shard]
+                for check in self.SHARD_CHECKS:
+                    if check['check_msg'] in ignore:
+                        continue
+                    res = check['f'](s_dict)
+                    if not res:
+                        out[collection]['coll_status'] = False
+                        if check['check_msg'] not in out[collection]['coll_messages']:
+                            out[collection]['coll_messages'].append(check['check_msg'])
+                        self.logger.debug(s_dict)
+        return out
+
+    def _check_shard_count(self, cores_dict):
+        if len(set([cores_dict[core]['doc_count'] for core in cores_dict])) > 1:
+            return False
+        return True
+
+    def _check_shard_status(self, cores_dict):
+        for core in cores_dict:
+            if cores_dict[core]['state'] != 'active':
+                return False
+        return True

solrclient-0.4.0/SolrClient/exceptions.py

@@ -0,0 +1,41 @@
+class SolrError(Exception):
+    """
+    Class to handle any issues that Solr Reports
+    """
+
+
+class SolrResponseError(SolrError):
+    """
+    Errors relatd to parsing Solr Response
+    """
+
+
+class ConnectionError(SolrError):
+    """
+    Errors connecting to Solr
+    """
+
+
+class ZookeeperError(SolrError):
+    """
+    Errors connecting to Zookeeper
+    """
+
+
+class NotFoundError(SolrError):
+    """
+    When a document wasn't found
+    """
+
+
+class MinRfError(SolrError):
+    """
+    When an index request didn't satisfy the required min_rf
+    """
+    rf = None
+    min_rf = None
+
+    def __init__(self, message, rf, min_rf, **kwargs):
+        self.rf = rf
+        self.min_rf = min_rf
+        super().__init__(self, message, **kwargs)

solrclient-0.4.0/SolrClient/helpers/__init__.py

@@ -0,0 +1 @@
+from .reindexer import Reindexer

solrclient-0.4.0/SolrClient/helpers/reindexer.py

@@ -0,0 +1,291 @@
+import datetime
+import logging
+import sys
+import gzip
+import argparse
+import os
+import json
+from datetime import datetime, timedelta
+from time import time, sleep
+from SolrClient import SolrClient, IndexQ
+
+
+class Reindexer():
+    '''
+    Initiates the re-indexer.
+
+    :param source: An instance of SolrClient.
+    :param dest: An instance of SolrClient or an instance of IndexQ.
+    :param string source_coll: Source collection name.
+    :param string dest_coll: Destination collection name; only required if destination is SolrClient.
+    :param int rows: Number of items to get in each query; default is 1000, however you will probably want to increase it.
+    :param string date_field: String name of a Solr date field to use in sort and resume.
+    :param bool devel: Whenever to turn on super verbouse logging for development. Standard DEBUG should suffice for most developemnt.
+    :param bool per_shard: Will add distrib=false to each query to get the data. Use this only if you will be running multiple instances of this to get the rest of the shards.
+    :param list ignore_fields: What fields to exclude from Solr queries. This is important since if you pull them out, you won't be able to index the documents in.
+    By default, it will try to determine and exclude copy fields as well as _version_. Pass in your own list to override or set it to False to prevent it from doing anything.
+    '''
+    def __init__(self,
+                source,
+                dest,
+                source_coll=None,
+                dest_coll=None,
+                rows=1000,
+                date_field=None,
+                devel=False,
+                per_shard=False,
+                ignore_fields=['_version_'],
+                ):
+
+
+        self.log = logging.getLogger('reindexer')
+
+        self._source = source
+        self._source_coll = source_coll
+        self._dest = dest
+        self._dest_coll = dest_coll
+        self._rows = rows
+        self._date_field = date_field
+        self._per_shard = per_shard
+        self._items_processed = 0
+        self._devel = devel
+        self._ignore_fields = ignore_fields
+
+
+        #Determine what source and destination should be
+        if type(source) is SolrClient and source_coll:
+            self._getter = self._from_solr
+             #Maybe break this out later for the sake of testing
+            if type(self._ignore_fields) is list and len(self._ignore_fields) == 1:
+                self._ignore_fields.extend(self._get_copy_fields())
+
+        elif type(source) is str and os.path.isdir(source):
+            self._getter = self._from_json
+        else:
+            raise ValueError("Incorrect Source Specified. Pass either a directory with json files or source SolrClient \
+                            instance with the name of the collection.")
+
+        if type(self._dest) is SolrClient and self._dest_coll:
+            self._putter = self._to_solr
+        elif type(dest) is IndexQ:
+            self._putter = self._to_IndexQ
+        else:
+           raise ValueError("Incorrect Destination Specified. Pass either a directory with json files or destination SolrClient \
+                            instance with the name of the collection.")
+        self.log.info("Reindexer created succesfully. ")
+
+
+    def _get_copy_fields(self):
+        if self._devel:
+            self.log.debug("Getting additional copy fields to exclude")
+            self.log.debug(self._source.schema.get_schema_copyfields(self._source_coll))
+        fields =  [field['dest'] for field in self._source.schema.get_schema_copyfields(self._source_coll)]
+        self.log.info("Field exclusions are: {}".format(", ".join(fields)))
+        return fields
+
+
+    def reindex(self, fq= [], **kwargs):
+        '''
+        Starts Reindexing Process. All parameter arguments will be passed down to the getter function.
+        :param string fq: FilterQuery to pass to source Solr to retrieve items. This can be used to limit the results.
+        '''
+        for items in self._getter(fq=fq, **kwargs):
+            self._putter(items)
+        if type(self._dest) is SolrClient and self._dest_coll:
+            self.log.info("Finished Indexing, sending a commit")
+            self._dest.commit(self._dest_coll, openSearcher=True)
+
+
+    def _from_solr(self, fq=[], report_frequency = 25):
+        '''
+        Method for retrieving batch data from Solr.
+        '''
+        cursor = '*'
+        stime = datetime.now()
+        query_count = 0
+        while True:
+            #Get data with starting cursorMark
+            query = self._get_query(cursor)
+            #Add FQ to the query. This is used by resume to filter on date fields and when specifying document subset.
+            #Not included in _get_query for more flexibiilty.
+
+            if fq:
+                if 'fq' in query:
+                    [query['fq'].append(x) for x in fq]
+                else:
+                    query['fq'] = fq
+
+            results = self._source.query(self._source_coll, query)
+            query_count += 1
+            if query_count % report_frequency == 0:
+                self.log.info("Processed {} Items in {} Seconds. Apprximately {} items/minute".format(
+                            self._items_processed, int((datetime.now()-stime).seconds),
+                            str(int(self._items_processed / ((datetime.now()-stime).seconds/60)))
+                            ))
+
+            if results.get_results_count():
+                #If we got items back, get the new cursor and yield the docs
+                self._items_processed += results.get_results_count()
+                cursor = results.get_cursor()
+                #Remove ignore fields
+                docs = self._trim_fields(results.docs)
+                yield docs
+                if results.get_results_count() < self._rows:
+                    #Less results than asked, probably done
+                    break
+            else:
+                #No Results, probably done :)
+                self.log.debug("Got zero Results with cursor: {}".format(cursor))
+                break
+
+
+    def _trim_fields(self, docs):
+        '''
+        Removes ignore fields from the data that we got from Solr.
+        '''
+        for doc in docs:
+            for field in self._ignore_fields:
+                if field in doc:
+                    del(doc[field])
+        return docs
+
+
+    def _get_query(self, cursor):
+        '''
+        Query tempalte for source Solr, sorts by id by default.
+        '''
+        query = {'q':'*:*',
+                'sort':'id desc',
+                'rows':self._rows,
+                'cursorMark':cursor}
+        if self._date_field:
+            query['sort'] = "{} asc, id desc".format(self._date_field)
+        if self._per_shard:
+            query['distrib'] = 'false'
+        return query
+
+
+    def _to_IndexQ(self, data):
+        '''
+        Sends data to IndexQ instance.
+        '''
+        self._dest.add(data)
+
+
+    def _to_solr(self, data):
+        '''
+        Sends data to a Solr instance.
+        '''
+        return self._dest.index_json(self._dest_coll, json.dumps(data,sort_keys=True))
+
+
+    def _get_date_range_query(self, start_date, end_date, timespan= 'DAY', date_field= None):
+        '''
+        Gets counts of items per specified date range.
+        :param collection: Solr Collection to use.
+        :param timespan: Solr Date Math compliant value for faceting ex HOUR, MONTH, DAY
+        '''
+        if date_field is None:
+            date_field = self._date_field
+        query ={'q':'*:*',
+                'rows':0,
+                'facet':'true',
+                'facet.range': date_field,
+                'facet.range.gap': '+1{}'.format(timespan),
+                'facet.range.end': '{}'.format(end_date),
+                'facet.range.start': '{}'.format(start_date),
+                'facet.range.include': 'all'
+                }
+        if self._per_shard:
+            query['distrib'] = 'false'
+        return query
+
+
+    def _get_edge_date(self, date_field, sort):
+        '''
+        This method is used to get start and end dates for the collection.
+        '''
+        return self._source.query(self._source_coll, {
+                'q':'*:*',
+                'rows':1,
+                'fq':'+{}:*'.format(date_field),
+                'sort':'{} {}'.format(date_field, sort)}).docs[0][date_field]
+
+
+    def _get_date_facet_counts(self, timespan, date_field, start_date=None, end_date=None):
+        '''
+        Returns Range Facet counts based on
+        '''
+        if 'DAY' not in timespan:
+            raise ValueError("At this time, only DAY date range increment is supported. Aborting..... ")
+
+        #Need to do this a bit better later. Don't like the string and date concatenations.
+        if not start_date:
+            start_date = self._get_edge_date(date_field, 'asc')
+            start_date = datetime.strptime(start_date,'%Y-%m-%dT%H:%M:%S.%fZ').date().isoformat()+'T00:00:00.000Z'
+        else:
+            start_date = start_date+'T00:00:00.000Z'
+
+        if not end_date:
+            end_date = self._get_edge_date(date_field, 'desc')
+            end_date = datetime.strptime(end_date,'%Y-%m-%dT%H:%M:%S.%fZ').date()
+            end_date += timedelta(days=1)
+            end_date = end_date.isoformat()+'T00:00:00.000Z'
+        else:
+            end_date = end_date+'T00:00:00.000Z'
+
+
+        self.log.info("Processing Items from {} to {}".format(start_date, end_date))
+
+        #Get facet counts for source and destination collections
+        source_facet = self._source.query(self._source_coll,
+            self._get_date_range_query(timespan=timespan, start_date=start_date, end_date=end_date)
+            ).get_facets_ranges()[date_field]
+        dest_facet = self._dest.query(
+            self._dest_coll, self._get_date_range_query(
+                    timespan=timespan, start_date=start_date, end_date=end_date
+                    )).get_facets_ranges()[date_field]
+        return source_facet, dest_facet
+
+
+    def resume(self, start_date=None, end_date=None, timespan='DAY', check= False):
+        '''
+        This method may help if the original run was interrupted for some reason. It will only work under the following conditions
+        * You have a date field that you can facet on
+        * Indexing was stopped for the duration of the copy
+
+        The way this tries to resume re-indexing is by running a date range facet on the source and destination collections. It then compares
+        the counts in both collections for each timespan specified. If the counts are different, it will re-index items for each range where
+        the counts are off. You can also pass in a start_date to only get items after a certain time period. Note that each date range will be indexed in
+        it's entirety, even if there is only one item missing.
+
+        Keep in mind this only checks the counts and not actual data. So make the indexes weren't modified between the reindexing execution and
+        running the resume operation.
+
+        :param start_date: Date to start indexing from. If not specified there will be no restrictions and all data will be processed. Note that
+        this value will be passed to Solr directly and not modified.
+        :param end_date: The date to index items up to. Solr Date Math compliant value for faceting; currenlty only DAY is supported.
+        :param timespan: Solr Date Math compliant value for faceting; currenlty only DAY is supported.
+        :param check: If set to True it will only log differences between the two collections without actually modifying the destination.
+        '''
+
+        if type(self._source) is not SolrClient or type(self._dest) is not SolrClient:
+            raise ValueError("To resume, both source and destination need to be Solr.")
+
+        source_facet, dest_facet = self._get_date_facet_counts(timespan, self._date_field, start_date=start_date, end_date=end_date)
+
+        for dt_range in sorted(source_facet):
+            if dt_range in dest_facet:
+                self.log.info("Date Range: {} Source: {} Destination:{} Difference:{}".format(
+                        dt_range, source_facet[dt_range], dest_facet[dt_range], (source_facet[dt_range]-dest_facet[dt_range])))
+                if check:
+                    continue
+                if source_facet[dt_range] > dest_facet[dt_range]:
+                    #Kicks off reindexing with an additional FQ
+                    self.reindex(fq=['{}:[{} TO {}]'.format(self._date_field, dt_range, dt_range+'+1{}'.format(timespan))])
+                    self.log.info("Complete Date Range {}".format(dt_range))
+            else:
+                self.log.error("Something went wrong; destinationSource: {}".format(source_facet))
+                self.log.error("Destination: {}".format(dest_facet))
+                raise ValueError("Date Ranges don't match up")
+        self._dest.commit(self._dest_coll, openSearcher=True)