crate 2.0.0.dev0__py3-none-any.whl

crate/testing/layer.py

@@ -0,0 +1,428 @@
+# -*- coding: utf-8; -*-
+#
+# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
+# license agreements.  See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership.  Crate licenses
+# this file to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# However, if you have executed another commercial license agreement
+# with Crate these terms will supersede the license and you may use the
+# software solely pursuant to the terms of the relevant commercial agreement.
+
+# ruff: noqa: S603  # `subprocess` call: check for execution of untrusted input
+# ruff: noqa: S202  # Uses of `tarfile.extractall()`
+
+import io
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import threading
+import time
+
+import urllib3
+
+try:
+    from urllib.request import urlopen
+except ImportError:
+    from urllib import urlopen  # type: ignore[attr-defined,no-redef]
+
+
+log = logging.getLogger(__name__)
+
+
+CRATE_CONFIG_ERROR = (
+    'crate_config must point to a folder or to a file named "crate.yml"'
+)
+HTTP_ADDRESS_RE = re.compile(
+    r".*\[(http|.*HttpServer.*)\s*] \[.*\] .*"
+    "publish_address {"
+    r"(?:inet\[[\w\d\.-]*/|\[)?"
+    r"(?:[\w\d\.-]+/)?"
+    r"(?P<addr>[\d\.:]+)"
+    r"(?:\])?"
+    "}"
+)
+
+
+def http_url_from_host_port(host, port):
+    if host and port:
+        if not isinstance(port, int):
+            try:
+                port = int(port)
+            except ValueError:
+                return None
+        return "{}:{}".format(prepend_http(host), port)
+    return None
+
+
+def prepend_http(host):
+    if not re.match(r"^https?\:\/\/.*", host):
+        return "http://{}".format(host)
+    return host
+
+
+def _download_and_extract(uri, directory):
+    sys.stderr.write(
+        "\nINFO:    Downloading CrateDB archive from {} into {}".format(
+            uri, directory
+        )
+    )
+    sys.stderr.flush()
+    with io.BytesIO(urlopen(uri).read()) as tmpfile:
+        with tarfile.open(fileobj=tmpfile) as t:
+            t.extractall(directory)
+
+
+def wait_for_http_url(log, timeout=30, verbose=False):
+    start = time.monotonic()
+    while True:
+        line = log.readline().decode("utf-8").strip()
+        elapsed = time.monotonic() - start
+        if verbose:
+            sys.stderr.write("[{:>4.1f}s]{}\n".format(elapsed, line))
+        m = HTTP_ADDRESS_RE.match(line)
+        if m:
+            return prepend_http(m.group("addr"))
+        elif elapsed > timeout:
+            return None
+
+
+class OutputMonitor:
+    def __init__(self):
+        self.consumers = []
+
+    def consume(self, iterable):
+        for line in iterable:
+            for consumer in self.consumers:
+                consumer.send(line)
+
+    def start(self, proc):
+        self._stop_out_thread = threading.Event()
+        self._out_thread = threading.Thread(
+            target=self.consume, args=(proc.stdout,)
+        )
+        self._out_thread.daemon = True
+        self._out_thread.start()
+
+    def stop(self):
+        if self._out_thread is not None:
+            self._stop_out_thread.set()
+            self._out_thread.join()
+
+
+class LineBuffer:
+    def __init__(self):
+        self.lines = []
+
+    def send(self, line):
+        self.lines.append(line.strip())
+
+
+class CrateLayer:
+    """
+    This layer starts a Crate server.
+    """
+
+    __bases__ = ()
+
+    tmpdir = tempfile.gettempdir()
+    wait_interval = 0.2
+
+    @staticmethod
+    def from_uri(
+        uri,
+        name,
+        http_port="4200-4299",
+        transport_port="4300-4399",
+        settings=None,
+        directory=None,
+        cleanup=True,
+        verbose=False,
+    ):
+        """Download the Crate tarball from a URI and create a CrateLayer
+
+        :param uri: The uri that points to the Crate tarball
+        :param name: layer and cluster name
+        :param http_port: The http port on which Crate will listen
+        :param transport_port: the transport port on which Crate will listen
+        :param settings: A dictionary that contains Crate settings
+        :param directory: Where the tarball will be extracted to.
+                          If this is None a temporary directory will be created.
+        :param clean: a boolean indicating if the directory should be removed
+                      on teardown.
+        :param verbose: Set the log verbosity of the test layer
+        """
+        directory = directory or tempfile.mkdtemp()
+        filename = os.path.basename(uri)
+        crate_dir = re.sub(r"\.tar(\.gz)?$", "", filename)
+        crate_home = os.path.join(directory, crate_dir)
+
+        if os.path.exists(crate_home):
+            sys.stderr.write(
+                "\nWARNING: Not extracting CrateDB tarball"
+                " because folder already exists"
+            )
+            sys.stderr.flush()
+        else:
+            _download_and_extract(uri, directory)
+
+        layer = CrateLayer(
+            name=name,
+            crate_home=crate_home,
+            port=http_port,
+            transport_port=transport_port,
+            settings=settings,
+            verbose=verbose,
+        )
+        if cleanup:
+            tearDown = layer.tearDown
+
+            def new_teardown(*args, **kws):
+                shutil.rmtree(directory)
+                tearDown(*args, **kws)
+
+            layer.tearDown = new_teardown  # type: ignore[method-assign]
+        return layer
+
+    def __init__(
+        self,
+        name,
+        crate_home,
+        crate_config=None,
+        port=None,
+        keepRunning=False,
+        transport_port=None,
+        crate_exec=None,
+        cluster_name=None,
+        host="127.0.0.1",
+        settings=None,
+        verbose=False,
+        env=None,
+    ):
+        """
+        :param name: layer name, is also used as the cluser name
+        :param crate_home: path to home directory of the crate installation
+        :param port: port on which crate should run
+        :param transport_port: port on which transport layer for crate should
+                               run
+        :param crate_exec: alternative executable command
+        :param crate_config: alternative crate config file location.
+                             Must be a directory or a file named 'crate.yml'
+        :param cluster_name: the name of the cluster to join/build. Will be
+                             generated automatically if omitted.
+        :param host: the host to bind to. defaults to 'localhost'
+        :param settings: further settings that do not deserve a keyword
+                         argument will be prefixed with ``es.``.
+        :param verbose: Set the log verbosity of the test layer
+        :param env: Set environment variables.
+        """
+        self.__name__ = name
+        if settings and isinstance(settings, dict):
+            # extra settings may override host/port specification!
+            self.http_url = http_url_from_host_port(
+                settings.get("network.host", host),
+                settings.get("http.port", port),
+            )
+        else:
+            self.http_url = http_url_from_host_port(host, port)
+
+        self.process = None
+        self.verbose = verbose
+        self.env = env or {}
+        self.env.setdefault("CRATE_USE_IPV4", "true")
+        self.env.setdefault("JAVA_HOME", os.environ.get("JAVA_HOME", ""))
+        self._stdout_consumers = []
+        self.conn_pool = urllib3.PoolManager(num_pools=1)
+
+        crate_home = os.path.abspath(crate_home)
+        if crate_exec is None:
+            start_script = "crate.bat" if sys.platform == "win32" else "crate"
+            crate_exec = os.path.join(crate_home, "bin", start_script)
+        if crate_config is None:
+            crate_config = os.path.join(crate_home, "config", "crate.yml")
+        elif (
+            os.path.isfile(crate_config)
+            and os.path.basename(crate_config) != "crate.yml"
+        ):
+            raise ValueError(CRATE_CONFIG_ERROR)
+        if cluster_name is None:
+            cluster_name = "Testing{0}".format(port or "Dynamic")
+        settings = self.create_settings(
+            crate_config,
+            cluster_name,
+            name,
+            host,
+            port or "4200-4299",
+            transport_port or "4300-4399",
+            settings,
+        )
+        # ES 5 cannot parse 'True'/'False' as booleans so convert to lowercase
+        start_cmd = (crate_exec,) + tuple(
+            [
+                "-C%s=%s"
+                % (
+                    (key, str(value).lower())
+                    if isinstance(value, bool)
+                    else (key, value)
+                )
+                for key, value in settings.items()
+            ]
+        )
+
+        self._wd = wd = os.path.join(CrateLayer.tmpdir, "crate_layer", name)
+        self.start_cmd = start_cmd + ("-Cpath.data=%s" % wd,)
+
+    def create_settings(
+        self,
+        crate_config,
+        cluster_name,
+        node_name,
+        host,
+        http_port,
+        transport_port,
+        further_settings=None,
+    ):
+        settings = {
+            "discovery.type": "zen",
+            "discovery.initial_state_timeout": 0,
+            "node.name": node_name,
+            "cluster.name": cluster_name,
+            "network.host": host,
+            "http.port": http_port,
+            "path.conf": os.path.dirname(crate_config),
+            "transport.tcp.port": transport_port,
+        }
+        if further_settings:
+            settings.update(further_settings)
+        return settings
+
+    def wdPath(self):
+        return self._wd
+
+    @property
+    def crate_servers(self):
+        if self.http_url:
+            return [self.http_url]
+        return []
+
+    def setUp(self):
+        self.start()
+
+    def _clean(self):
+        if os.path.exists(self._wd):
+            shutil.rmtree(self._wd)
+
+    def start(self):
+        self._clean()
+        self.process = subprocess.Popen(
+            self.start_cmd, env=self.env, stdout=subprocess.PIPE
+        )
+        returncode = self.process.poll()
+        if returncode is not None:
+            raise SystemError(
+                "Failed to start server rc={0} cmd={1}".format(
+                    returncode, self.start_cmd
+                )
+            )
+
+        if not self.http_url:
+            # try to read http_url from startup logs
+            # this is necessary if no static port is assigned
+            self.http_url = wait_for_http_url(
+                self.process.stdout, verbose=self.verbose
+            )
+
+        self.monitor = OutputMonitor()
+        self.monitor.start(self.process)
+
+        if not self.http_url:
+            self.stop()
+        else:
+            sys.stderr.write("HTTP: {}\n".format(self.http_url))
+            self._wait_for_start()
+            self._wait_for_master()
+            sys.stderr.write("\nCrate instance ready.\n")
+
+    def stop(self):
+        self.conn_pool.clear()
+        if self.process:
+            self.process.terminate()
+            self.process.communicate(timeout=10)
+            self.process.stdout.close()
+            self.process = None
+        self.monitor.stop()
+        self._clean()
+
+    def tearDown(self):
+        self.stop()
+
+    def _wait_for(self, validator):
+        start = time.monotonic()
+
+        line_buf = LineBuffer()
+        self.monitor.consumers.append(line_buf)
+
+        while True:
+            wait_time = time.monotonic() - start
+            try:
+                if validator():
+                    break
+            except Exception as e:
+                self.stop()
+                raise e
+
+            if wait_time > 30:
+                for line in line_buf.lines:
+                    log.error(line)
+                self.stop()
+                raise SystemError("Failed to start Crate instance in time.")
+            sys.stderr.write(".")
+            time.sleep(self.wait_interval)
+
+        self.monitor.consumers.remove(line_buf)
+
+    def _wait_for_start(self):
+        """Wait for instance to be started"""
+
+        # since crate 0.10.0 http may be ready before the cluster
+        # is fully available block here so that early requests
+        # after the layer starts don't result in 503
+        def validator():
+            try:
+                resp = self.conn_pool.request("HEAD", self.http_url)
+                return resp.status == 200
+            except Exception:
+                return False
+
+        self._wait_for(validator)
+
+    def _wait_for_master(self):
+        """Wait for master node"""
+
+        def validator():
+            resp = self.conn_pool.urlopen(
+                "POST",
+                "{server}/_sql".format(server=self.http_url),
+                headers={"Content-Type": "application/json"},
+                body='{"stmt": "select master_node from sys.cluster"}',
+            )
+            data = json.loads(resp.data.decode("utf-8"))
+            return resp.status == 200 and data["rows"][0][0]
+
+        self._wait_for(validator)

crate/testing/util.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8; -*-
+#
+# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
+# license agreements.  See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership.  Crate licenses
+# this file to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# However, if you have executed another commercial license agreement
+# with Crate these terms will supersede the license and you may use the
+# software solely pursuant to the terms of the relevant commercial agreement.
+import unittest
+
+
+class ClientMocked:
+    active_servers = ["http://localhost:4200"]
+
+    def __init__(self):
+        self.response = {}
+        self._server_infos = ("http://localhost:4200", "my server", "2.0.0")
+
+    def sql(self, stmt=None, parameters=None, bulk_parameters=None):
+        return self.response
+
+    def server_infos(self, server):
+        return self._server_infos
+
+    def set_next_response(self, response):
+        self.response = response
+
+    def set_next_server_infos(self, server, server_name, version):
+        self._server_infos = (server, server_name, version)
+
+    def close(self):
+        pass
+
+
+class ParametrizedTestCase(unittest.TestCase):
+    """
+    TestCase classes that want to be parametrized should
+    inherit from this class.
+
+    https://eli.thegreenplace.net/2011/08/02/python-unit-testing-parametrized-test-cases
+    """
+
+    def __init__(self, methodName="runTest", param=None):
+        super(ParametrizedTestCase, self).__init__(methodName)
+        self.param = param
+
+    @staticmethod
+    def parametrize(testcase_klass, param=None):
+        """Create a suite containing all tests taken from the given
+        subclass, passing them the parameter 'param'.
+        """
+        testloader = unittest.TestLoader()
+        testnames = testloader.getTestCaseNames(testcase_klass)
+        suite = unittest.TestSuite()
+        for name in testnames:
+            suite.addTest(testcase_klass(name, param=param))
+        return suite
+
+
+class ExtraAssertions(unittest.TestCase):
+    """
+    Additional assert methods for unittest.
+
+    - https://github.com/python/cpython/issues/71339
+    - https://bugs.python.org/issue14819
+    - https://bugs.python.org/file43047/extra_assertions.patch
+    """
+
+    def assertIsSubclass(self, cls, superclass, msg=None):
+        try:
+            r = issubclass(cls, superclass)
+        except TypeError:
+            if not isinstance(cls, type):
+                self.fail(
+                    self._formatMessage(msg, "%r is not a class" % (cls,))
+                )
+            raise
+        if not r:
+            self.fail(
+                self._formatMessage(
+                    msg, "%r is not a subclass of %r" % (cls, superclass)
+                )
+            )

crate-2.0.0.dev0.dist-info/LICENSE

@@ -0,0 +1,178 @@
+
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS