clickhouse-orm 3.0.1__py2.py3-none-any.whl

clickhouse_orm/system_models.py

@@ -0,0 +1,170 @@
+"""
+This file contains system readonly models that can be got from the database
+https://clickhouse.tech/docs/en/system_tables/
+"""
+
+from __future__ import annotations
+
+from .database import Database
+from .fields import DateTimeField, StringField, UInt8Field, UInt32Field, UInt64Field
+from .models import Model
+from .utils import comma_join
+
+
+class SystemPart(Model):
+    """
+    Contains information about parts of a table in the MergeTree family.
+    This model operates only fields, described in the reference. Other fields are ignored.
+    https://clickhouse.tech/docs/en/system_tables/system.parts/
+    """
+
+    OPERATIONS = frozenset({"DETACH", "DROP", "ATTACH", "FREEZE", "FETCH"})
+
+    _readonly = True
+    _system = True
+
+    database = StringField()  # Name of the database where the table that this part belongs to is located.
+    table = StringField()  # Name of the table that this part belongs to.
+    engine = StringField()  # Name of the table engine, without parameters.
+    partition = StringField()  # Name of the partition, in the format YYYYMM.
+    name = StringField()  # Name of the part.
+
+    # This field is present in the docs (https://clickhouse.tech/docs/en/single/index.html#system-parts),
+    # but is absent in ClickHouse (in version 1.1.54245)
+    # replicated = UInt8Field()  # Whether the part belongs to replicated data.
+
+    # Whether the part is used in a table, or is no longer needed and will be deleted soon.
+    # Inactive parts remain after merging.
+    active = UInt8Field()
+
+    # Number of marks - multiply by the index granularity (usually 8192)
+    # to get the approximate number of rows in the part.
+    marks = UInt64Field()
+
+    bytes = UInt64Field()  # Number of bytes when compressed.
+
+    # Time the directory with the part was modified. Usually corresponds to the part's creation time.
+    modification_time = DateTimeField()
+    remove_time = DateTimeField()  # For inactive parts only - the time when the part became inactive.
+
+    # The number of places where the part is used. A value greater than 2 indicates
+    # that this part participates in queries or merges.
+    refcount = UInt32Field()
+
+    @classmethod
+    def table_name(cls):
+        return "parts"
+
+    """
+    Next methods return SQL for some operations, which can be done with partitions
+    https://clickhouse.tech/docs/en/query_language/queries/#manipulations-with-partitions-and-parts
+    """
+
+    def _partition_operation_sql(self, operation, settings=None, from_part=None):
+        """
+        Performs some operation over partition
+
+        - `db`: Database object to execute operation on
+        - `operation`: Operation to execute from SystemPart.OPERATIONS set
+        - `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+        Returns: Operation execution result
+        """
+        operation = operation.upper()
+        assert operation in self.OPERATIONS, "operation must be in [%s]" % comma_join(self.OPERATIONS)
+
+        sql = "ALTER TABLE `%s`.`%s` %s PARTITION %s" % (self._database.db_name, self.table, operation, self.partition)
+        if from_part is not None:
+            sql += " FROM %s" % from_part
+        self._database.raw(sql, settings=settings, stream=False)
+
+    def detach(self, settings=None):
+        """
+        Move a partition to the 'detached' directory and forget it.
+
+        - `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+        Returns: SQL Query
+        """
+        return self._partition_operation_sql("DETACH", settings=settings)
+
+    def drop(self, settings=None):
+        """
+        Delete a partition
+
+        - `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+        Returns: SQL Query
+        """
+        return self._partition_operation_sql("DROP", settings=settings)
+
+    def attach(self, settings=None):
+        """
+         Add a new part or partition from the 'detached' directory to the table.
+
+        - `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+        Returns: SQL Query
+        """
+        return self._partition_operation_sql("ATTACH", settings=settings)
+
+    def freeze(self, settings=None):
+        """
+        Create a backup of a partition.
+
+        - `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+        Returns: SQL Query
+        """
+        return self._partition_operation_sql("FREEZE", settings=settings)
+
+    def fetch(self, zookeeper_path, settings=None):
+        """
+        Download a partition from another server.
+
+        - `zookeeper_path`: Path in zookeeper to fetch from
+        - `settings`: Settings for executing request to ClickHouse over db.raw() method
+
+        Returns: SQL Query
+        """
+        return self._partition_operation_sql("FETCH", settings=settings, from_part=zookeeper_path)
+
+    @classmethod
+    def get(cls, database, conditions=""):
+        """
+        Get all data from system.parts table
+
+        - `database`: A database object to fetch data from.
+        - `conditions`: WHERE clause conditions. Database condition is added automatically
+
+        Returns: A list of SystemPart objects
+        """
+        assert isinstance(database, Database), "database must be database.Database class instance"
+        assert isinstance(conditions, str), "conditions must be a string"
+        if conditions:
+            conditions += " AND"
+        field_names = ",".join(cls.fields())
+        return database.select(
+            "SELECT %s FROM `system`.%s WHERE %s database='%s'"
+            % (field_names, cls.table_name(), conditions, database.db_name),
+            model_class=cls,
+        )
+
+    @classmethod
+    def get_active(cls, database, conditions=""):
+        """
+        Gets active data from system.parts table
+
+        - `database`: A database object to fetch data from.
+        - `conditions`: WHERE clause conditions. Database and active conditions are added automatically
+
+        Returns: A list of SystemPart objects
+        """
+        if conditions:
+            conditions += " AND "
+        conditions += "active"
+        return SystemPart.get(database, conditions=conditions)
+
+
+# Expose only relevant classes in import *
+__all__ = [c.__name__ for c in [SystemPart]]

clickhouse_orm/utils.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import codecs
+import importlib
+import pkgutil
+import re
+from datetime import date, datetime, timedelta, tzinfo
+from inspect import isclass
+from typing import TYPE_CHECKING, NamedTuple
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from types import ModuleType
+    from typing import Any
+
+
+class Page(NamedTuple):
+    """A simple data structure for paginated results."""
+
+    objects: list[Any]
+    number_of_objects: int
+    pages_total: int
+    number: int
+    page_size: int
+
+
+def escape(value: str, quote: bool = True) -> str:
+    """
+    If the value is a string, escapes any special characters and optionally
+    surrounds it with single quotes. If the value is not a string (e.g. a number),
+    converts it to one.
+    """
+    value = codecs.escape_encode(value.encode("utf-8"))[0].decode("utf-8")
+    if quote:
+        value = "'" + value + "'"
+
+    return value
+
+
+def unescape(value: str) -> str | None:
+    if value == "\\N":
+        return None
+    return codecs.escape_decode(value)[0].decode("utf-8")
+
+
+def string_or_func(obj):
+    return obj.to_sql() if hasattr(obj, "to_sql") else obj
+
+
+def arg_to_sql(arg: Any) -> str:
+    """
+    Converts a function argument to SQL string according to its type.
+    Supports functions, model fields, strings, dates, datetimes, timedeltas, booleans,
+    None, numbers, timezones, arrays/iterables.
+    """
+    from clickhouse_orm import DateTimeField, F, Field, QuerySet, StringField
+
+    if isinstance(arg, F):
+        return arg.to_sql()
+    if isinstance(arg, Field):
+        return "`%s`" % arg
+    if isinstance(arg, str):
+        return StringField().to_db_string(arg)
+    if isinstance(arg, datetime):
+        return "toDateTime(%s)" % DateTimeField().to_db_string(arg)
+    if isinstance(arg, date):
+        return "toDate('%s')" % arg.isoformat()
+    if isinstance(arg, timedelta):
+        return "toIntervalSecond(%d)" % int(arg.total_seconds())
+    if isinstance(arg, bool):
+        return str(int(arg))
+    if isinstance(arg, tzinfo):
+        return StringField().to_db_string(arg.tzname(None))
+    if arg is None:
+        return "NULL"
+    if isinstance(arg, QuerySet):
+        return "(%s)" % arg
+    if isinstance(arg, tuple):
+        return "(" + comma_join(arg_to_sql(x) for x in arg) + ")"
+    if is_iterable(arg):
+        return "[" + comma_join(arg_to_sql(x) for x in arg) + "]"
+    return str(arg)
+
+
+def parse_tsv(line: bytes | str) -> list[str]:
+    if isinstance(line, bytes):
+        line = line.decode()
+    if line and line[-1] == "\n":
+        line = line[:-1]
+    return [unescape(value) for value in line.split("\t")]
+
+
+def parse_array(array_string: str) -> list[Any]:
+    """
+    Parse an array or tuple string as returned by clickhouse. For example:
+        "['hello', 'world']" ==> ["hello", "world"]
+        "(1,2,3)"            ==> [1, 2, 3]
+    """
+    # Sanity check
+    if len(array_string) < 2 or array_string[0] not in "[(" or array_string[-1] not in "])":
+        raise ValueError('Invalid array string: "%s"' % array_string)
+    # Drop opening brace
+    array_string = array_string[1:]
+    # Go over the string, lopping off each value at the beginning until nothing is left
+    values = []
+    while True:
+        if array_string in "])":
+            # End of array
+            return values
+        elif array_string[0] in ", ":
+            # In between values
+            array_string = array_string[1:]
+        elif array_string[0] == "'":
+            # Start of quoted value, find its end
+            match = re.search(r"[^\\]'", array_string)
+            if match is None:
+                raise ValueError('Missing closing quote: "%s"' % array_string)
+            values.append(array_string[1 : match.start() + 1])
+            array_string = array_string[match.end() :]
+        else:
+            # Start of non-quoted value, find its end
+            match = re.search(r",|\]", array_string)
+            values.append(array_string[0 : match.start()])
+            array_string = array_string[match.end() - 1 :]
+
+
+def import_submodules(package_name: str) -> dict[str, ModuleType]:
+    """
+    Import all submodules of a module.
+    """
+    package = importlib.import_module(package_name)
+    return {
+        name: importlib.import_module(package_name + "." + name)
+        for _, name, _ in pkgutil.iter_modules(package.__path__)
+    }
+
+
+def comma_join(items: Iterable[str]) -> str:
+    """
+    Joins an iterable of strings with commas.
+    """
+    return ", ".join(items)
+
+
+def is_iterable(obj: Any) -> bool:
+    """
+    Checks if the given object is iterable.
+    """
+    try:
+        iter(obj)
+        return True
+    except TypeError:
+        return False
+
+
+def get_subclass_names(locals: dict[str, Any], base_class: type):
+    return [c.__name__ for c in locals.values() if isclass(c) and issubclass(c, base_class)]
+
+
+class NoValue:
+    """
+    A sentinel for fields with an expression for a default value,
+    that were not assigned a value yet.
+    """
+
+    def __repr__(self):
+        return "NO_VALUE"
+
+    def __copy__(self):
+        return self
+
+    def __deepcopy__(self, memo):
+        return self
+
+
+NO_VALUE = NoValue()

clickhouse_orm-3.0.1.dist-info/METADATA

@@ -0,0 +1,90 @@
+Metadata-Version: 2.4
+Name: clickhouse_orm
+Version: 3.0.1
+Summary: A simple ORM for working with the Clickhouse database. Maintainance fork of infi.clickhouse_orm.
+Author-email: Oliver Margetts <oliver.margetts@gmail.com>
+Description-Content-Type: text/markdown
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Database
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: pytz
+Requires-Dist: docker==7.1.0 ; extra == "dev"
+Requires-Dist: pytest==9.0.2 ; extra == "dev"
+Requires-Dist: ruff==0.14.14 ; extra == "dev"
+Project-URL: Homepage, https://github.com/SuadeLabs/clickhouse_orm
+Project-URL: Repository, https://github.com/SuadeLabs/clickhouse_orm
+Provides-Extra: dev
+
+A fork of [infi.clikchouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) aimed at more frequent maintenance and bugfixes.
+
+[![Tests](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml/badge.svg)](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml)
+![PyPI](https://img.shields.io/pypi/v/clickhouse_orm)
+
+Introduction
+============
+
+This project is simple ORM for working with the [ClickHouse database](https://clickhouse.yandex/).
+It allows you to define model classes whose instances can be written to the database and read from it.
+
+Let's jump right in with a simple example of monitoring CPU usage. First we need to define the model class,
+connect to the database and create a table for the model:
+
+```python
+from clickhouse_orm import Database, Model, DateTimeField, UInt16Field, Float32Field, Memory, F
+
+class CPUStats(Model):
+
+    timestamp = DateTimeField()
+    cpu_id = UInt16Field()
+    cpu_percent = Float32Field()
+
+    engine = Memory()
+
+db = Database('demo')
+db.create_table(CPUStats)
+```
+
+Now we can collect usage statistics per CPU, and write them to the database:
+
+```python
+import psutil, time, datetime
+
+psutil.cpu_percent(percpu=True) # first sample should be discarded
+while True:
+    time.sleep(1)
+    stats = psutil.cpu_percent(percpu=True)
+    timestamp = datetime.datetime.now()
+    db.insert([
+        CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent)
+        for cpu_id, cpu_percent in enumerate(stats)
+    ])
+```
+
+Querying the table is easy, using either the query builder or raw SQL:
+
+```python
+# Calculate what percentage of the time CPU 1 was over 95% busy
+queryset = CPUStats.objects_in(db)
+total = queryset.filter(CPUStats.cpu_id == 1).count()
+busy = queryset.filter(CPUStats.cpu_id == 1, CPUStats.cpu_percent > 95).count()
+print('CPU 1 was busy {:.2f}% of the time'.format(busy * 100.0 / total))
+
+# Calculate the average usage per CPU
+for row in queryset.aggregate(CPUStats.cpu_id, average=F.avg(CPUStats.cpu_percent)):
+    print('CPU {row.cpu_id}: {row.average:.2f}%'.format(row=row))
+```
+
+This and other examples can be found in the `examples` folder.
+
+To learn more please visit the [documentation](docs/toc.md).
+

clickhouse_orm-3.0.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+clickhouse_orm/__init__.py,sha256=ksYW9scTnD6VVCj8Rzv7B41VkvHDn-JWpyIh1Tjserc,478
+clickhouse_orm/database.py,sha256=v4I2NKxkwF6Bm2hJAM6vCg-n9ss10THA9AIoxuHnU1g,17828
+clickhouse_orm/engines.py,sha256=AV80AmLRM3hlEXj3o6GPjjKDPk8YyRdTEns9SZCppA0,11166
+clickhouse_orm/fields.py,sha256=Wfm3NkctRhnaw2bc9KqlncV04FCvBEb_74ky9ZF2aq4,24039
+clickhouse_orm/funcs.py,sha256=8S1kdwCjIJ45imvA2XkxHjDhGOYlNg4aS1vMU2CQ224,42108
+clickhouse_orm/migrations.py,sha256=_AmD6uQQv3CEm0wc7JWawMcYsKQRId5nDFXr4IT-J6Y,10470
+clickhouse_orm/models.py,sha256=pUkLBClnfyXPg5xuHTpobNhFqHtKuHlXTNwWv5PnV_U,23088
+clickhouse_orm/query.py,sha256=3-v1Z5G1xO1c8ywVhmZiz0tpAvBmH19xzeyxVpxc6uU,24103
+clickhouse_orm/system_models.py,sha256=pU-Pvq69GH6QD4iKIrSRxw3Q6n-qlgQzGLqpYwAF12U,6447
+clickhouse_orm/utils.py,sha256=2fEmTrBSDNpPzbbnUOZeCQ2NX5httkOKrJKzgdTezH4,5179
+clickhouse_orm-3.0.1.dist-info/licenses/LICENSE,sha256=MP0KDNu_tFQJrr35YkSFnwwp2F8p2a7Q1Mi2lOs8FLw,1503
+clickhouse_orm-3.0.1.dist-info/WHEEL,sha256=Dyt6SBfaasWElUrURkknVFAZDHSTwxg3PaTza7RSbkY,100
+clickhouse_orm-3.0.1.dist-info/METADATA,sha256=ttRTc2omK1LNWMgA4FDcj02yuKom3wujU4A0dFsxkz8,3400
+clickhouse_orm-3.0.1.dist-info/RECORD,,

clickhouse_orm-3.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: flit 3.12.0
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any

clickhouse_orm-3.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2021 Suade Labs
+Copyright (c) 2017 INFINIDAT
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.