clickhouse-orm 2.2.2__tar.gz → 3.1.0__tar.gz

clickhouse_orm-3.1.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.4
+Name: clickhouse_orm
+Version: 3.1.0
+Summary: A simple ORM for working with the Clickhouse database. Maintainance fork of infi.clickhouse_orm.
+Author-email: Oliver Margetts <oliver.margetts@gmail.com>
+Description-Content-Type: text/markdown
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Database
+License-File: LICENSE
+Requires-Dist: requests
+Requires-Dist: pytz
+Requires-Dist: docker==7.1.0 ; extra == "dev"
+Requires-Dist: pytest==9.0.2 ; extra == "dev"
+Requires-Dist: ruff==0.14.14 ; extra == "dev"
+Project-URL: Homepage, https://github.com/SuadeLabs/clickhouse_orm
+Project-URL: Repository, https://github.com/SuadeLabs/clickhouse_orm
+Provides-Extra: dev
+
+A fork of [infi.clikchouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) aimed at more frequent maintenance and bugfixes.
+
+[![Tests](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml/badge.svg)](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml)
+![PyPI](https://img.shields.io/pypi/v/clickhouse_orm)
+
+Introduction
+============
+
+This project is simple ORM for working with the [ClickHouse database](https://clickhouse.yandex/).
+It allows you to define model classes whose instances can be written to the database and read from it.
+
+Let's jump right in with a simple example of monitoring CPU usage. First we need to define the model class,
+connect to the database and create a table for the model:
+
+```python
+from clickhouse_orm import Database, Model, DateTimeField, UInt16Field, Float32Field, Memory, F
+
+class CPUStats(Model):
+
+    timestamp = DateTimeField()
+    cpu_id = UInt16Field()
+    cpu_percent = Float32Field()
+
+    engine = Memory()
+
+db = Database('demo')
+db.create_table(CPUStats)
+```
+
+Now we can collect usage statistics per CPU, and write them to the database:
+
+```python
+import psutil, time, datetime
+
+psutil.cpu_percent(percpu=True) # first sample should be discarded
+with db.session():  # use a requests session for efficiency
+    while True:
+        time.sleep(1)
+        stats = psutil.cpu_percent(percpu=True)
+        timestamp = datetime.datetime.now()
+        db.insert([
+            CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent)
+            for cpu_id, cpu_percent in enumerate(stats)
+        ])
+```
+
+Querying the table is easy, using either the query builder or raw SQL:
+
+```python
+# Calculate what percentage of the time CPU 1 was over 95% busy
+queryset = CPUStats.objects_in(db)
+total = queryset.filter(CPUStats.cpu_id == 1).count()
+busy = queryset.filter(CPUStats.cpu_id == 1, CPUStats.cpu_percent > 95).count()
+print('CPU 1 was busy {:.2f}% of the time'.format(busy * 100.0 / total))
+
+# Calculate the average usage per CPU
+for row in queryset.aggregate(CPUStats.cpu_id, average=F.avg(CPUStats.cpu_percent)):
+    print('CPU {row.cpu_id}: {row.average:.2f}%'.format(row=row))
+```
+
+This and other examples can be found in the `examples` folder.
+
+To learn more please visit the [documentation](docs/toc.md).
+

clickhouse_orm-3.1.0/README.md

@@ -0,0 +1,63 @@
+A fork of [infi.clikchouse_orm](https://github.com/Infinidat/infi.clickhouse_orm) aimed at more frequent maintenance and bugfixes.
+
+[![Tests](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml/badge.svg)](https://github.com/SuadeLabs/clickhouse_orm/actions/workflows/python-test.yml)
+![PyPI](https://img.shields.io/pypi/v/clickhouse_orm)
+
+Introduction
+============
+
+This project is simple ORM for working with the [ClickHouse database](https://clickhouse.yandex/).
+It allows you to define model classes whose instances can be written to the database and read from it.
+
+Let's jump right in with a simple example of monitoring CPU usage. First we need to define the model class,
+connect to the database and create a table for the model:
+
+```python
+from clickhouse_orm import Database, Model, DateTimeField, UInt16Field, Float32Field, Memory, F
+
+class CPUStats(Model):
+
+    timestamp = DateTimeField()
+    cpu_id = UInt16Field()
+    cpu_percent = Float32Field()
+
+    engine = Memory()
+
+db = Database('demo')
+db.create_table(CPUStats)
+```
+
+Now we can collect usage statistics per CPU, and write them to the database:
+
+```python
+import psutil, time, datetime
+
+psutil.cpu_percent(percpu=True) # first sample should be discarded
+with db.session():  # use a requests session for efficiency
+    while True:
+        time.sleep(1)
+        stats = psutil.cpu_percent(percpu=True)
+        timestamp = datetime.datetime.now()
+        db.insert([
+            CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent)
+            for cpu_id, cpu_percent in enumerate(stats)
+        ])
+```
+
+Querying the table is easy, using either the query builder or raw SQL:
+
+```python
+# Calculate what percentage of the time CPU 1 was over 95% busy
+queryset = CPUStats.objects_in(db)
+total = queryset.filter(CPUStats.cpu_id == 1).count()
+busy = queryset.filter(CPUStats.cpu_id == 1, CPUStats.cpu_percent > 95).count()
+print('CPU 1 was busy {:.2f}% of the time'.format(busy * 100.0 / total))
+
+# Calculate the average usage per CPU
+for row in queryset.aggregate(CPUStats.cpu_id, average=F.avg(CPUStats.cpu_percent)):
+    print('CPU {row.cpu_id}: {row.average:.2f}%'.format(row=row))
+```
+
+This and other examples can be found in the `examples` folder.
+
+To learn more please visit the [documentation](docs/toc.md).

{clickhouse_orm-2.2.2 → clickhouse_orm-3.1.0}/clickhouse_orm/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from inspect import isclass
 
 from .database import *  # noqa: F401, F403

{clickhouse_orm-2.2.2 → clickhouse_orm-3.1.0}/clickhouse_orm/database.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 import datetime
 import logging
 import re
+from contextlib import contextmanager
 from math import ceil
 from string import Template
 
@@ -13,13 +16,11 @@ from .utils import Page, import_submodules, parse_tsv
 logger = logging.getLogger("clickhouse_orm")
 
 
-class DatabaseException(Exception):
+class DatabaseException(Exception):  # noqa: N818
     """
     Raised when a database operation fails.
     """
 
-    pass
-
 
 class ServerError(DatabaseException):
     """
@@ -35,7 +36,7 @@ class ServerError(DatabaseException):
             # just skip custom init
             # if non-standard message format
             self.message = message
-            super(ServerError, self).__init__(message)
+            super().__init__(message)
 
     ERROR_PATTERNS = (
         # ClickHouse prior to v19.3.3
@@ -55,6 +56,14 @@ class ServerError(DatabaseException):
         """,
             re.VERBOSE | re.DOTALL,
         ),
+        # ClickHouse v21+
+        re.compile(
+            r"""
+            Code:\ (?P<code>\d+).
+            \ (?P<type1>[^ \n]+):\ (?P<msg>.+)
+        """,
+            re.VERBOSE | re.DOTALL,
+        ),
     )
 
     @classmethod
@@ -75,19 +84,21 @@ class ServerError(DatabaseException):
 
     def __str__(self):
         if self.code is not None:
-            return "{} ({})".format(self.message, self.code)
+            return f"{self.message} ({self.code})"
 
 
-class Database(object):
+class Database:
     """
     Database instances connect to a specific ClickHouse database for running queries,
     inserting data and other operations.
     """
 
+    _default_url = "http://localhost:8123/"
+
     def __init__(
         self,
         db_name,
-        db_url="http://localhost:8123/",
+        db_url=None,
         username=None,
         password=None,
         readonly=False,
@@ -95,6 +106,7 @@ class Database(object):
         timeout=60,
         verify_ssl_cert=True,
         log_statements=False,
+        session=None,
     ):
         """
         Initializes a database instance. Unless it's readonly, the database will be
@@ -111,13 +123,14 @@ class Database(object):
         - `log_statements`: when True, all database statements are logged.
         """
         self.db_name = db_name
-        self.db_url = db_url
-        self.readonly = False
+        self.db_url = db_url or self._default_url
+        self.readonly = self.connection_readonly = False
         self.timeout = timeout
-        self.request_session = requests.Session()
-        self.request_session.verify = verify_ssl_cert
-        if username:
-            self.request_session.auth = (username, password or "")
+        self.verify_ssl_cert = verify_ssl_cert
+        self.request_session = None
+        self.__username = username
+        self.__password = password
+
         self.log_statements = log_statements
         self.settings = {}
         self.db_exists = False  # this is required before running _is_existing_database
@@ -137,6 +150,22 @@ class Database(object):
         # Version 19.0 and above support LowCardinality
         self.has_low_cardinality_support = self.server_version >= (19, 0)
 
+    @contextmanager
+    def session(self):
+        """Contextmanager to use a persistent session for requests.
+
+        This can be quicker if making lots of small queries.
+        """
+        with requests.Session() as session:
+            session.verify = self.verify_ssl_cert
+            if self.__username:
+                session.auth = (self.__username, self.__password or "")
+            self.request_session = session
+            try:
+                yield self
+            finally:
+                self.request_session = None
+
     def create_database(self):
         """
         Creates the database on the ClickHouse server if it does not already exist.
@@ -388,7 +417,20 @@ class Database(object):
             if self.log_statements:
                 logger.info(data)
         params = self._build_params(settings)
-        r = self.request_session.post(self.db_url, params=params, data=data, stream=stream, timeout=self.timeout)
+
+        if self.request_session:
+            r = self.request_session.post(self.db_url, params=params, data=data, stream=stream, timeout=self.timeout)
+        else:
+            r = requests.post(
+                self.db_url,
+                params=params,
+                data=data,
+                stream=stream,
+                timeout=self.timeout,
+                verify=self.verify_ssl_cert,
+                auth=(self.__username, self.__password or "") if self.__username else None,
+            )
+
         if r.status_code != 200:
             raise ServerError(r.text)
         return r
@@ -432,7 +474,7 @@ class Database(object):
         except ServerError as e:
             logger.exception("Cannot determine server version (%s), assuming 1.1.0", e)
             ver = "1.1.0"
-        return tuple(int(n) for n in ver.split(".")) if as_tuple else ver
+        return tuple(int(n) for n in ver.split(".") if n.isdigit()) if as_tuple else ver
 
     def _is_existing_database(self):
         r = self._send("SELECT count() FROM system.databases WHERE name = '%s'" % self.db_name)

{clickhouse_orm-2.2.2 → clickhouse_orm-3.1.0}/clickhouse_orm/engines.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 
 from .utils import comma_join, get_subclass_names
@@ -5,7 +7,7 @@ from .utils import comma_join, get_subclass_names
 logger = logging.getLogger("clickhouse_orm")
 
 
-class Engine(object):
+class Engine:
     def create_table_sql(self, db):
         raise NotImplementedError()  # pragma: no cover
 
@@ -44,9 +46,9 @@ class MergeTree(Engine):
             list,
             tuple,
         ), "partition_key must be tuple or list if present"
-        assert (replica_table_path is None) == (
-            replica_name is None
-        ), "both replica_table_path and replica_name must be specified"
+        assert (replica_table_path is None) == (replica_name is None), (
+            "both replica_table_path and replica_name must be specified"
+        )
 
         # These values conflict with each other (old and new syntax of table engines.
         # So let's control only one of them is given.
@@ -145,7 +147,7 @@ class CollapsingMergeTree(MergeTree):
         partition_key=None,
         primary_key=None,
     ):
-        super(CollapsingMergeTree, self).__init__(
+        super().__init__(
             date_col,
             order_by,
             sampling_expr,
@@ -158,7 +160,7 @@ class CollapsingMergeTree(MergeTree):
         self.sign_col = sign_col
 
     def _build_sql_params(self, db):
-        params = super(CollapsingMergeTree, self)._build_sql_params(db)
+        params = super()._build_sql_params(db)
         params.append(self.sign_col)
         return params
 
@@ -176,7 +178,7 @@ class SummingMergeTree(MergeTree):
         partition_key=None,
         primary_key=None,
     ):
-        super(SummingMergeTree, self).__init__(
+        super().__init__(
             date_col,
             order_by,
             sampling_expr,
@@ -190,7 +192,7 @@ class SummingMergeTree(MergeTree):
         self.summing_cols = summing_cols
 
     def _build_sql_params(self, db):
-        params = super(SummingMergeTree, self)._build_sql_params(db)
+        params = super()._build_sql_params(db)
         if self.summing_cols:
             params.append("(%s)" % comma_join(self.summing_cols))
         return params
@@ -209,7 +211,7 @@ class ReplacingMergeTree(MergeTree):
         partition_key=None,
         primary_key=None,
     ):
-        super(ReplacingMergeTree, self).__init__(
+        super().__init__(
             date_col,
             order_by,
             sampling_expr,
@@ -222,7 +224,7 @@ class ReplacingMergeTree(MergeTree):
         self.ver_col = ver_col
 
     def _build_sql_params(self, db):
-        params = super(ReplacingMergeTree, self)._build_sql_params(db)
+        params = super()._build_sql_params(db)
         if self.ver_col:
             params.append(self.ver_col)
         return params
@@ -332,7 +334,7 @@ class Distributed(Engine):
 
     def _build_sql_params(self, db):
         if self.table_name is None:
-            raise ValueError("Cannot create {} engine: specify an underlying table".format(self.__class__.__name__))
+            raise ValueError(f"Cannot create {self.__class__.__name__} engine: specify an underlying table")
 
         params = ["`%s`" % p for p in [self.cluster, db.db_name, self.table_name]]
         if self.sharding_key: