python-openobserve 0.5.1__tar.gz → 0.6.0__tar.gz

{python_openobserve-0.5.1 → python_openobserve-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-openobserve
-Version: 0.5.1
+Version: 0.6.0
 Summary: OpenObserve client for Python
 License-File: LICENSES/CC-BY-4.0.txt
 License-File: LICENSES/CC-BY-SA-4.0.txt
@@ -8,42 +8,42 @@ License-File: LICENSES/GPL-3.0-or-later.txt
 Keywords: openobserve,elasticsearch,logging,unstructured data
 Author: Justin Guese
 Author-email: guese.justin@gmail.com
-Requires-Python: >=3.10,<3.15
+Requires-Python: >=3.11,<3.16
 Classifier: Programming Language :: Python :: 3
 Provides-Extra: ci
 Provides-Extra: docs
 Provides-Extra: pandas
 Provides-Extra: polars
 Provides-Extra: tests
-Requires-Dist: cryptography (>=48.0.1,<48.1.0)
+Requires-Dist: cryptography (>=50.0.0,<50.1.0)
 Requires-Dist: h11 (>=0.16.0,<0.17.0)
 Requires-Dist: httpcore (>=1.0.9,<1.1.0)
 Requires-Dist: httpx (>=0.28,<1.0)
-Requires-Dist: jmespath (==1.0.1) ; extra == "ci"
-Requires-Dist: jmespath (==1.0.1) ; extra == "tests"
-Requires-Dist: memray (==1.19.3) ; extra == "ci"
+Requires-Dist: jmespath (==1.1.0) ; extra == "ci"
+Requires-Dist: jmespath (==1.1.0) ; extra == "tests"
+Requires-Dist: memray (==1.20.0) ; extra == "ci"
 Requires-Dist: mkdocs (==1.6.1) ; extra == "docs"
-Requires-Dist: mkdocs-material (==9.6.15) ; extra == "docs"
-Requires-Dist: mkdocstrings (==0.29.1) ; extra == "docs"
-Requires-Dist: mypy (==1.15.0) ; extra == "ci"
-Requires-Dist: mypy-extensions (==1.0.0) ; extra == "ci"
-Requires-Dist: pandas (==2.2.3) ; extra == "ci"
+Requires-Dist: mkdocs-material (==9.8.0) ; extra == "docs"
+Requires-Dist: mkdocstrings (==1.0.8) ; extra == "docs"
+Requires-Dist: mypy (==2.4.1) ; extra == "ci"
+Requires-Dist: mypy-extensions (==1.2.0) ; extra == "ci"
+Requires-Dist: pandas (==3.1.1) ; extra == "ci"
 Requires-Dist: pandas (>=2.2) ; extra == "pandas"
-Requires-Dist: pandas-stubs (==2.2.2.240807) ; extra == "ci"
-Requires-Dist: polars (==1.30.0) ; extra == "polars"
-Requires-Dist: protobuf (>=6.33.5)
+Requires-Dist: pandas-stubs (==3.1.0.261029) ; extra == "ci"
+Requires-Dist: polars (==2.1.0) ; extra == "polars"
+Requires-Dist: protobuf (>=7.35.1)
 Requires-Dist: pydantic (>=2.10,<3.0)
 Requires-Dist: pyjwt (>=2.12.1,<3.0.0)
-Requires-Dist: pytest (==9.0.3) ; extra == "ci"
-Requires-Dist: pytest (==9.0.3) ; extra == "tests"
-Requires-Dist: pytest-cov (==6.0.0) ; extra == "ci"
-Requires-Dist: pytest-memray (==1.7.0) ; extra == "ci"
-Requires-Dist: python-dotenv (==1.2.2) ; extra == "ci"
-Requires-Dist: python-dotenv (==1.2.2) ; extra == "tests"
+Requires-Dist: pytest (==9.2.0) ; extra == "ci"
+Requires-Dist: pytest (==9.2.0) ; extra == "tests"
+Requires-Dist: pytest-cov (==7.1.0) ; extra == "ci"
+Requires-Dist: pytest-memray (==1.11.0) ; extra == "ci"
+Requires-Dist: python-dotenv (==1.2.4) ; extra == "ci"
+Requires-Dist: python-dotenv (==1.2.4) ; extra == "tests"
 Requires-Dist: python-multipart (>=0.0.22)
 Requires-Dist: requests (>=2.33.0,<3.0.0)
-Requires-Dist: sqlglot (>=26.4.0,<26.5.0)
-Requires-Dist: types-jmespath (==1.0.2.20240106) ; extra == "ci"
+Requires-Dist: sqlglot (>=30.29.0,<30.30.0)
+Requires-Dist: types-jmespath (==1.1.0.20260724) ; extra == "ci"
 Project-URL: Homepage, https://github.com/JustinGuese/python-openobserve
 Project-URL: Repository, https://github.com/JustinGuese/python-openobserve
 Description-Content-Type: text/markdown

{python_openobserve-0.5.1 → python_openobserve-0.6.0}/pyproject.toml

@@ -3,10 +3,10 @@
 
 [project]
 name = "python-openobserve"
-version = "0.5.1"
+version = "0.6.0"
 description = "OpenObserve client for Python"
 readme = "README.md"
-requires-python = ">=3.10,<3.15"
+requires-python = ">=3.11,<3.16"
 license-files = [
     "LICENSES/CC-BY-4.0.txt",
     "LICENSES/CC-BY-SA-4.0.txt",
@@ -19,14 +19,14 @@ classifiers = [
 ]
 
 dependencies = [
-    "cryptography (>=48.0.1,<48.1.0)",
+    "cryptography (>=50.0.0,<50.1.0)",
     "h11~=0.16.0",
     "httpcore~=1.0.9",
     "httpx~=0.28",
-    "protobuf>=6.33.5",
+    "protobuf>=7.35.1",
     "pydantic~=2.10",
     "python-multipart>=0.0.22",
-    "sqlglot~=26.4.0",
+    "sqlglot~=30.29.0",
     "pyjwt (>=2.12.1,<3.0.0)",
     "requests (>=2.33.0,<3.0.0)",
 ]
@@ -40,56 +40,56 @@ python_openobserve = "python_openobserve:openobserve"
 
 [project.optional-dependencies]
 ci = [
-    "jmespath==1.0.1",
-    "memray==1.19.3",
-    "pytest-memray==1.7.0",
-    "mypy==1.15.0",
-    "mypy-extensions==1.0.0",
-    "pandas==2.2.3",
-    "pandas-stubs==2.2.2.240807",
-    "pytest==9.0.3",
-    "pytest-cov==6.0.0",
-    "python-dotenv==1.2.2",
-    "types-jmespath==1.0.2.20240106",
+    "jmespath==1.1.0",
+    "memray==1.20.0",
+    "pytest-memray==1.11.0",
+    "mypy==2.4.1",
+    "mypy-extensions==1.2.0",
+    "pandas==3.1.1",
+    "pandas-stubs==3.1.0.261029",
+    "pytest==9.2.0",
+    "pytest-cov==7.1.0",
+    "python-dotenv==1.2.4",
+    "types-jmespath==1.1.0.20260724",
 ]
 docs = [
     "mkdocs==1.6.1",
-    "mkdocs-material==9.6.15",
-    "mkdocstrings==0.29.1",
+    "mkdocs-material==9.8.0",
+    "mkdocstrings==1.0.8",
 ]
-tests = ["pytest==9.0.3", "jmespath==1.0.1", "python-dotenv==1.2.2"]
+tests = ["pytest==9.2.0", "jmespath==1.1.0", "python-dotenv==1.2.4"]
 pandas = ["pandas>=2.2"]
-polars = ["polars==1.30.0"]
+polars = ["polars==2.1.0"]
 
 [tool.poetry.group.test.dependencies]
-pytest = "9.0.3"
-jmespath = "1.0.1"
-python-dotenv = "1.2.2"
+pytest = "*"
+jmespath = "*"
+python-dotenv = "*"
 
 [tool.poetry.group.docs.dependencies]
-mkdocs = "1.6.1"
-mkdocs-material = "9.6.15"
-mkdocstrings = "0.29.1"
+mkdocs = "*"
+mkdocs-material = "*"
+mkdocstrings = "*"
 
 [tool.poetry.group.ci.dependencies]
-jmespath = "1.0.1"
-memray = "1.19.3"
-pytest-memray = "1.7.0"
-mypy = "1.15.0"
-mypy-extensions = "1.0.0"
-pandas = "2.2.3"
-openpyxl = "3.1.5"
-pandas-stubs = "2.2.2.240807"
-pytest = "9.0.3"
-pytest-cov = "6.0.0"
-python-dotenv = "1.2.2"
-types-jmespath = "1.0.2.20240106"
+jmespath = "*"
+memray = "*"
+pytest-memray = "*"
+mypy = "*"
+mypy-extensions = "*"
+pandas = "*"
+openpyxl = "*"
+pandas-stubs = "*"
+pytest = "*"
+pytest-cov = "*"
+python-dotenv = "*"
+types-jmespath = "*"
 
 [tool.poetry.group.pandas.dependencies]
 pandas = ">2.2"
 
 [tool.poetry.group.polars.dependencies]
-polars = "1.30.0"
+polars = "*"
 
 [build-system]
 requires = ["poetry-core"]

{python_openobserve-0.5.1 → python_openobserve-0.6.0}/python_openobserve/openobserve.py

@@ -15,11 +15,11 @@ import json
 import os
 import sys
 import re
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import List, Dict, Union, Optional, Any, cast
 from pathlib import Path
 
-import httpx
+import httpx  # type: ignore
 import sqlglot  # type: ignore
 
 try:
@@ -33,7 +33,7 @@ except ImportError:
     HAVE_MODULE_PANDAS = False
 
 try:
-    import polars
+    import polars  # type: ignore
 
     HAVE_MODULE_POLARS = True
 except ImportError:
@@ -437,7 +437,7 @@ class OpenObserve:
                 try:
                     # ensure timestamp format
                     if col in ["_timestamp"] + timestamp_columns:
-                        df_res[col] = polars.to_datetime(df_res[col])
+                        df_res = df_res.with_columns(polars.col(col).str.to_datetime())
                 except Exception as err:
                     raise Exception(
                         err,
@@ -446,6 +446,112 @@ class OpenObserve:
                     ) from err
         return df_res
 
+    # pylint: disable=R0914
+    def search2export(
+        self,
+        sql: str,
+        file_prefix: str,
+        *,
+        start_time: datetime = datetime.now(),
+        end_time: datetime = datetime.now(),
+        export_format: str = "json",
+        split_period: str = "D",
+        verbosity: int = 0,
+        timeout: int = 300,
+        timestamp_conversion_auto: bool = False,
+    ) -> bool:
+        """
+        OpenObserve search export function
+        Split options to manage large volume
+
+        Args:
+          sql: input sql query
+          start_time: start of search interval, either datetime, either int/epoch
+          end_time: end of search interval, either datetime, either int/epoch
+          export_format: raw json from OpenObserve (json) or csv
+          split_period: interval per file, H, D, W or M
+          file_prefix: file prefix for export
+          verbosity: how verbose to run from 0/less to 5/more
+          timeout: http timeout
+          timestamp_conversion_auto: try to convert automatically column containing time as name
+          timestamp_columns: convert given columns to timestamp
+        """
+        if not file_prefix:
+            print(f"FATAL! no file_prefix input: {file_prefix}")
+            return False
+        period_start = start_time
+        interval = timedelta(days=1)
+        time_fmt = "%Y-%m-%dT%H:%M"
+        if split_period == "H":
+            interval = timedelta(hours=1)
+        if split_period == "W":
+            interval = timedelta(weeks=1)
+            time_fmt = "%Y-%m-%d"
+        # FIXME! timedelta does not support months input
+        # alternative: relativedelta but extra dependency
+        # if split_period == "M":
+        #     interval = timedelta(months=1)
+        #     time_fmt = "%Y-%m-%d"
+        if split_period not in ("H", "D", "W"):
+            print(f"FATAL! invalid split period input: {split_period}")
+            return False
+        period_end = start_time + interval
+
+        if export_format == "json":
+            while period_end < end_time:
+                file_name = (
+                    f"{file_prefix}--"
+                    f"{period_start.strftime(time_fmt)}--{period_end.strftime(time_fmt)}"
+                    "--export.json"
+                )
+                res_json = self.search(
+                    sql,
+                    start_time=period_start,
+                    end_time=period_end,
+                    query_size=1000000000000,
+                    verbosity=verbosity,
+                    timeout=timeout,
+                    timestamp_conversion_auto=timestamp_conversion_auto,
+                    # leaving conversion to pandas
+                    # timestamp_columns=timestamp_columns,
+                )
+                with open(
+                    file_name,
+                    "w",
+                    encoding="utf-8",
+                ) as f:
+                    json.dump(res_json, f)
+                # loop
+                period_start = period_end
+                period_end = period_start + interval
+        if export_format == "csv":
+            while period_end < end_time:
+                file_name = (
+                    f"{file_prefix}--"
+                    f"{period_start.strftime(time_fmt)}--{period_end.strftime(time_fmt)}"
+                    "--export.csv"
+                )
+                df_res = self.search2df(
+                    sql,
+                    start_time=period_start,
+                    end_time=period_end,
+                    query_size=1000000000000,
+                    verbosity=verbosity,
+                    timeout=timeout,
+                    timestamp_conversion_auto=timestamp_conversion_auto,
+                    # leaving conversion to pandas
+                    # timestamp_columns=timestamp_columns,
+                )
+                df_res.to_csv(
+                    file_name,
+                    index=False,
+                )
+                # loop
+                period_start = period_end
+                period_end = period_start + interval
+
+        return True
+
     # pylint: disable=too-many-branches,too-many-locals
     def export_objects_split(
         self,
@@ -603,7 +709,6 @@ class OpenObserve:
                 for name, df in data.items():
                     df.to_excel(f"{file_path}{name}.xlsx")
         else:  # default json
-
             if split is True and flat is False:
                 # split json
                 data = {
@@ -627,9 +732,7 @@ class OpenObserve:
                 sys.exit(1)
             else:
                 data = {
-                    name: self.list_objects(
-                        api_path, verbosity=verbosity
-                    )  # type: ignore[misc]
+                    name: self.list_objects(api_path, verbosity=verbosity)  # type: ignore[misc]
                     for name, api_path in object_types.items()
                 }