datacanva 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datacanva-1.0.0/.gitignore +2 -0
- datacanva-1.0.0/.vscode/settings.json +4 -0
- datacanva-1.0.0/LICENSE +21 -0
- datacanva-1.0.0/PKG-INFO +33 -0
- datacanva-1.0.0/README.md +18 -0
- datacanva-1.0.0/deploy.sh +8 -0
- datacanva-1.0.0/pyproject.toml +24 -0
- datacanva-1.0.0/src/datacanva/__init__.py +52 -0
- datacanva-1.0.0/src/datacanva/api/__init__.py +84 -0
- datacanva-1.0.0/src/datacanva/api/content_hub.py +23 -0
- datacanva-1.0.0/src/datacanva/api/warehouse.py +57 -0
- datacanva-1.0.0/src/datacanva/service/__init__.py +35 -0
- datacanva-1.0.0/src/datacanva/service/content_hub.py +32 -0
- datacanva-1.0.0/src/datacanva/service/warehouse.py +161 -0
- datacanva-1.0.0/src/datacanva/type/criteria.py +53 -0
- datacanva-1.0.0/src/datacanva/type/expression_tree.py +30 -0
- datacanva-1.0.0/src/datacanva/type/warehouse.py +29 -0
datacanva-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Lively Impact Technology Limited
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
datacanva-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: datacanva
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Python module which connects to the DataCanva REST API
|
|
5
|
+
Author-email: Lively Impact Technology Limited <info@livelyimpact.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Classifier: Operating System :: OS Independent
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Requires-Dist: pandas>=2.3.3
|
|
12
|
+
Requires-Dist: pyjson5>=2.0.0
|
|
13
|
+
Requires-Dist: requests>=2.28.1
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
## Overview
|
|
17
|
+
|
|
18
|
+
`datacanva` is a Python package that provides a simple and efficient way to
|
|
19
|
+
interact with the DataCanva web app's REST API.
|
|
20
|
+
With this package, you can easily integrate DataCanva's functionalities
|
|
21
|
+
into your Python workflows for data visualization, analysis, and more.
|
|
22
|
+
|
|
23
|
+
## Installation
|
|
24
|
+
|
|
25
|
+
Install the package directly from PyPI:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install datacanva
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## License
|
|
32
|
+
|
|
33
|
+
This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
## Overview
|
|
2
|
+
|
|
3
|
+
`datacanva` is a Python package that provides a simple and efficient way to
|
|
4
|
+
interact with the DataCanva web app's REST API.
|
|
5
|
+
With this package, you can easily integrate DataCanva's functionalities
|
|
6
|
+
into your Python workflows for data visualization, analysis, and more.
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
Install the package directly from PyPI:
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install datacanva
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## License
|
|
17
|
+
|
|
18
|
+
This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "datacanva"
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
authors = [
|
|
5
|
+
{ name="Lively Impact Technology Limited", email="info@livelyimpact.com" },
|
|
6
|
+
]
|
|
7
|
+
description = "Python module which connects to the DataCanva REST API"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.9"
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Operating System :: OS Independent",
|
|
13
|
+
]
|
|
14
|
+
license = "MIT"
|
|
15
|
+
license-files = ["LICEN[CS]E*"]
|
|
16
|
+
dependencies = [
|
|
17
|
+
"requests>=2.28.1",
|
|
18
|
+
"pyjson5>=2.0.0",
|
|
19
|
+
"pandas>=2.3.3"
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[build-system]
|
|
23
|
+
requires = ["hatchling >= 1.26"]
|
|
24
|
+
build-backend = "hatchling.build"
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import pyjson5
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from logging import Logger, getLogger
|
|
5
|
+
|
|
6
|
+
from .api import Api
|
|
7
|
+
from .api.warehouse import WarehouseApi
|
|
8
|
+
from .api.content_hub import ContentHubApi
|
|
9
|
+
|
|
10
|
+
from .service.warehouse import WarehouseService
|
|
11
|
+
from .service.content_hub import ContentHubService
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def load_config(file_path: str):
|
|
15
|
+
locations = [
|
|
16
|
+
Path(file_path if file_path is not None else ".datacanva"),
|
|
17
|
+
Path.cwd() / ".datacanva",
|
|
18
|
+
Path.home() / ".datacanva",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
for location in locations:
|
|
22
|
+
try:
|
|
23
|
+
if os.path.exists(location):
|
|
24
|
+
with open(location, "r") as file:
|
|
25
|
+
return pyjson5.decode(file.read())
|
|
26
|
+
except Exception as e:
|
|
27
|
+
print(e)
|
|
28
|
+
|
|
29
|
+
return {}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class DataCanvaClient:
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
base_url: str = None,
|
|
37
|
+
api_key: str = None,
|
|
38
|
+
config_file: str = None,
|
|
39
|
+
logger: Logger = None,
|
|
40
|
+
):
|
|
41
|
+
config = load_config(config_file)
|
|
42
|
+
|
|
43
|
+
if not base_url:
|
|
44
|
+
base_url = config.get("base_url")
|
|
45
|
+
|
|
46
|
+
if not api_key:
|
|
47
|
+
api_key = config.get("api_key")
|
|
48
|
+
|
|
49
|
+
self.logger = logger if logger is not None else getLogger("DataCanvaClient")
|
|
50
|
+
self.api = Api(base_url=base_url, api_key=api_key)
|
|
51
|
+
self.warehouse = WarehouseService(self.logger, WarehouseApi(self.api))
|
|
52
|
+
self.content_hub = ContentHubService(self.logger, ContentHubApi(self.api))
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Api:
|
|
5
|
+
|
|
6
|
+
def __init__(self, base_url: str, api_key: str):
|
|
7
|
+
if not base_url:
|
|
8
|
+
raise Exception("Required argument 'base_url' is not provided.")
|
|
9
|
+
|
|
10
|
+
if not api_key:
|
|
11
|
+
raise Exception("Required argument 'api_key' is not provided.")
|
|
12
|
+
|
|
13
|
+
self.base_url = base_url
|
|
14
|
+
self.api_key = api_key
|
|
15
|
+
|
|
16
|
+
self.session = requests.Session()
|
|
17
|
+
self.session.headers.update({"X-Api-Key": api_key})
|
|
18
|
+
|
|
19
|
+
def get(self, path: str, **args):
|
|
20
|
+
return self._handle_response(
|
|
21
|
+
self.session.get(self._with_base_url(path), **args)
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
def put(self, path: str, **args):
|
|
25
|
+
return self._handle_response(
|
|
26
|
+
self.session.put(self._with_base_url(path), **args)
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
def post(self, path: str, **args):
|
|
30
|
+
return self._handle_response(
|
|
31
|
+
self.session.post(self._with_base_url(path), **args)
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
def delete(self, path: str):
|
|
35
|
+
return self._handle_response(self.session.delete(self._with_base_url(path)))
|
|
36
|
+
|
|
37
|
+
def _with_base_url(self, path: str):
|
|
38
|
+
return f"{self.base_url}/{path.lstrip('/')}"
|
|
39
|
+
|
|
40
|
+
def _handle_response(self, res: requests.Response):
|
|
41
|
+
body: dict[str, any] | None = None
|
|
42
|
+
try:
|
|
43
|
+
if res.text:
|
|
44
|
+
body = res.json()
|
|
45
|
+
except:
|
|
46
|
+
raise Exception(
|
|
47
|
+
f"Response body is not a valid JSON value ({res.status_code}): {res.text}"
|
|
48
|
+
)
|
|
49
|
+
self._raise_error(res, body)
|
|
50
|
+
return body
|
|
51
|
+
|
|
52
|
+
def _raise_error(self, res: requests.Response, json: dict[str, any] = None):
|
|
53
|
+
if res.status_code != 200:
|
|
54
|
+
body = json if json is not None else res.json()
|
|
55
|
+
raise Exception(f"({res.status_code}) {body['message']}")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class CrudApi:
|
|
60
|
+
|
|
61
|
+
def __init__(self, api: Api, base_path: str):
|
|
62
|
+
self.api = api
|
|
63
|
+
self.base_path = base_path
|
|
64
|
+
|
|
65
|
+
def get_list(self, params={}):
|
|
66
|
+
return self.api.get(self._with_base_path(), params=params)
|
|
67
|
+
|
|
68
|
+
def get(self, id: int):
|
|
69
|
+
return self.api.get(self._with_base_path(f"{id}"))
|
|
70
|
+
|
|
71
|
+
def post(self, item):
|
|
72
|
+
return self.api.post(self._with_base_path(), json=item)
|
|
73
|
+
|
|
74
|
+
def put(self, id: int, item):
|
|
75
|
+
return self.api.put(self._with_base_path(f"{id}"), item)
|
|
76
|
+
|
|
77
|
+
def delete(self, id: int):
|
|
78
|
+
return self.api.delete(self._with_base_path(f"{id}"))
|
|
79
|
+
|
|
80
|
+
def _with_base_path(self, path: str = ""):
|
|
81
|
+
return f"{self.base_path}/{path.lstrip('/')}".rstrip("/")
|
|
82
|
+
|
|
83
|
+
def _with_base_url(self, path: str = ""):
|
|
84
|
+
return self.api._with_base_url(self._with_base_path(path))
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from . import Api, CrudApi
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ContentHubApi(CrudApi):
|
|
5
|
+
|
|
6
|
+
def __init__(self, api: Api):
|
|
7
|
+
super().__init__(api, "/content-hub")
|
|
8
|
+
|
|
9
|
+
def get_content(self, id: int, path: str):
|
|
10
|
+
res = self.api.session.get(
|
|
11
|
+
self._with_base_url(f"/{id}/content/{path.strip('/')}"), stream=True
|
|
12
|
+
)
|
|
13
|
+
self.api._raise_error(res)
|
|
14
|
+
return res
|
|
15
|
+
|
|
16
|
+
def put_content(self, id: int, path: str, file):
|
|
17
|
+
return self.api.put(
|
|
18
|
+
self._with_base_path(f"/{id}/content/{path.strip('/')}"),
|
|
19
|
+
files={"file": file},
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
def delete_content(self, id: int, path: str):
|
|
23
|
+
return self.api.delete(self._with_base_path(f"/{id}/content/{path.strip('/')}"))
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from ..type.criteria import *
|
|
2
|
+
from . import Api, CrudApi
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class WriteDataRequest(TypedDict):
|
|
6
|
+
writeMode: WriteMode
|
|
7
|
+
schema: Schema
|
|
8
|
+
data: list[dict]
|
|
9
|
+
generateSchema: bool
|
|
10
|
+
skipValidation: bool
|
|
11
|
+
upsertBy: list[str]
|
|
12
|
+
patchMode: bool
|
|
13
|
+
deleteMode: bool
|
|
14
|
+
description: str
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class GetDataRequest(TypedDict):
|
|
18
|
+
acquireReadLock: bool = True
|
|
19
|
+
page: int = 0
|
|
20
|
+
size: int = 1000
|
|
21
|
+
sort: list[tuple[str, str]]
|
|
22
|
+
criteria: Criteria
|
|
23
|
+
description: str
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class WarehouseApi(CrudApi):
|
|
27
|
+
|
|
28
|
+
def __init__(self, api: Api):
|
|
29
|
+
super().__init__(api, "/warehouse/entity")
|
|
30
|
+
|
|
31
|
+
def write_data(self, id: int, request: WriteDataRequest):
|
|
32
|
+
return self.api.post(self._with_base_path(f"/{id}/data"), json=request)
|
|
33
|
+
|
|
34
|
+
def get_data(self, id: int, request: GetDataRequest):
|
|
35
|
+
return self.api.post(
|
|
36
|
+
self._with_base_path(f"/{id}/data/get"),
|
|
37
|
+
json={
|
|
38
|
+
**request,
|
|
39
|
+
"criteria": (
|
|
40
|
+
request["criteria"].__dict__()
|
|
41
|
+
if request["criteria"] is not None
|
|
42
|
+
else None
|
|
43
|
+
),
|
|
44
|
+
},
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def clear_data(self, id: int):
|
|
48
|
+
return self.api.delete(self._with_base_path(f"/{id}/data"))
|
|
49
|
+
|
|
50
|
+
def generate_schema(self, id: int):
|
|
51
|
+
return self.api.post(self._with_base_path(f"/{id}/schema/generate"))
|
|
52
|
+
|
|
53
|
+
def validate_schema(self, id: int):
|
|
54
|
+
return self.api.post(self._with_base_path(f"/{id}/schema/validate"))
|
|
55
|
+
|
|
56
|
+
def get_lock_queue(self, id: int):
|
|
57
|
+
return self.api.get(self._with_base_path(f"/{id}/lock"))
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from typing import TypeVar, Generic
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from logging import Logger
|
|
4
|
+
|
|
5
|
+
from ..api import CrudApi
|
|
6
|
+
|
|
7
|
+
ItemType = TypeVar("ItemType")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Service(Generic[ItemType]):
|
|
11
|
+
|
|
12
|
+
def __init__(
|
|
13
|
+
self, logger: Logger, crud_api: CrudApi, item: Callable[..., ItemType]
|
|
14
|
+
):
|
|
15
|
+
self.logger = logger
|
|
16
|
+
self.crud_api = crud_api
|
|
17
|
+
self.item = item
|
|
18
|
+
|
|
19
|
+
def get_by_name(self, creator_username: str, entity_name: str):
|
|
20
|
+
try:
|
|
21
|
+
res = self.crud_api.get_list(
|
|
22
|
+
{
|
|
23
|
+
"name.eq": entity_name,
|
|
24
|
+
"creatorUsername.eq": creator_username,
|
|
25
|
+
}
|
|
26
|
+
)
|
|
27
|
+
items = res["items"]
|
|
28
|
+
if len(items) == 0:
|
|
29
|
+
self.logger.error(f"Item not found: {creator_username}/{entity_name}")
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
return self.item(self, items[0])
|
|
33
|
+
except Exception as e:
|
|
34
|
+
self.logger.error(f"Failed to get item by name: {e}")
|
|
35
|
+
return None
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from logging import Logger
|
|
2
|
+
|
|
3
|
+
from ..api.content_hub import ContentHubApi
|
|
4
|
+
from . import Service
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ContentHub:
|
|
8
|
+
|
|
9
|
+
def __init__(self, service: "ContentHubService", data: dict):
|
|
10
|
+
self.service = service
|
|
11
|
+
self.id = data["id"]
|
|
12
|
+
self.name = data["name"]
|
|
13
|
+
self.creator = data["creatorUsername"]
|
|
14
|
+
|
|
15
|
+
def __str__(self):
|
|
16
|
+
return f"ContentHub/{self.creator}/{self.name}({self.id})"
|
|
17
|
+
|
|
18
|
+
def get_content(self, path: str):
|
|
19
|
+
return self.service.api.get_content(self.id, path=path)
|
|
20
|
+
|
|
21
|
+
def put_content(self, path: str, file):
|
|
22
|
+
return self.service.api.put_content(self.id, path=path, file=file)
|
|
23
|
+
|
|
24
|
+
def delete_content(self, path: str):
|
|
25
|
+
return self.service.api.delete_content(self.id, path=path)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ContentHubService(Service[ContentHub]):
|
|
29
|
+
|
|
30
|
+
def __init__(self, logger: Logger, api: ContentHubApi):
|
|
31
|
+
super().__init__(logger, api, ContentHub)
|
|
32
|
+
self.api = api
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
from logging import Logger
|
|
2
|
+
|
|
3
|
+
import pandas
|
|
4
|
+
from pandas._typing import DtypeObj
|
|
5
|
+
|
|
6
|
+
from ..api.warehouse import *
|
|
7
|
+
from . import Service
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def map_dtype(dtype: DtypeObj):
|
|
11
|
+
if pandas.api.types.is_integer_dtype(dtype):
|
|
12
|
+
return DataType.INT
|
|
13
|
+
elif pandas.api.types.is_float_dtype(dtype):
|
|
14
|
+
return DataType.FLOAT
|
|
15
|
+
elif pandas.api.types.is_string_dtype(dtype):
|
|
16
|
+
return DataType.STRING
|
|
17
|
+
elif pandas.api.types.is_bool_dtype(dtype):
|
|
18
|
+
return DataType.BOOLEAN
|
|
19
|
+
elif pandas.api.types.is_datetime64_any_dtype(dtype):
|
|
20
|
+
return DataType.TIMESTAMP
|
|
21
|
+
else:
|
|
22
|
+
raise ValueError(f"Unsupported type: {dtype}")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def dataframe_to_schema(dataframe: pandas.DataFrame):
|
|
26
|
+
fields: list[SchemaField] = []
|
|
27
|
+
for column in dataframe.columns:
|
|
28
|
+
dtype = dataframe[column].dtype
|
|
29
|
+
fields.append(
|
|
30
|
+
SchemaField(
|
|
31
|
+
name=column,
|
|
32
|
+
dataType=map_dtype(dtype),
|
|
33
|
+
)
|
|
34
|
+
)
|
|
35
|
+
return Schema(fields=fields)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Entity:
|
|
39
|
+
|
|
40
|
+
def __init__(self, service: "WarehouseService", data: dict):
|
|
41
|
+
self.service = service
|
|
42
|
+
self.id = int(data["id"])
|
|
43
|
+
self.name = str(data["name"])
|
|
44
|
+
self.creator = str(data["creatorUsername"])
|
|
45
|
+
|
|
46
|
+
def __str__(self):
|
|
47
|
+
return f"Warehouse/{self.creator}/{self.name}({self.id})"
|
|
48
|
+
|
|
49
|
+
def get_schema(self):
|
|
50
|
+
try:
|
|
51
|
+
res = self.service.crud_api.get(self.id)
|
|
52
|
+
return Schema(res["schema"])
|
|
53
|
+
except Exception as e:
|
|
54
|
+
self.service.logger.error(f"Failed to get entity (id: {self.id}): {e}")
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
def get_items_count(self, criteria: Criteria = None):
|
|
58
|
+
try:
|
|
59
|
+
req = GetDataRequest(
|
|
60
|
+
acquireReadLock=True,
|
|
61
|
+
page=0,
|
|
62
|
+
size=1,
|
|
63
|
+
criteria=criteria,
|
|
64
|
+
)
|
|
65
|
+
res = self.service.api.get_data(self.id, req)
|
|
66
|
+
return res["totalItems"]
|
|
67
|
+
except Exception as e:
|
|
68
|
+
self.service.logger.error(f"Failed to get entity items count: {e}")
|
|
69
|
+
return 0
|
|
70
|
+
|
|
71
|
+
def get_data(
|
|
72
|
+
self,
|
|
73
|
+
page=0,
|
|
74
|
+
size=1000,
|
|
75
|
+
sort: list[str] = [],
|
|
76
|
+
criteria: Criteria = None,
|
|
77
|
+
drop_columns: list[str] = ["_id", "_tags"],
|
|
78
|
+
):
|
|
79
|
+
try:
|
|
80
|
+
req = GetDataRequest(
|
|
81
|
+
acquireReadLock=True,
|
|
82
|
+
page=page,
|
|
83
|
+
size=size,
|
|
84
|
+
criteria=criteria,
|
|
85
|
+
sort=sort,
|
|
86
|
+
description="DataLab User Get Data",
|
|
87
|
+
)
|
|
88
|
+
res = self.service.api.get_data(self.id, req)
|
|
89
|
+
dataframe = pandas.DataFrame(res["items"]).drop(
|
|
90
|
+
columns=drop_columns, errors="ignore"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
# convert datetime columns
|
|
94
|
+
schema = self.get_schema()
|
|
95
|
+
if schema is not None:
|
|
96
|
+
for field in schema["fields"]:
|
|
97
|
+
if field["dataType"] == "TIMESTAMP":
|
|
98
|
+
dataframe[field["name"]] = pandas.to_datetime(
|
|
99
|
+
dataframe[field["name"]]
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
return dataframe
|
|
103
|
+
except Exception as e:
|
|
104
|
+
self.service.logger.error(f"Failed to get entity data: {e}")
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
def write_data(
|
|
108
|
+
self,
|
|
109
|
+
dataframe: pandas.DataFrame,
|
|
110
|
+
write_mode: WriteMode = WriteMode.UPSERT,
|
|
111
|
+
upsert_by: list[str] = None,
|
|
112
|
+
):
|
|
113
|
+
try:
|
|
114
|
+
schema = dataframe_to_schema(dataframe)
|
|
115
|
+
|
|
116
|
+
output_dataframe = dataframe.copy()
|
|
117
|
+
for field in schema["fields"]:
|
|
118
|
+
if field["dataType"] == "TIMESTAMP":
|
|
119
|
+
output_dataframe[field["name"]] = (
|
|
120
|
+
output_dataframe[field["name"]].astype(int) // 10**6
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
req = WriteDataRequest(
|
|
124
|
+
schema=schema,
|
|
125
|
+
writeMode=write_mode,
|
|
126
|
+
upsertBy=upsert_by,
|
|
127
|
+
data=output_dataframe.to_dict(orient="records"),
|
|
128
|
+
description="DataLab User Write Data",
|
|
129
|
+
)
|
|
130
|
+
self.service.api.write_data(self.id, req)
|
|
131
|
+
except Exception as e:
|
|
132
|
+
self.service.logger.error(f"Failed to write entity data: {e}")
|
|
133
|
+
|
|
134
|
+
def delete_data(
|
|
135
|
+
self,
|
|
136
|
+
dataframe: pandas.DataFrame,
|
|
137
|
+
delete_by: list[str],
|
|
138
|
+
):
|
|
139
|
+
try:
|
|
140
|
+
output_dataframe = dataframe.copy()
|
|
141
|
+
output_dataframe = output_dataframe.drop(
|
|
142
|
+
columns=output_dataframe.select_dtypes(include=["datetime64", "datetimetz"]).columns
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
req = WriteDataRequest(
|
|
146
|
+
writeMode=WriteMode.UPSERT,
|
|
147
|
+
deleteMode=True,
|
|
148
|
+
upsertBy=delete_by,
|
|
149
|
+
data=output_dataframe.to_dict(orient="records"),
|
|
150
|
+
description="DataLab User Delete Data",
|
|
151
|
+
)
|
|
152
|
+
self.service.api.write_data(self.id, req)
|
|
153
|
+
except Exception as e:
|
|
154
|
+
self.service.logger.error(f"Failed to delete entity data: {e}")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class WarehouseService(Service[Entity]):
|
|
158
|
+
|
|
159
|
+
def __init__(self, logger: Logger, api: WarehouseApi):
|
|
160
|
+
super().__init__(logger, api, Entity)
|
|
161
|
+
self.api = api
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from .expression_tree import ExpressionTreeNode
|
|
2
|
+
from ..type.warehouse import *
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class ConditionalOperator(str, Enum):
|
|
6
|
+
EQUAL_TO = "EQUAL_TO"
|
|
7
|
+
NOT_EQUAL_TO = "NOT_EQUAL_TO"
|
|
8
|
+
GREATER_THAN_OR_EQUAL_TO = "GREATER_THAN_OR_EQUAL_TO"
|
|
9
|
+
GREATER_THAN = "GREATER_THAN"
|
|
10
|
+
LESS_THAN_OR_EQUAL_TO = "LESS_THAN_OR_EQUAL_TO"
|
|
11
|
+
LESS_THAN = "LESS_THAN"
|
|
12
|
+
IN = "IN"
|
|
13
|
+
NOT_IN = "NOT_IN"
|
|
14
|
+
MATCH_PATTERN = "MATCH_PATTERN"
|
|
15
|
+
STARTS_WITH = "STARTS_WITH"
|
|
16
|
+
ENDS_WITH = "ENDS_WITH"
|
|
17
|
+
CONTAINS = "CONTAINS"
|
|
18
|
+
IS_NULL = "IS_NULL"
|
|
19
|
+
IS_NOT_NULL = "IS_NOT_NULL"
|
|
20
|
+
WITHIN_RECENT_DAYS = "WITHIN_RECENT_DAYS"
|
|
21
|
+
WITHIN_RECENT_HOURS = "WITHIN_RECENT_HOURS"
|
|
22
|
+
WITHIN_RECENT_MINUTES = "WITHIN_RECENT_MINUTES"
|
|
23
|
+
WITHIN_TODAY = "WITHIN_TODAY"
|
|
24
|
+
WITHIN_THIS_WEEK = "WITHIN_THIS_WEEK"
|
|
25
|
+
WITHIN_THIS_MONTH = "WITHIN_THIS_MONTH"
|
|
26
|
+
WITHIN_THIS_QUARTER = "WITHIN_THIS_QUARTER"
|
|
27
|
+
WITHIN_THIS_YEAR = "WITHIN_THIS_YEAR"
|
|
28
|
+
WITHIN_YESTERDAY = "WITHIN_YESTERDAY"
|
|
29
|
+
WITHIN_LAST_WEEK = "WITHIN_LAST_WEEK"
|
|
30
|
+
WITHIN_LAST_MONTH = "WITHIN_LAST_MONTH"
|
|
31
|
+
WITHIN_LAST_QUARTER = "WITHIN_LAST_QUARTER"
|
|
32
|
+
WITHIN_LAST_YEAR = "WITHIN_LAST_YEAR"
|
|
33
|
+
HAS_ALL_TAGS = "HAS_ALL_TAGS"
|
|
34
|
+
HAS_ANY_TAGS = "HAS_ANY_TAGS"
|
|
35
|
+
TAGS_COUNT_GREATER_THAN = "TAGS_COUNT_GREATER_THAN"
|
|
36
|
+
TAGS_COUNT_LESS_THAN = "TAGS_COUNT_LESS_THAN"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class LogicalOperator(str, Enum):
|
|
40
|
+
AND = "AND"
|
|
41
|
+
OR = "OR"
|
|
42
|
+
NOT = "NOT"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Criterion(TypedDict):
|
|
46
|
+
field: str
|
|
47
|
+
value: str
|
|
48
|
+
valueType: DataType
|
|
49
|
+
listValue: list[str]
|
|
50
|
+
operator: ConditionalOperator
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
Criteria = ExpressionTreeNode[Criterion, LogicalOperator]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from typing import Generic, TypeVar, List, Optional
|
|
2
|
+
|
|
3
|
+
# Define type variables for ExpressionType and OperatorType
|
|
4
|
+
ExpressionType = TypeVar("ExpressionType")
|
|
5
|
+
OperatorType = TypeVar("OperatorType")
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ExpressionTreeNode(Generic[ExpressionType, OperatorType]):
|
|
9
|
+
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
# OperatorType is None if the node is a leaf
|
|
13
|
+
operator: Optional[OperatorType] = None,
|
|
14
|
+
# ExpressionType is None if the node is not a leaf
|
|
15
|
+
expression: Optional[ExpressionType] = None,
|
|
16
|
+
# Recursive definition
|
|
17
|
+
children: Optional[
|
|
18
|
+
List["ExpressionTreeNode[ExpressionType, OperatorType]"]
|
|
19
|
+
] = None,
|
|
20
|
+
):
|
|
21
|
+
self.operator = operator
|
|
22
|
+
self.expression = expression
|
|
23
|
+
self.children = children
|
|
24
|
+
|
|
25
|
+
def __dict__(self):
|
|
26
|
+
return {
|
|
27
|
+
"operator": self.operator,
|
|
28
|
+
"expression": self.expression,
|
|
29
|
+
"children": self.children,
|
|
30
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from typing import TypedDict
|
|
2
|
+
from enum import Enum
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class WriteMode(str, Enum):
|
|
6
|
+
APPEND = "APPEND"
|
|
7
|
+
UPSERT = "UPSERT"
|
|
8
|
+
OVERWRITE = "OVERWRITE"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DataType(str, Enum):
|
|
12
|
+
INT = "INT"
|
|
13
|
+
FLOAT = "FLOAT"
|
|
14
|
+
STRING = "STRING"
|
|
15
|
+
BOOLEAN = "BOOLEAN"
|
|
16
|
+
TIMESTAMP = "TIMESTAMP"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SchemaField(TypedDict):
|
|
20
|
+
name: str
|
|
21
|
+
dataType: str
|
|
22
|
+
nonNullable: bool
|
|
23
|
+
primaryKey: bool
|
|
24
|
+
unique: bool
|
|
25
|
+
displayFormat: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Schema(TypedDict):
|
|
29
|
+
fields: list[SchemaField]
|