smart-datastore-local 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smart_datastore_local-0.1.1/PKG-INFO +24 -0
- smart_datastore_local-0.1.1/README.md +1 -0
- smart_datastore_local-0.1.1/pyproject.toml +22 -0
- smart_datastore_local-0.1.1/setup.cfg +4 -0
- smart_datastore_local-0.1.1/setup.py +58 -0
- smart_datastore_local-0.1.1/smart_datastore_local/src/__init__.py +0 -0
- smart_datastore_local-0.1.1/smart_datastore_local/src/smart_datastore.py +172 -0
- smart_datastore_local-0.1.1/smart_datastore_local.egg-info/PKG-INFO +24 -0
- smart_datastore_local-0.1.1/smart_datastore_local.egg-info/SOURCES.txt +10 -0
- smart_datastore_local-0.1.1/smart_datastore_local.egg-info/dependency_links.txt +1 -0
- smart_datastore_local-0.1.1/smart_datastore_local.egg-info/requires.txt +4 -0
- smart_datastore_local-0.1.1/smart_datastore_local.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: smart-datastore-local
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: PyPI Package for Circles smart-datastore-local Python
|
|
5
|
+
Home-page: https://github.com/circles-zone/smart-datastore-local-python-package
|
|
6
|
+
Author: Circles
|
|
7
|
+
Author-email: info@circlez.ai
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: logger-local
|
|
12
|
+
Requires-Dist: database-sql-local>=0.1.2
|
|
13
|
+
Requires-Dist: python-sdk-remote
|
|
14
|
+
Requires-Dist: database-infrastructure-local>=0.1.3
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: author-email
|
|
17
|
+
Dynamic: classifier
|
|
18
|
+
Dynamic: description
|
|
19
|
+
Dynamic: description-content-type
|
|
20
|
+
Dynamic: home-page
|
|
21
|
+
Dynamic: requires-dist
|
|
22
|
+
Dynamic: summary
|
|
23
|
+
|
|
24
|
+
PyPI Package for Circles smart-datastore-local Python
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
TODO Please update the README of the package
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# This file should be in the future instead of setup.py
|
|
2
|
+
# https://python-poetry.org/docs/pyproject
|
|
3
|
+
# https://stackoverflow.com/questions/78048223/adding-folder-with-data-with-pyproject-toml
|
|
4
|
+
|
|
5
|
+
# This file is mandatory for the `poetry version patch`
|
|
6
|
+
|
|
7
|
+
# It seems we need to copy this file also to serverless-com repo, as required by dialog-workflow-python-package
|
|
8
|
+
|
|
9
|
+
[build-system]
|
|
10
|
+
requires = ["setuptools>=61.0"]
|
|
11
|
+
build-backend = "setuptools.build_meta"
|
|
12
|
+
|
|
13
|
+
[tool.poetry]
|
|
14
|
+
# TODO: Please update the name, similar to storage-local (suffix -local)
|
|
15
|
+
name = "<project-name>"
|
|
16
|
+
# I believe we are still using the version from setup.py and not from here until Potery will work
|
|
17
|
+
version = "0.0.1" # https://pypi.org/project/<project-name> i.e. https://pypi.org/project/storage-local/
|
|
18
|
+
description = "<project-name> Python Package"
|
|
19
|
+
readme = "README.md"
|
|
20
|
+
authors = [
|
|
21
|
+
"Circlez.ai <info@circlez.ai>",
|
|
22
|
+
]
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# serverless.com repo "python -m build" works without setup.py
|
|
2
|
+
# Should update package_dir if not using our recommended directory structure
|
|
3
|
+
|
|
4
|
+
import setuptools
|
|
5
|
+
|
|
6
|
+
# TODO: Change the PACKAGE_NAME to the package's name - Either xxx-local or
|
|
7
|
+
# xxx-remote (without the -python-package suffix). Only lowercase, no underlines.
|
|
8
|
+
# Used by pypa/gh-action-pypi-publish
|
|
9
|
+
# Package Name should be identical to the inner directory name
|
|
10
|
+
# Changing the package name here, will cause changing the package directory
|
|
11
|
+
# name as well
|
|
12
|
+
# PACKAGE_NAME should be singular if handling only one instance
|
|
13
|
+
# PACKAGE_NAME should not include the word "main"
|
|
14
|
+
|
|
15
|
+
# e.g.: queue-local, without python-package suffix
|
|
16
|
+
PACKAGE_NAME = "smart-datastore-local"
|
|
17
|
+
|
|
18
|
+
package_dir = PACKAGE_NAME.replace("-", "_")
|
|
19
|
+
# If we need backward-compatible:
|
|
20
|
+
# old_package_dir = "old_package_name"
|
|
21
|
+
|
|
22
|
+
setuptools.setup(
|
|
23
|
+
name=PACKAGE_NAME,
|
|
24
|
+
# Increase this number every time you make a change you
|
|
25
|
+
# want to publish. After 0.0.9 switch to 0.0.10 and not 0.1.0
|
|
26
|
+
version='0.1.1', # https://pypi.org/project/smart-datastore-local/
|
|
27
|
+
author="Circles",
|
|
28
|
+
author_email="info@circlez.ai",
|
|
29
|
+
description=f"PyPI Package for Circles {PACKAGE_NAME} Python",
|
|
30
|
+
long_description=f"PyPI Package for Circles {PACKAGE_NAME} Python",
|
|
31
|
+
long_description_content_type='text/markdown',
|
|
32
|
+
url=f"https://github.com/circles-zone/{PACKAGE_NAME}-python-package",
|
|
33
|
+
packages=[package_dir],
|
|
34
|
+
# packages=[package_dir, old_package_dir],
|
|
35
|
+
|
|
36
|
+
package_dir={package_dir: f'{package_dir}/src'},
|
|
37
|
+
# TODO Unfortunately in event-main-local-restapi there are no
|
|
38
|
+
# repo-directory and no package directory (flat directory structure)
|
|
39
|
+
# package_dir={package_dir: f'src'},
|
|
40
|
+
|
|
41
|
+
# package_dir={package_dir: f'{package_dir}/src', old_package_dir: f'{package_dir}/src'},
|
|
42
|
+
package_data={package_dir: ['*.py']},
|
|
43
|
+
classifiers=[
|
|
44
|
+
"Programming Language :: Python :: 3",
|
|
45
|
+
# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license
|
|
46
|
+
# "License :: BSD-3-Clause",
|
|
47
|
+
"Operating System :: OS Independent",
|
|
48
|
+
],
|
|
49
|
+
# TODO: Update which packages to include with this package in production
|
|
50
|
+
# (dependencies) - Not for development/testing
|
|
51
|
+
install_requires=[
|
|
52
|
+
'logger-local', # TODO: in -remote package please use logger-remote instead.
|
|
53
|
+
'database-sql-local>=0.1.2', # TODO: In -remote package please delete this line.
|
|
54
|
+
'python-sdk-remote',
|
|
55
|
+
# 0.0.35 includes generic_crud_abstract.py GenericCrudAbstract
|
|
56
|
+
"database-infrastructure-local>=0.1.3"
|
|
57
|
+
]
|
|
58
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
from database_infrastructure_local.constants import LoggerComponentEnum
|
|
2
|
+
from database_infrastructure_local.generic_crud_abstract \
|
|
3
|
+
import GenericCrudAbstract
|
|
4
|
+
from database_infrastructure_local.constants import DEFAULT_SQL_SELECT_LIMIT
|
|
5
|
+
from database_mysql_local.generic_crud_mysql import GenericCrudMysql
|
|
6
|
+
|
|
7
|
+
import sys
|
|
8
|
+
import os
|
|
9
|
+
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__),
|
|
10
|
+
'..', 'src')))
|
|
11
|
+
|
|
12
|
+
# from logger_local import LoggerLocal # TODO
|
|
13
|
+
from logger_local.LoggerLocal import Logger
|
|
14
|
+
|
|
15
|
+
# TODO Update the python-package-template
|
|
16
|
+
DEVELOPER_EMAIL_ADDRESS = 'tal@circlez.ai'
|
|
17
|
+
SMART_DATASTORE_LOCAL_PYTHON_PACKAGE_COMPONENT_ID = -1 # TODO
|
|
18
|
+
SMART_DATASTORE_LOCAL_PYTHON_PACKAGE_COMPONENT_NAME = 'database_mysql_local\\smart_datastore' # TODO
|
|
19
|
+
SMART_DATASTORE_LOCAL_CODE_LOGGER_OBJECT = {
|
|
20
|
+
'component_id': SMART_DATASTORE_LOCAL_PYTHON_PACKAGE_COMPONENT_ID,
|
|
21
|
+
'component_name': SMART_DATASTORE_LOCAL_PYTHON_PACKAGE_COMPONENT_NAME,
|
|
22
|
+
'component_category': LoggerComponentEnum.ComponentCategory.Code.value,
|
|
23
|
+
'developer_email': DEVELOPER_EMAIL_ADDRESS, # TODO Remove
|
|
24
|
+
'developer_email_address': DEVELOPER_EMAIL_ADDRESS
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
# logger = LoggerLocal()
|
|
28
|
+
logger = Logger.create_logger(object=SMART_DATASTORE_LOCAL_CODE_LOGGER_OBJECT)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# TODO Can we make it a static method and avoid the default_table_name
|
|
32
|
+
# parameter
|
|
33
|
+
def is_redis_table(table_name: str, default_table_name: str) -> bool:
|
|
34
|
+
if table_name or default_table_name in ("test_mysql_table",
|
|
35
|
+
"test_mysql_ml_table",
|
|
36
|
+
"test_gender_table"):
|
|
37
|
+
return True
|
|
38
|
+
else:
|
|
39
|
+
return False
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# TODO Can we make it a method, rename it and remove the default_table_name
|
|
43
|
+
# parameter
|
|
44
|
+
def generic_method(*, schema_name: str = None, table_name: str = None,
|
|
45
|
+
data_dict: dict = None,
|
|
46
|
+
ignore_duplicate: bool = False,
|
|
47
|
+
commit_changes: bool = True,
|
|
48
|
+
# Added as this is function and not a method
|
|
49
|
+
default_table_name: str = None
|
|
50
|
+
) -> int:
|
|
51
|
+
|
|
52
|
+
# TODO Run in parallel
|
|
53
|
+
# self.generic_crud_mysql.insert
|
|
54
|
+
result_mysql = None
|
|
55
|
+
try:
|
|
56
|
+
# result_mysql = generic_crud_mysql.insert(schema_name=schema_name,
|
|
57
|
+
# table_name=table_name,
|
|
58
|
+
# data_dict=data_dict,
|
|
59
|
+
# ignore_duplicate=ignore_duplicate,
|
|
60
|
+
# commit_changes=commit_changes)
|
|
61
|
+
pass
|
|
62
|
+
except Exception as e:
|
|
63
|
+
print(f"Exception: {e}")
|
|
64
|
+
logger.error(f"Exception: {e}")
|
|
65
|
+
|
|
66
|
+
if is_redis_table(table_name=table_name,
|
|
67
|
+
default_table_name=default_table_name):
|
|
68
|
+
try:
|
|
69
|
+
# result_redis = generic_crud_redis.insert(
|
|
70
|
+
# schema_name=schema_name,
|
|
71
|
+
# table_name=table_name,
|
|
72
|
+
# data_dict=data_dict,
|
|
73
|
+
# ignore_duplicate=ignore_duplicate,
|
|
74
|
+
# commit_changes=commit_changes)
|
|
75
|
+
pass
|
|
76
|
+
except Exception as e:
|
|
77
|
+
print(f"Exception: {e}")
|
|
78
|
+
logger.info(f"Exception: {e}")
|
|
79
|
+
else:
|
|
80
|
+
logger.info(f"table_name {table_name} not in REDIS yet. Consider adding it in SmartDatastore. Later it will come from the database")
|
|
81
|
+
return result_mysql
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# Generic Datastore
|
|
85
|
+
class SmartDatastore (GenericCrudAbstract):
|
|
86
|
+
def __init__(self, default_schema_name: str = None,
|
|
87
|
+
default_table_name: str = None,
|
|
88
|
+
default_view_table_name: str = None,
|
|
89
|
+
default_view_with_deleted_and_test_data: str = None,
|
|
90
|
+
default_column_name: str = None,
|
|
91
|
+
is_test_data: bool = False):
|
|
92
|
+
super().__init__()
|
|
93
|
+
self.generic_crud_mysql = GenericCrudMysql(
|
|
94
|
+
default_schema_name=default_schema_name,
|
|
95
|
+
default_table_name=default_table_name,
|
|
96
|
+
default_view_table_name=default_view_table_name,
|
|
97
|
+
default_view_with_deleted_and_test_data=default_view_with_deleted_and_test_data,
|
|
98
|
+
default_column_name=default_column_name,
|
|
99
|
+
is_test_data=is_test_data)
|
|
100
|
+
self.generic_crud_redis = GenericCrudMysql(
|
|
101
|
+
default_schema_name=default_schema_name,
|
|
102
|
+
default_table_name=default_table_name,
|
|
103
|
+
default_view_table_name=default_view_table_name,
|
|
104
|
+
default_view_with_deleted_and_test_data=default_view_with_deleted_and_test_data,
|
|
105
|
+
default_column_name=default_column_name,
|
|
106
|
+
is_test_data=is_test_data)
|
|
107
|
+
|
|
108
|
+
def insert(self, *, schema_name: str = None, table_name: str = None,
|
|
109
|
+
data_dict: dict = None,
|
|
110
|
+
ignore_duplicate: bool = False, commit_changes: bool = True
|
|
111
|
+
) -> int:
|
|
112
|
+
result_redis = None
|
|
113
|
+
# TODO Run in parallel
|
|
114
|
+
# self.generic_crud_mysql.insert
|
|
115
|
+
try:
|
|
116
|
+
logger.info(f"inserting data_dict {data_dict} in MySQL")
|
|
117
|
+
result_mysql = self.generic_crud_mysql.insert(
|
|
118
|
+
schema_name=schema_name,
|
|
119
|
+
table_name=table_name,
|
|
120
|
+
data_dict=data_dict,
|
|
121
|
+
ignore_duplicate=ignore_duplicate,
|
|
122
|
+
commit_changes=commit_changes)
|
|
123
|
+
except Exception as e:
|
|
124
|
+
print(f"Exception: {e}")
|
|
125
|
+
logger.error(f"Exception: {e}")
|
|
126
|
+
|
|
127
|
+
logger.info(f"SmartDatastore: checking if table_name {table_name} is on REDIS")
|
|
128
|
+
if is_redis_table(table_name=table_name,
|
|
129
|
+
default_table_name=self.generic_crud_mysql.default_table_name):
|
|
130
|
+
logger.info(f"SmartDatastore: table_name {table_name} is on REDIS")
|
|
131
|
+
try:
|
|
132
|
+
result_redis = self.generic_crud_redis.insert(
|
|
133
|
+
schema_name=schema_name,
|
|
134
|
+
table_name=table_name,
|
|
135
|
+
data_dict=data_dict,
|
|
136
|
+
ignore_duplicate=ignore_duplicate,
|
|
137
|
+
commit_changes=commit_changes)
|
|
138
|
+
logger.info(f"SmartDatastore: inserted data_dict {data_dict} in REDIS result_redis={result_redis}")
|
|
139
|
+
except Exception as e:
|
|
140
|
+
print(f"Exception: {e}")
|
|
141
|
+
logger.info(f"Exception: {e}")
|
|
142
|
+
else:
|
|
143
|
+
logger.info(f"SmartDatastore: table_name {table_name} not in REDIS yet. Consider adding it in SmartDatastore. Later it will come from the database")
|
|
144
|
+
|
|
145
|
+
# TODO Compare result_mysql and result_redis
|
|
146
|
+
# if (result_mysql is not None and result_redis is not None
|
|
147
|
+
# and result_mysql != result_redis):
|
|
148
|
+
# logger.error(f"SmartDatastore: result_mysql {result_mysql} != \
|
|
149
|
+
# result_redis {result_redis}")
|
|
150
|
+
# raise Exception(f"SmartDatastore: result_mysql {result_mysql} != \
|
|
151
|
+
# result_redis {result_redis}")
|
|
152
|
+
return result_mysql
|
|
153
|
+
|
|
154
|
+
def select_multi_tuple_by_where(self, *, schema_name: str = None,
|
|
155
|
+
view_table_name: str = None,
|
|
156
|
+
select_clause_value: str = None,
|
|
157
|
+
where: str = None, params: tuple = None,
|
|
158
|
+
distinct: bool = False,
|
|
159
|
+
limit: int = DEFAULT_SQL_SELECT_LIMIT,
|
|
160
|
+
order_by: str = None) -> list:
|
|
161
|
+
logger.info(f"SmartDatastore: select_multi_tuple_by_where: {select_clause_value}")
|
|
162
|
+
print(f"SmartDatastore: select_multi_tuple_by_where: {select_clause_value}", flush=True)
|
|
163
|
+
result_mysql = self.generic_crud_mysql.select_multi_tuple_by_where(
|
|
164
|
+
schema_name=schema_name, view_table_name=view_table_name,
|
|
165
|
+
select_clause_value=select_clause_value,
|
|
166
|
+
where=where, params=params, distinct=distinct, limit=limit,
|
|
167
|
+
order_by=order_by)
|
|
168
|
+
return result_mysql
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
# for method_name in GenericCrudAbstract.__abstractmethods__:
|
|
172
|
+
# setattr(SmartDatastore, method_name, generic_method(method_name))
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: smart-datastore-local
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: PyPI Package for Circles smart-datastore-local Python
|
|
5
|
+
Home-page: https://github.com/circles-zone/smart-datastore-local-python-package
|
|
6
|
+
Author: Circles
|
|
7
|
+
Author-email: info@circlez.ai
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Requires-Dist: logger-local
|
|
12
|
+
Requires-Dist: database-sql-local>=0.1.2
|
|
13
|
+
Requires-Dist: python-sdk-remote
|
|
14
|
+
Requires-Dist: database-infrastructure-local>=0.1.3
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: author-email
|
|
17
|
+
Dynamic: classifier
|
|
18
|
+
Dynamic: description
|
|
19
|
+
Dynamic: description-content-type
|
|
20
|
+
Dynamic: home-page
|
|
21
|
+
Dynamic: requires-dist
|
|
22
|
+
Dynamic: summary
|
|
23
|
+
|
|
24
|
+
PyPI Package for Circles smart-datastore-local Python
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
setup.py
|
|
4
|
+
smart_datastore_local.egg-info/PKG-INFO
|
|
5
|
+
smart_datastore_local.egg-info/SOURCES.txt
|
|
6
|
+
smart_datastore_local.egg-info/dependency_links.txt
|
|
7
|
+
smart_datastore_local.egg-info/requires.txt
|
|
8
|
+
smart_datastore_local.egg-info/top_level.txt
|
|
9
|
+
smart_datastore_local/src/__init__.py
|
|
10
|
+
smart_datastore_local/src/smart_datastore.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
smart_datastore_local
|