rdxz2-utill 0.0.4__tar.gz → 0.0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of rdxz2-utill might be problematic. Click here for more details.
- {rdxz2_utill-0.0.4/src/rdxz2_utill.egg-info → rdxz2_utill-0.0.6}/PKG-INFO +15 -15
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/pyproject.toml +15 -15
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6/src/rdxz2_utill.egg-info}/PKG-INFO +15 -15
- rdxz2_utill-0.0.6/src/rdxz2_utill.egg-info/requires.txt +19 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_bq.py +2 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_pg.py +22 -20
- rdxz2_utill-0.0.4/src/rdxz2_utill.egg-info/requires.txt +0 -19
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/LICENSE +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/README.md +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/setup.cfg +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/rdxz2_utill.egg-info/SOURCES.txt +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/rdxz2_utill.egg-info/dependency_links.txt +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/rdxz2_utill.egg-info/entry_points.txt +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/rdxz2_utill.egg-info/top_level.txt +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/__init__.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/cmd/__init__.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/cmd/_bq.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/cmd/_conf.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/cmd/_enc.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/cmd/_main.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/cmd/_pg.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/cmd/utill.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_compare.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_const.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_csv.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_datetime.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_dict.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_encryption.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_env.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_file.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_gcs.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_input.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_json.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_mb.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_queue.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_string.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_style.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_tunnel.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/my_xlsx.py +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/templates/mb.json +0 -0
- {rdxz2_utill-0.0.4 → rdxz2_utill-0.0.6}/src/utill/templates/pg.json +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rdxz2-utill
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.6
|
|
4
4
|
Summary: Your daily Python utility
|
|
5
5
|
Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -25,29 +25,29 @@ License: MIT License
|
|
|
25
25
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
26
|
SOFTWARE.
|
|
27
27
|
Classifier: Programming Language :: Python :: 3
|
|
28
|
-
Classifier: Programming Language :: Python :: 3.
|
|
28
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
29
29
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
30
30
|
Classifier: License :: OSI Approved :: MIT License
|
|
31
31
|
Classifier: Operating System :: OS Independent
|
|
32
|
-
Requires-Python: >=3.
|
|
32
|
+
Requires-Python: >=3.10
|
|
33
33
|
Description-Content-Type: text/markdown
|
|
34
34
|
License-File: LICENSE
|
|
35
|
-
Requires-Dist: click
|
|
36
|
-
Requires-Dist: duckdb
|
|
37
|
-
Requires-Dist: humanize
|
|
38
|
-
Requires-Dist: loguru
|
|
39
|
-
Requires-Dist: paramiko
|
|
40
|
-
Requires-Dist: pydantic-settings
|
|
41
|
-
Requires-Dist: pydantic
|
|
42
|
-
Requires-Dist: requests
|
|
35
|
+
Requires-Dist: click
|
|
36
|
+
Requires-Dist: duckdb
|
|
37
|
+
Requires-Dist: humanize
|
|
38
|
+
Requires-Dist: loguru
|
|
39
|
+
Requires-Dist: paramiko
|
|
40
|
+
Requires-Dist: pydantic-settings
|
|
41
|
+
Requires-Dist: pydantic
|
|
42
|
+
Requires-Dist: requests
|
|
43
43
|
Requires-Dist: sshtunnel==0.4.0
|
|
44
44
|
Provides-Extra: google-cloud
|
|
45
|
-
Requires-Dist: google-cloud-bigquery
|
|
46
|
-
Requires-Dist: google-cloud-storage
|
|
45
|
+
Requires-Dist: google-cloud-bigquery; extra == "google-cloud"
|
|
46
|
+
Requires-Dist: google-cloud-storage; extra == "google-cloud"
|
|
47
47
|
Provides-Extra: postgresql
|
|
48
|
-
Requires-Dist: psycopg
|
|
48
|
+
Requires-Dist: psycopg; extra == "postgresql"
|
|
49
49
|
Provides-Extra: pdf
|
|
50
|
-
Requires-Dist: PyPDF2
|
|
50
|
+
Requires-Dist: PyPDF2; extra == "pdf"
|
|
51
51
|
Dynamic: license-file
|
|
52
52
|
|
|
53
53
|
# Using this library
|
|
@@ -4,30 +4,30 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "rdxz2-utill"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.6"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Richard Dharmawan", email="richard.dharmawan@gmail.com" },
|
|
10
10
|
]
|
|
11
11
|
description = "Your daily Python utility"
|
|
12
12
|
readme = "README.md"
|
|
13
|
-
requires-python = ">=3.
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
14
|
license = { file = "LICENSE" }
|
|
15
15
|
classifiers = [
|
|
16
16
|
"Programming Language :: Python :: 3",
|
|
17
|
-
"Programming Language :: Python :: 3.
|
|
17
|
+
"Programming Language :: Python :: 3.10",
|
|
18
18
|
"Programming Language :: Python :: 3 :: Only",
|
|
19
19
|
"License :: OSI Approved :: MIT License",
|
|
20
20
|
"Operating System :: OS Independent",
|
|
21
21
|
]
|
|
22
22
|
dependencies = [
|
|
23
|
-
"click
|
|
24
|
-
"duckdb
|
|
25
|
-
"humanize
|
|
26
|
-
"loguru
|
|
27
|
-
"paramiko
|
|
28
|
-
"pydantic-settings
|
|
29
|
-
"pydantic
|
|
30
|
-
"requests
|
|
23
|
+
"click",
|
|
24
|
+
"duckdb",
|
|
25
|
+
"humanize",
|
|
26
|
+
"loguru",
|
|
27
|
+
"paramiko",
|
|
28
|
+
"pydantic-settings",
|
|
29
|
+
"pydantic",
|
|
30
|
+
"requests",
|
|
31
31
|
"sshtunnel==0.4.0",
|
|
32
32
|
]
|
|
33
33
|
|
|
@@ -36,14 +36,14 @@ utill = "utill.cmd.utill:main"
|
|
|
36
36
|
|
|
37
37
|
[project.optional-dependencies]
|
|
38
38
|
google-cloud = [
|
|
39
|
-
"google-cloud-bigquery
|
|
40
|
-
"google-cloud-storage
|
|
39
|
+
"google-cloud-bigquery",
|
|
40
|
+
"google-cloud-storage",
|
|
41
41
|
]
|
|
42
42
|
postgresql = [
|
|
43
|
-
"psycopg
|
|
43
|
+
"psycopg",
|
|
44
44
|
]
|
|
45
45
|
pdf = [
|
|
46
|
-
"PyPDF2
|
|
46
|
+
"PyPDF2",
|
|
47
47
|
]
|
|
48
48
|
|
|
49
49
|
[tool.setuptools.packages.find]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rdxz2-utill
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.6
|
|
4
4
|
Summary: Your daily Python utility
|
|
5
5
|
Author-email: Richard Dharmawan <richard.dharmawan@gmail.com>
|
|
6
6
|
License: MIT License
|
|
@@ -25,29 +25,29 @@ License: MIT License
|
|
|
25
25
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
26
|
SOFTWARE.
|
|
27
27
|
Classifier: Programming Language :: Python :: 3
|
|
28
|
-
Classifier: Programming Language :: Python :: 3.
|
|
28
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
29
29
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
30
30
|
Classifier: License :: OSI Approved :: MIT License
|
|
31
31
|
Classifier: Operating System :: OS Independent
|
|
32
|
-
Requires-Python: >=3.
|
|
32
|
+
Requires-Python: >=3.10
|
|
33
33
|
Description-Content-Type: text/markdown
|
|
34
34
|
License-File: LICENSE
|
|
35
|
-
Requires-Dist: click
|
|
36
|
-
Requires-Dist: duckdb
|
|
37
|
-
Requires-Dist: humanize
|
|
38
|
-
Requires-Dist: loguru
|
|
39
|
-
Requires-Dist: paramiko
|
|
40
|
-
Requires-Dist: pydantic-settings
|
|
41
|
-
Requires-Dist: pydantic
|
|
42
|
-
Requires-Dist: requests
|
|
35
|
+
Requires-Dist: click
|
|
36
|
+
Requires-Dist: duckdb
|
|
37
|
+
Requires-Dist: humanize
|
|
38
|
+
Requires-Dist: loguru
|
|
39
|
+
Requires-Dist: paramiko
|
|
40
|
+
Requires-Dist: pydantic-settings
|
|
41
|
+
Requires-Dist: pydantic
|
|
42
|
+
Requires-Dist: requests
|
|
43
43
|
Requires-Dist: sshtunnel==0.4.0
|
|
44
44
|
Provides-Extra: google-cloud
|
|
45
|
-
Requires-Dist: google-cloud-bigquery
|
|
46
|
-
Requires-Dist: google-cloud-storage
|
|
45
|
+
Requires-Dist: google-cloud-bigquery; extra == "google-cloud"
|
|
46
|
+
Requires-Dist: google-cloud-storage; extra == "google-cloud"
|
|
47
47
|
Provides-Extra: postgresql
|
|
48
|
-
Requires-Dist: psycopg
|
|
48
|
+
Requires-Dist: psycopg; extra == "postgresql"
|
|
49
49
|
Provides-Extra: pdf
|
|
50
|
-
Requires-Dist: PyPDF2
|
|
50
|
+
Requires-Dist: PyPDF2; extra == "pdf"
|
|
51
51
|
Dynamic: license-file
|
|
52
52
|
|
|
53
53
|
# Using this library
|
|
@@ -247,6 +247,8 @@ class BQ():
|
|
|
247
247
|
def download_csv(self, query: str, dst_filename: str, combine: bool = True, pre_query: str = None):
|
|
248
248
|
if not dst_filename.endswith('.csv'):
|
|
249
249
|
raise ValueError('Destination filename must ends with .csv!')
|
|
250
|
+
|
|
251
|
+
dst_filename = os.path.expanduser(dst_filename)
|
|
250
252
|
|
|
251
253
|
dirname = dst_filename.removesuffix('.csv')
|
|
252
254
|
|
|
@@ -99,59 +99,61 @@ class PG:
|
|
|
99
99
|
for data in copy:
|
|
100
100
|
f.write(data)
|
|
101
101
|
|
|
102
|
-
def pg_to_pg(self, pg: "PG",
|
|
103
|
-
tmp_filename = generate_random_string() + '.csv'
|
|
102
|
+
def pg_to_pg(self, pg: "PG", src_table_name: str, dst_table_name: str, cols: list[str] = None) -> None:
|
|
103
|
+
tmp_filename = generate_random_string(alphanum=True) + '.csv'
|
|
104
104
|
cols_str = ','.join([f'"{x}"' for x in cols]) if (cols is not None and cols != []) else '*'
|
|
105
105
|
try:
|
|
106
|
-
self.download_csv(f'SELECT {cols_str} FROM {
|
|
107
|
-
pg.upload_csv(tmp_filename,
|
|
106
|
+
self.download_csv(f'SELECT {cols_str} FROM {src_table_name}', tmp_filename)
|
|
107
|
+
pg.upload_csv(tmp_filename, dst_table_name)
|
|
108
108
|
except:
|
|
109
109
|
raise
|
|
110
110
|
finally:
|
|
111
111
|
os.remove(tmp_filename) if os.path.exists(tmp_filename) else None
|
|
112
112
|
|
|
113
113
|
def check_table_existence(self, table_name: str) -> bool:
|
|
114
|
-
if not self.execute_query('''SELECT count(1) AS "cnt" FROM "information_schema"."tables" WHERE "table_schema" || '.' || "table_name" =
|
|
114
|
+
if not self.execute_query('''SELECT count(1) AS "cnt" FROM "information_schema"."tables" WHERE "table_schema" || '.' || "table_name" = %s;''', table_name).fetchone()[0]:
|
|
115
115
|
raise Exception(f'Target table \'{table_name}\' not created, please create it first!')
|
|
116
116
|
|
|
117
|
-
def upload_tuples(self, cols: list[str],
|
|
118
|
-
self.check_table_existence(
|
|
117
|
+
def upload_tuples(self, cols: list[str], src_tuples: list[tuple], src_table_name: str) -> None:
|
|
118
|
+
self.check_table_existence(src_table_name)
|
|
119
119
|
|
|
120
120
|
cols_str = ','.join([f'"{x}"' for x in cols])
|
|
121
|
-
query = f'''COPY {
|
|
121
|
+
query = f'''COPY {src_table_name}({cols_str}) FROM STDIN'''
|
|
122
122
|
logger.debug(f'🔎 Query:\n{query}')
|
|
123
123
|
with self.cursor.copy(query) as copy:
|
|
124
|
-
for row in
|
|
124
|
+
for row in src_tuples:
|
|
125
125
|
copy.write_row(row)
|
|
126
126
|
|
|
127
|
-
def upload_list_of_dict(self,
|
|
128
|
-
self.check_table_existence(
|
|
127
|
+
def upload_list_of_dict(self, src_data: list[dict], dst_table_name: str) -> None:
|
|
128
|
+
self.check_table_existence(dst_table_name)
|
|
129
129
|
|
|
130
|
-
if len(
|
|
130
|
+
if len(src_data) == 0:
|
|
131
131
|
raise ValueError('No data to upload!')
|
|
132
132
|
|
|
133
|
-
cols =
|
|
133
|
+
cols = src_data[0].keys()
|
|
134
134
|
cols_str = ','.join([f'"{x}"' for x in cols])
|
|
135
|
-
query = f'''COPY {
|
|
135
|
+
query = f'''COPY {dst_table_name}({cols_str}) FROM STDIN'''
|
|
136
136
|
logger.debug(f'🔎 Query:\n{query}')
|
|
137
137
|
with self.cursor.copy(query) as copy:
|
|
138
|
-
for row in
|
|
138
|
+
for row in src_data:
|
|
139
139
|
copy.write_row(tuple(row[col] for col in cols))
|
|
140
140
|
|
|
141
|
-
def upload_csv(self,
|
|
142
|
-
|
|
141
|
+
def upload_csv(self, src_filename: str, dst_table_name: str) -> None:
|
|
142
|
+
src_filename = os.path.expanduser(src_filename)
|
|
143
143
|
|
|
144
|
-
|
|
144
|
+
self.check_table_existence(dst_table_name)
|
|
145
|
+
|
|
146
|
+
cols_str = ','.join([f'"{x}"' for x in next(csv.reader(open(src_filename, 'r')))])
|
|
145
147
|
query = dedent(
|
|
146
148
|
f'''
|
|
147
|
-
COPY {
|
|
149
|
+
COPY {dst_table_name}({cols_str})
|
|
148
150
|
FROM STDIN
|
|
149
151
|
DELIMITER ','
|
|
150
152
|
CSV HEADER;
|
|
151
153
|
'''
|
|
152
154
|
)
|
|
153
155
|
logger.debug(f'🔎 Query:\n{query}')
|
|
154
|
-
with open(os.path.expanduser(
|
|
156
|
+
with open(os.path.expanduser(src_filename), 'r') as f:
|
|
155
157
|
with self.cursor.copy(query) as copy:
|
|
156
158
|
while data := f.read(1024):
|
|
157
159
|
copy.write(data)
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
click==8.1.8
|
|
2
|
-
duckdb==1.1.3
|
|
3
|
-
humanize==4.11.0
|
|
4
|
-
loguru==0.7.3
|
|
5
|
-
paramiko==3.5.0
|
|
6
|
-
pydantic-settings==2.7.1
|
|
7
|
-
pydantic==2.10.6
|
|
8
|
-
requests==2.32.3
|
|
9
|
-
sshtunnel==0.4.0
|
|
10
|
-
|
|
11
|
-
[google-cloud]
|
|
12
|
-
google-cloud-bigquery==3.29.0
|
|
13
|
-
google-cloud-storage==3.0.0
|
|
14
|
-
|
|
15
|
-
[pdf]
|
|
16
|
-
PyPDF2==3.0.1
|
|
17
|
-
|
|
18
|
-
[postgresql]
|
|
19
|
-
psycopg==3.2.2
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|