pydatacube 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pydatacube-0.1.7/LICENSE +24 -0
- pydatacube-0.1.7/PKG-INFO +27 -0
- pydatacube-0.1.7/README.md +6 -0
- pydatacube-0.1.7/pyproject.toml +34 -0
- pydatacube-0.1.7/setup.cfg +4 -0
- pydatacube-0.1.7/src/pyDataCube/EntryPoint.py +245 -0
- pydatacube-0.1.7/src/pyDataCube/Enums.py +9 -0
- pydatacube-0.1.7/src/pyDataCube/Interface.py +33 -0
- pydatacube-0.1.7/src/pyDataCube/__init__.py +2 -0
- pydatacube-0.1.7/src/pyDataCube/converter/__init__.py +0 -0
- pydatacube-0.1.7/src/pyDataCube/converter/export.py +10 -0
- pydatacube-0.1.7/src/pyDataCube/converter/importer.py +270 -0
- pydatacube-0.1.7/src/pyDataCube/converter/qb_integrity_constraints.py +23 -0
- pydatacube-0.1.7/src/pyDataCube/converter/validator.py +28 -0
- pydatacube-0.1.7/src/pyDataCube/cube/AggregateFunction.py +14 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Attribute.py +135 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Axis.py +29 -0
- pydatacube-0.1.7/src/pyDataCube/cube/BaseCube.py +216 -0
- pydatacube-0.1.7/src/pyDataCube/cube/BooleanConnective.py +7 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Cube.py +82 -0
- pydatacube-0.1.7/src/pyDataCube/cube/CubeOperators.py +146 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Cuboid.py +127 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Dimension.py +68 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Hierarchy.py +53 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Level.py +43 -0
- pydatacube-0.1.7/src/pyDataCube/cube/LevelMember.py +193 -0
- pydatacube-0.1.7/src/pyDataCube/cube/LevelMemberType.py +6 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Measure.py +63 -0
- pydatacube-0.1.7/src/pyDataCube/cube/NonTopLevel.py +111 -0
- pydatacube-0.1.7/src/pyDataCube/cube/Predicate.py +47 -0
- pydatacube-0.1.7/src/pyDataCube/cube/PredicateOperator.py +9 -0
- pydatacube-0.1.7/src/pyDataCube/cube/TopLevel.py +9 -0
- pydatacube-0.1.7/src/pyDataCube/cube/View.py +390 -0
- pydatacube-0.1.7/src/pyDataCube/cube/__init__.py +0 -0
- pydatacube-0.1.7/src/pyDataCube/engines/__init__.py +3 -0
- pydatacube-0.1.7/src/pyDataCube/engines/engine.py +4 -0
- pydatacube-0.1.7/src/pyDataCube/engines/postgres.py +14 -0
- pydatacube-0.1.7/src/pyDataCube/session/__init__.py +0 -0
- pydatacube-0.1.7/src/pyDataCube/session/cube_metadata.py +111 -0
- pydatacube-0.1.7/src/pyDataCube/session/infer_cube.py +224 -0
- pydatacube-0.1.7/src/pyDataCube/session/session.py +98 -0
- pydatacube-0.1.7/src/pyDataCube/session/sql_queries.py +106 -0
- pydatacube-0.1.7/src/pyDataCube/timers.py +87 -0
- pydatacube-0.1.7/src/pydatacube.egg-info/PKG-INFO +27 -0
- pydatacube-0.1.7/src/pydatacube.egg-info/SOURCES.txt +53 -0
- pydatacube-0.1.7/src/pydatacube.egg-info/dependency_links.txt +1 -0
- pydatacube-0.1.7/src/pydatacube.egg-info/requires.txt +6 -0
- pydatacube-0.1.7/src/pydatacube.egg-info/top_level.txt +1 -0
- pydatacube-0.1.7/test/test_cube.py +682 -0
- pydatacube-0.1.7/test/test_cube_operators.py +228 -0
- pydatacube-0.1.7/test/test_infer_cube.py +44 -0
- pydatacube-0.1.7/test/test_level_member.py +44 -0
- pydatacube-0.1.7/test/test_non_top_level.py +48 -0
- pydatacube-0.1.7/test/test_regular_dimension.py +60 -0
- pydatacube-0.1.7/test/test_top_level.py +23 -0
pydatacube-0.1.7/LICENSE
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
This is free and unencumbered software released into the public domain.
|
|
2
|
+
|
|
3
|
+
Anyone is free to copy, modify, publish, use, compile, sell, or
|
|
4
|
+
distribute this software, either in source code form or as a compiled
|
|
5
|
+
binary, for any purpose, commercial or non-commercial, and by any
|
|
6
|
+
means.
|
|
7
|
+
|
|
8
|
+
In jurisdictions that recognize copyright laws, the author or authors
|
|
9
|
+
of this software dedicate any and all copyright interest in the
|
|
10
|
+
software to the public domain. We make this dedication for the benefit
|
|
11
|
+
of the public at large and to the detriment of our heirs and
|
|
12
|
+
successors. We intend this dedication to be an overt act of
|
|
13
|
+
relinquishment in perpetuity of all present and future rights to this
|
|
14
|
+
software under copyright law.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
19
|
+
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
|
20
|
+
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
21
|
+
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
|
22
|
+
OTHER DEALINGS IN THE SOFTWARE.
|
|
23
|
+
|
|
24
|
+
For more information, please refer to <https://unlicense.org>
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pydatacube
|
|
3
|
+
Version: 0.1.7
|
|
4
|
+
Summary: Querying data cubes with Python using MDX inspired declarative syntax
|
|
5
|
+
Author-email: Sigmundur Vang <siva@cs.aau.dk>
|
|
6
|
+
License-Expression: Unlicense
|
|
7
|
+
Project-URL: Homepage, https://gitlab.com/sigmundur_phd/pycube
|
|
8
|
+
Project-URL: Source, https://gitlab.com/sigmundur_phd/pycube.git
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.8
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: numpy
|
|
15
|
+
Requires-Dist: psycopg2
|
|
16
|
+
Requires-Dist: pandas
|
|
17
|
+
Requires-Dist: levenshtein
|
|
18
|
+
Requires-Dist: rdflib
|
|
19
|
+
Requires-Dist: sqlalchemy
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# pyDataCube
|
|
23
|
+
## Running the experiments
|
|
24
|
+
Commit 1c84c423dc439e842a5906a86235c3e7439f0c7e was used for the experiments.
|
|
25
|
+
The timer object that measured the time spent in the database was injected into pandas' `read_sql` method (and its auxiliary methods) in `io/sql.py`.
|
|
26
|
+
The modified `sql.py` file is included in the base of this repository as `pandas_sql.py`.
|
|
27
|
+
This modified `sql.py` was, together with the version associated with the above commit hash, used for gaining the experiments results in the journal paper
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# pyDataCube
|
|
2
|
+
## Running the experiments
|
|
3
|
+
Commit 1c84c423dc439e842a5906a86235c3e7439f0c7e was used for the experiments.
|
|
4
|
+
The timer object that measured the time spent in the database was injected into pandas' `read_sql` method (and its auxiliary methods) in `io/sql.py`.
|
|
5
|
+
The modified `sql.py` file is included in the base of this repository as `pandas_sql.py`.
|
|
6
|
+
This modified `sql.py` was, together with the version associated with the above commit hash, used for gaining the experiments results in the journal paper
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pydatacube"
|
|
7
|
+
version = "0.1.7"
|
|
8
|
+
description = "Querying data cubes with Python using MDX inspired declarative syntax"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = "Unlicense"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"numpy",
|
|
15
|
+
"psycopg2",
|
|
16
|
+
"pandas",
|
|
17
|
+
"levenshtein",
|
|
18
|
+
"rdflib",
|
|
19
|
+
"sqlalchemy"
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
urls = { "Homepage" = "https://gitlab.com/sigmundur_phd/pycube", "Source" = "https://gitlab.com/sigmundur_phd/pycube.git" }
|
|
23
|
+
|
|
24
|
+
authors = [
|
|
25
|
+
{ name = "Sigmundur Vang", email = "siva@cs.aau.dk" }
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
classifiers = [
|
|
29
|
+
"Programming Language :: Python :: 3",
|
|
30
|
+
"Operating System :: OS Independent"
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from src.pyDataCube.engines.postgres import postgres
|
|
4
|
+
from src.pyDataCube.session.session import *
|
|
5
|
+
|
|
6
|
+
DATABASE_USER = "sigmundur"
|
|
7
|
+
DATABASE_PASSWORD = ""
|
|
8
|
+
DATABASE_HOST = "127.0.0.1"
|
|
9
|
+
DATABASE_PORT = "5432"
|
|
10
|
+
DATABASE_NAME = "ssb_snowflake_dev"
|
|
11
|
+
|
|
12
|
+
postgres_engine: Postgres = postgres(DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, DATABASE_HOST, DATABASE_PORT)
|
|
13
|
+
postgres = create_session(postgres_engine)
|
|
14
|
+
view = postgres.load_view('lineorder')
|
|
15
|
+
print(view.day.year.attributes)
|
|
16
|
+
print(view.measures)
|
|
17
|
+
|
|
18
|
+
# # converter = QB4OLAPExport(view.cube, "./output.ttl")
|
|
19
|
+
# # converter.export()
|
|
20
|
+
# importer = QB4OLAPImporter("output.ttl", postgres_engine)
|
|
21
|
+
# session = importer.importer()
|
|
22
|
+
# view2: View = session.load_view("lineorder")
|
|
23
|
+
# # validator = Validator(view.cube._metadata)
|
|
24
|
+
# # result = validator.validate_qb()
|
|
25
|
+
|
|
26
|
+
mere_view = view.columns(view.day.month.mo_month.members()) \
|
|
27
|
+
.rows(view.part.brand1.b_brand1.members()) \
|
|
28
|
+
.pages(view.day.day.d_daynuminweek.members())
|
|
29
|
+
result100 = mere_view.output()
|
|
30
|
+
|
|
31
|
+
hej = 1
|
|
32
|
+
# view42 = view.columns(view.day.month.mo_month.members())
|
|
33
|
+
# output42 = view42.output()
|
|
34
|
+
#
|
|
35
|
+
# view.lo_quantity.set_aggregate_function("avg")
|
|
36
|
+
#
|
|
37
|
+
# view9 = view.columns(view.day.year.y_year[1997].children())\
|
|
38
|
+
# .rows(view.part.category.ca_category.members())\
|
|
39
|
+
# .pages([view.customer.city.ci_city["UNITED ST9"]])\
|
|
40
|
+
# .where((view.day.day.d_daynuminmonth <= 7)
|
|
41
|
+
# & ((view.supplier.nation.n_nation == "MOZAMBIQUE")
|
|
42
|
+
# | (view.supplier.nation.n_nation == "RUSSIA")))\
|
|
43
|
+
# .using(view.lo_quantity,
|
|
44
|
+
# lost_income=view.lo_extendedprice * view.lo_discount)
|
|
45
|
+
#
|
|
46
|
+
# urslit = view9.output()
|
|
47
|
+
#
|
|
48
|
+
# test = view.day.year.y_year[1997]
|
|
49
|
+
# test2 = test.children()
|
|
50
|
+
# view.day.month.mo_month.set_sort_key(view.day.month.mo_yearmonth)
|
|
51
|
+
# test3 = test.children()
|
|
52
|
+
#
|
|
53
|
+
# hej = 1
|
|
54
|
+
#
|
|
55
|
+
mere_view = view.columns(view.day.month.mo_month.members()) \
|
|
56
|
+
.rows(view.part.category.ca_category.members()) \
|
|
57
|
+
.pages(view.day.year.y_year.members()) \
|
|
58
|
+
.using(view.lo_quantity, view.lo_tax)
|
|
59
|
+
result_uden_order = mere_view.output()
|
|
60
|
+
|
|
61
|
+
hej = 1
|
|
62
|
+
# view.day.month.mo_month.set_sort_key(view.day.month.mo_yearmonthnum)
|
|
63
|
+
#
|
|
64
|
+
# mere_view = view.columns(view.day.month.mo_month.members()) \
|
|
65
|
+
# .rows(view.part.category.ca_category.members()) \
|
|
66
|
+
# .pages(view.day.year.y_year.members()) \
|
|
67
|
+
# .using(view.lo_quantity, view.lo_tax)
|
|
68
|
+
# result_med_order = mere_view.output()
|
|
69
|
+
#
|
|
70
|
+
# hej = 1
|
|
71
|
+
#
|
|
72
|
+
|
|
73
|
+
city = view.customer.city.ci_city.members()[0]
|
|
74
|
+
|
|
75
|
+
new_view = view \
|
|
76
|
+
.axis(0, view.day.year.y_year[1992].children())\
|
|
77
|
+
.axis(1, view.part.category.ca_category.members())\
|
|
78
|
+
.axis(2, [view.customer.city.ci_city.members()[0]])\
|
|
79
|
+
.where((view.day.day.d_daynuminmonth <= 7)
|
|
80
|
+
& ((view.supplier.nation.n_nation == "ARGENTINA")
|
|
81
|
+
| (view.supplier.nation.n_nation == "FRANCE")))\
|
|
82
|
+
.using(view.lo_revenue,
|
|
83
|
+
lost_income=
|
|
84
|
+
view.lo_extendedprice * view.lo_discount)
|
|
85
|
+
|
|
86
|
+
# new_view2 = view \
|
|
87
|
+
# .axis(0, view.day.year.y_year[1992].children()) \
|
|
88
|
+
# .axis(1, view.part.category.ca_category.members()) \
|
|
89
|
+
# .axis(2, [city]) \
|
|
90
|
+
# .where((view.day.day.d_daynuminmonth <= 7)
|
|
91
|
+
# & ((view.supplier.nation.n_nation == "CANADA")
|
|
92
|
+
# | (view.supplier.nation.n_nation == "JAPAN"))) \
|
|
93
|
+
# .using(view.lo_revenue,
|
|
94
|
+
# lost_income=
|
|
95
|
+
# view.lo_extendedprice * view.lo_discount)
|
|
96
|
+
|
|
97
|
+
article_result = new_view.output()
|
|
98
|
+
# check_result = new_view2.output()
|
|
99
|
+
|
|
100
|
+
# # view.delete_measure(view.lo_revenue)
|
|
101
|
+
#
|
|
102
|
+
hej = 1
|
|
103
|
+
#
|
|
104
|
+
# # result2 = view.output()
|
|
105
|
+
#
|
|
106
|
+
# view9 = view.where(
|
|
107
|
+
# (view.day.year.y_year == 1993)
|
|
108
|
+
# & (view.lo_discount > 0)
|
|
109
|
+
# & (view.lo_discount < 4)
|
|
110
|
+
# & (view.lo_quantity < 25)) \
|
|
111
|
+
# .using(revenue=view.lo_extendedprice * view.lo_discount)
|
|
112
|
+
#
|
|
113
|
+
# result2 = view9.output()
|
|
114
|
+
#
|
|
115
|
+
#
|
|
116
|
+
# hej = 1
|
|
117
|
+
#
|
|
118
|
+
# # test = view.output()
|
|
119
|
+
#
|
|
120
|
+
# # test10 = view.part.category.ca_category["MFGR#44"]["MFGR#4427"]
|
|
121
|
+
# # test11 = view.day.day.d_dayofweek.Thursday
|
|
122
|
+
# #
|
|
123
|
+
# # test1 = view.day.month.mo_month.members()
|
|
124
|
+
# # test1 = view.day.year.y_year[1992].January
|
|
125
|
+
# # test2 = [x for xs in [i.children() for i in test1] for x in xs]
|
|
126
|
+
#
|
|
127
|
+
# view10 = view.columns(view.day.month.mo_month.members()) \
|
|
128
|
+
# .rows(view.part.part.p_name.members()) \
|
|
129
|
+
# .using(view.lo_revenue)
|
|
130
|
+
# result10 = view10.output()
|
|
131
|
+
#
|
|
132
|
+
# view3 = view.columns(view.day.year.y_year.members()) \
|
|
133
|
+
# .rows(view.part.brand1.b_brand1.members()) \
|
|
134
|
+
# .where((view.part.category.ca_category == "MFGR#12") &
|
|
135
|
+
# (view.supplier.region.r_region == "AMERICA")) \
|
|
136
|
+
# .using(view.lo_revenue, revenue=view.lo_extendedprice * view.lo_discount)
|
|
137
|
+
# result = view3.output()
|
|
138
|
+
#
|
|
139
|
+
# test = view.columns(view.customer.city.ci_city.members()) \
|
|
140
|
+
# .rows(view.supplier.city.ci_city.members()) \
|
|
141
|
+
# .pages(view.day.year.y_year.members()) \
|
|
142
|
+
# .using(view.lo_revenue)
|
|
143
|
+
# result2 = test.output()
|
|
144
|
+
#
|
|
145
|
+
# view4 = view.columns(view.day.month.mo_month.members())
|
|
146
|
+
# result3 = view4.output()
|
|
147
|
+
#
|
|
148
|
+
# hej = 1
|
|
149
|
+
#
|
|
150
|
+
# # ## ssb query 2.1
|
|
151
|
+
# # view.columns(view.date.year.y_year.members()) \
|
|
152
|
+
# # .rows(view.part.brand1.b_brand1.members()) \
|
|
153
|
+
# # .where(view.part.category.ca_category = "MFGR#12" &
|
|
154
|
+
# # view.supplier.region.r_region = "AMERICA") \
|
|
155
|
+
# # .measures(view.lo_revenue)
|
|
156
|
+
#
|
|
157
|
+
#
|
|
158
|
+
# # print()
|
|
159
|
+
# # print("Measures: ", cube.measures())
|
|
160
|
+
# # print("Dimensions: ", cube.dimensions())
|
|
161
|
+
# # print("Date hierarchy: ", cube.date.hierarchies())
|
|
162
|
+
# # print("Supplier name dictionary: ", cube.supplier.supplier_name.__dict__)
|
|
163
|
+
# # print("Date dimension dictionary: ", cube.date.__dict__)
|
|
164
|
+
# # print("Date year level dictionary: ", cube.date.date_year.__dict__)
|
|
165
|
+
# # print("2022 Level member: ", cube.date.date_year["2022"]["January"])
|
|
166
|
+
#
|
|
167
|
+
# # print("Output of the cube (cube): ", cube.output())
|
|
168
|
+
# # print("Date dimension current level: ", cube._dimension_list[1].current_level)
|
|
169
|
+
# # c1 = cube._drill_down(cube.date)
|
|
170
|
+
# # print("Date dimension current level: ", c1._dimension_list[1].current_level)
|
|
171
|
+
# # print("c1 dictionary: ", c1.__dict__)
|
|
172
|
+
# # print("Output of the cube (c1): ", c1.output())
|
|
173
|
+
#
|
|
174
|
+
# # c2 = c1._drill_down(c1.date)
|
|
175
|
+
# # print("Cube (c2) dictionary: ", c2.date.__dict__)
|
|
176
|
+
# # c3 = c2._slice(c2.date, c2.date.date_year._2022._January)
|
|
177
|
+
# # print(c3._dimension_list[1].__dict__)
|
|
178
|
+
#
|
|
179
|
+
#
|
|
180
|
+
# # print(c2._dimension_list[1].current_level)
|
|
181
|
+
# # c3 = c2._roll_up(c2.date)
|
|
182
|
+
# # print(c3._dimension_list[1].current_level)
|
|
183
|
+
# # c4 = c3._roll_up(c3.date)
|
|
184
|
+
# # print(c4._dimension_list[1].current_level)
|
|
185
|
+
#
|
|
186
|
+
#
|
|
187
|
+
# # print("Date dimension dictionary: ", cube.date.__dict__)
|
|
188
|
+
# # cube.date._roll_up()
|
|
189
|
+
# # print("Date dimension dictionary: ", cube.date.__dict__)
|
|
190
|
+
#
|
|
191
|
+
# # print("2022 Level member dictionary: ", cube.date.date_year._2022.__dict__)
|
|
192
|
+
# # print("January Level member dictionary: ", cube.date.date_year._2022._January.__dict__)
|
|
193
|
+
# # print("Day 1 Level member dictionary: ", cube.date.date_year._2022._January._1.__dict__)
|
|
194
|
+
#
|
|
195
|
+
# # ## Supplier testing
|
|
196
|
+
# # print("Supplier hierarchy: ", cube.supplier.hierarchies())
|
|
197
|
+
# # print("Nation dictionary: ", cube.supplier.supplier_nation.__dict__)
|
|
198
|
+
# # print("Denmark dictionary: ", cube.supplier.supplier_nation._Denmark.__dict__)
|
|
199
|
+
#
|
|
200
|
+
#
|
|
201
|
+
# # cube.columns([cube.date.date_year._2022._January])
|
|
202
|
+
# # print("The output: ", cube.output())
|
|
203
|
+
#
|
|
204
|
+
#
|
|
205
|
+
# # print()
|
|
206
|
+
# # print(cube.store.store_address["Jyllandsgade 1"])
|
|
207
|
+
# # print(cube.store.store_address["Jyllandsgade 2"])
|
|
208
|
+
# #
|
|
209
|
+
# # cube.where(cube.supplier.supplier_nation == "Denmark" and cube.store.store_address == "Jyllandsgade 1")
|
|
210
|
+
#
|
|
211
|
+
#
|
|
212
|
+
#
|
|
213
|
+
# # conn = psycopg2.connect(
|
|
214
|
+
# # dbname="salesdb_snowflake_test",
|
|
215
|
+
# # user="sigmundur",
|
|
216
|
+
# # password=""
|
|
217
|
+
# # )
|
|
218
|
+
#
|
|
219
|
+
#
|
|
220
|
+
# # # Cube -> View syntactic suger not implemented
|
|
221
|
+
# # view._axes = []
|
|
222
|
+
# # view = view.axis(0, view.cube.date.date_month.members())
|
|
223
|
+
#
|
|
224
|
+
#
|
|
225
|
+
# # output = view.output()
|
|
226
|
+
# # print(output)
|
|
227
|
+
# # with conn:
|
|
228
|
+
# # with conn.cursor() as curs:
|
|
229
|
+
# # curs.execute(output)
|
|
230
|
+
# # print(curs.fetchall())
|
|
231
|
+
#
|
|
232
|
+
#
|
|
233
|
+
# def generate_data():
|
|
234
|
+
# supplier = 1221
|
|
235
|
+
# store = 4444
|
|
236
|
+
# product = 9012
|
|
237
|
+
# total_sales_price = np.random.randint(10, high=1000, size=721)
|
|
238
|
+
# unit_sales = np.random.randint(1, high=20, size=721)
|
|
239
|
+
# sale_date = [i for i in range(1, 722)]
|
|
240
|
+
#
|
|
241
|
+
# with open("/home/sigmundur/test/test.csv", "w") as out:
|
|
242
|
+
# csv_out = csv.writer(out)
|
|
243
|
+
# for i in range(len(sale_date)):
|
|
244
|
+
# csv_out.writerow((supplier, store, product, sale_date[i], total_sales_price[i], unit_sales[i]))
|
|
245
|
+
#
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Interface(ABC):
|
|
5
|
+
pass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CubeInterface(Interface):
|
|
9
|
+
# @abstractmethod
|
|
10
|
+
# def drill_out(self, dimension_name):
|
|
11
|
+
# pass
|
|
12
|
+
|
|
13
|
+
@abstractmethod
|
|
14
|
+
def columns(self, column):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
@abstractmethod
|
|
18
|
+
def rows(self, row):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
@abstractmethod
|
|
22
|
+
def where(self, **kwargs):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DimensionInterface(Interface):
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def roll_up(self):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
@abstractmethod
|
|
32
|
+
def drill_down(self):
|
|
33
|
+
pass
|
|
File without changes
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
from typing import Iterator
|
|
2
|
+
|
|
3
|
+
import psycopg2
|
|
4
|
+
from psycopg2.extensions import connection as psyconn
|
|
5
|
+
from rdflib import Graph, Namespace, Node
|
|
6
|
+
|
|
7
|
+
from ..cube.Attribute import Attribute
|
|
8
|
+
from ..cube.Axis import Axis
|
|
9
|
+
from ..cube.BaseCube import BaseCube
|
|
10
|
+
from ..cube.Dimension import Dimension
|
|
11
|
+
from ..cube.Hierarchy import Hierarchy
|
|
12
|
+
from ..cube.Level import Level
|
|
13
|
+
from ..cube.LevelMember import LevelMember
|
|
14
|
+
from ..cube.TopLevel import TopLevel
|
|
15
|
+
from ..engines.postgres import Postgres
|
|
16
|
+
from ..engines.engine import Engine
|
|
17
|
+
from ..cube.AggregateFunction import AggregateFunction
|
|
18
|
+
from ..cube.Measure import Measure
|
|
19
|
+
from ..cube.NonTopLevel import NonTopLevel
|
|
20
|
+
from ..cube.View import View
|
|
21
|
+
from ..session.session import Session
|
|
22
|
+
from ..session.sql_queries import get_non_key_columns_query, get_pk_and_fk_columns_query, lowest_levels_query
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class QB4OLAPImporter:
|
|
26
|
+
def __init__(self, filename: str, engine: Engine) -> None:
|
|
27
|
+
self.filename: str = filename
|
|
28
|
+
self.engine: Engine = engine
|
|
29
|
+
self.fact_table: str = ""
|
|
30
|
+
self.measures: list[Measure] = []
|
|
31
|
+
self.dimensions: list[Dimension] = []
|
|
32
|
+
self.views: list[View] = []
|
|
33
|
+
self.QB: Namespace = Namespace("http://purl.org/linked-data/cube#")
|
|
34
|
+
self.RDF: Namespace = Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
|
|
35
|
+
self.QB4O: Namespace = Namespace("http://purl.org/qb4olap/cubes#")
|
|
36
|
+
self.processed_dimensions: dict[Node, list[Node]] = {}
|
|
37
|
+
self.processed_keys: list[str] = []
|
|
38
|
+
|
|
39
|
+
def importer(self) -> Session:
|
|
40
|
+
g = Graph()
|
|
41
|
+
g.parse(self.filename, format="ttl")
|
|
42
|
+
|
|
43
|
+
dsd_nodes: list[Node] = self._find_dsd_nodes(g)
|
|
44
|
+
if not dsd_nodes:
|
|
45
|
+
raise Exception(f"No data structure definition found in {self.filename}")
|
|
46
|
+
else:
|
|
47
|
+
for dsd in dsd_nodes:
|
|
48
|
+
self._process_dsd(g, dsd)
|
|
49
|
+
|
|
50
|
+
return Session(self.views, self.engine)
|
|
51
|
+
|
|
52
|
+
def _process_dsd(self, g: Graph, dsd: Node) -> None:
|
|
53
|
+
self.fact_table = self._extract_name(dsd)
|
|
54
|
+
for _, _, component_specification in g.triples((dsd, self.QB.component, None)):
|
|
55
|
+
self._process_component_specification(g, component_specification)
|
|
56
|
+
|
|
57
|
+
for dimension in self.dimensions:
|
|
58
|
+
dimension.metadata = g
|
|
59
|
+
|
|
60
|
+
cube: BaseCube = BaseCube(self.fact_table, self.dimensions, self.measures, self.fact_table, g, self.engine)
|
|
61
|
+
cube.base_cube = cube
|
|
62
|
+
axes: list[Axis] = []
|
|
63
|
+
for dimension in self.dimensions:
|
|
64
|
+
hierarchy: Hierarchy = dimension.hierarchies[0]
|
|
65
|
+
level: NonTopLevel = hierarchy.lowest_level
|
|
66
|
+
attribute: Attribute = level.level_member_attr
|
|
67
|
+
level_members: list[LevelMember] = attribute.members()
|
|
68
|
+
axes.append(Axis(dimension, hierarchy, level, attribute, level_members))
|
|
69
|
+
|
|
70
|
+
measures: list[Measure] = cube.measure_list[0] if cube.measure_list else []
|
|
71
|
+
predicates: None = None
|
|
72
|
+
self.views.append(View(cube=cube, name=cube.name))
|
|
73
|
+
|
|
74
|
+
def _process_component_specification(self, g: Graph, comp_spec: Node) -> None:
|
|
75
|
+
for _, comp_prop, Comp_Prop in g.triples((comp_spec, None, None)):
|
|
76
|
+
match comp_prop:
|
|
77
|
+
case self.QB.measure:
|
|
78
|
+
self.measures.append(self._process_measure(g, comp_spec))
|
|
79
|
+
case self.QB4O.level:
|
|
80
|
+
self.dimensions.append(self._process_dimension(g, Comp_Prop))
|
|
81
|
+
case self.QB4O.hasAggregateFunction:
|
|
82
|
+
continue
|
|
83
|
+
case _:
|
|
84
|
+
print(f"Only considering {self.QB4O.measure} and {self.QB4O.level} properties in DataStructureDefinition. Found {_}")
|
|
85
|
+
|
|
86
|
+
def _process_measure(self, g: Graph, comp_spec: Node) -> Measure:
|
|
87
|
+
name: str = ""
|
|
88
|
+
sql: str = ""
|
|
89
|
+
agg_func: AggregateFunction | None = None
|
|
90
|
+
for _, comp_prop, Comp_Prop in g.triples((comp_spec, None, None)):
|
|
91
|
+
match comp_prop:
|
|
92
|
+
case self.QB.measure:
|
|
93
|
+
measure_str: str = Comp_Prop.toPython()
|
|
94
|
+
name = measure_str[measure_str.find("#") + 1:]
|
|
95
|
+
sql = f"{self.fact_table}.{name}"
|
|
96
|
+
case self.QB4O.hasAggregateFunction:
|
|
97
|
+
try:
|
|
98
|
+
agg_func = self._process_aggregate_function(Comp_Prop)
|
|
99
|
+
except NotImplementedError as e:
|
|
100
|
+
raise e
|
|
101
|
+
except Exception as e:
|
|
102
|
+
raise Exception(f"Unknown triple ({comp_spec, comp_prop, Comp_Prop})") from e
|
|
103
|
+
case _:
|
|
104
|
+
raise Exception(f"Unknown component property {comp_prop} leading Component Property {Comp_Prop} leading to component specification {comp_spec} (triple ({comp_spec}, {comp_prop}, {Comp_Prop}) is invalid)")
|
|
105
|
+
if not (name and sql and agg_func):
|
|
106
|
+
raise Exception(f"Incomplete measure specification {comp_spec}")
|
|
107
|
+
else:
|
|
108
|
+
return Measure(name, agg_func, sql)
|
|
109
|
+
|
|
110
|
+
def _process_aggregate_function(self, agg_func: Node) -> AggregateFunction:
|
|
111
|
+
func: str = agg_func.toPython()
|
|
112
|
+
name: str = func[func.find("#") + 1:]
|
|
113
|
+
match name:
|
|
114
|
+
case "sum":
|
|
115
|
+
return AggregateFunction("SUM", lambda x, y: x + y)
|
|
116
|
+
case "min":
|
|
117
|
+
return AggregateFunction("MIN", lambda x, y: x if x < y else y)
|
|
118
|
+
case "max":
|
|
119
|
+
return AggregateFunction("MAX", lambda x, y: x if x > y else y)
|
|
120
|
+
case "count":
|
|
121
|
+
return AggregateFunction("COUNT", lambda acc, x: acc + 1)
|
|
122
|
+
case "avg":
|
|
123
|
+
return AggregateFunction("AVG", lambda x, y: x)
|
|
124
|
+
case _:
|
|
125
|
+
raise Exception(f"Unknown aggregate function {agg_func}")
|
|
126
|
+
|
|
127
|
+
def _get_keys(self, lvl_name: str) -> tuple[str, str]:
|
|
128
|
+
with self._get_db_connection() as conn:
|
|
129
|
+
with conn.cursor() as cur:
|
|
130
|
+
cur.execute(get_pk_and_fk_columns_query(lvl_name))
|
|
131
|
+
pk: str
|
|
132
|
+
fk: str
|
|
133
|
+
pk, fk = "", ""
|
|
134
|
+
for t in cur.fetchall():
|
|
135
|
+
if t[1] == 'PRIMARY KEY':
|
|
136
|
+
pk = t[0]
|
|
137
|
+
else:
|
|
138
|
+
fk = t[0]
|
|
139
|
+
return pk, fk
|
|
140
|
+
|
|
141
|
+
def _get_level_member_attribute(self, lvl_name: str, lvl_attrs: list[str]) -> str:
|
|
142
|
+
with self._get_db_connection() as conn:
|
|
143
|
+
with conn.cursor() as cur:
|
|
144
|
+
cur.execute(get_non_key_columns_query(lvl_name))
|
|
145
|
+
all_attrs = list(map(lambda x: x[0], cur.fetchall()))
|
|
146
|
+
|
|
147
|
+
lvl_member_attrs: list[str] = [x for x in all_attrs if x not in lvl_attrs]
|
|
148
|
+
if len(lvl_member_attrs) > 1:
|
|
149
|
+
raise Exception(f"Found multiple level member attributes: {lvl_member_attrs} for level '{lvl_name}'")
|
|
150
|
+
if len(lvl_member_attrs) == 0:
|
|
151
|
+
raise Exception(f"Found no level member attribute for level '{lvl_name}'")
|
|
152
|
+
return lvl_member_attrs[0]
|
|
153
|
+
|
|
154
|
+
def _get_next_level(self, g: Graph, Comp_Prop: Node) -> list[Node]:
|
|
155
|
+
return [x[2] for x in g.triples((Comp_Prop, self.QB4O.parentLevel, None))]
|
|
156
|
+
|
|
157
|
+
def _process_dimension(self, g: Graph, Comp_Prop: Node) -> Dimension:
|
|
158
|
+
dim_props: list[Node] = list(g.objects(Comp_Prop, self.QB4O.inDimension))
|
|
159
|
+
if len(dim_props) > 1:
|
|
160
|
+
chosen_dimension: Node = [x for x in dim_props if x not in self.processed_dimensions.setdefault(Comp_Prop, [])][0]
|
|
161
|
+
self.processed_dimensions[Comp_Prop].append(chosen_dimension)
|
|
162
|
+
else:
|
|
163
|
+
chosen_dimension: Node = dim_props[0]
|
|
164
|
+
dimension_name: str = self._extract_name(chosen_dimension)
|
|
165
|
+
|
|
166
|
+
hierarchies: list[HierarchyContainer] = []
|
|
167
|
+
hierarchy_counter: int = 0
|
|
168
|
+
def traverse(path: list[Node]) -> None:
|
|
169
|
+
current_node = path[-1]
|
|
170
|
+
next_levels = self._get_next_level(g, current_node)
|
|
171
|
+
|
|
172
|
+
if not next_levels:
|
|
173
|
+
hierarchy_container = HierarchyContainer()
|
|
174
|
+
nonlocal hierarchy_counter
|
|
175
|
+
hierarchy_counter += 1
|
|
176
|
+
hierarchy_container.name = f"h{hierarchy_counter}"
|
|
177
|
+
|
|
178
|
+
for node in path:
|
|
179
|
+
container = LevelContainer()
|
|
180
|
+
container.lvl_name = self._extract_name(node)
|
|
181
|
+
container.lvl_attrs = list(map(lambda x: self._extract_name(x[2]),
|
|
182
|
+
g.triples((node, self.QB4O.hasAttribute, None))))
|
|
183
|
+
container.lvl_member_attr = self._get_level_member_attribute(container.lvl_name,
|
|
184
|
+
container.lvl_attrs)
|
|
185
|
+
container.pk, container.fk = self._get_keys(container.lvl_name)
|
|
186
|
+
hierarchy_container.levels.append(container)
|
|
187
|
+
|
|
188
|
+
hierarchies.append(hierarchy_container)
|
|
189
|
+
return
|
|
190
|
+
|
|
191
|
+
for next_node in next_levels:
|
|
192
|
+
traverse(path + [next_node])
|
|
193
|
+
|
|
194
|
+
traverse([Comp_Prop])
|
|
195
|
+
|
|
196
|
+
more_hierarchies: list[Hierarchy] = []
|
|
197
|
+
for hierarchy in hierarchies:
|
|
198
|
+
lvls: list[Level] = []
|
|
199
|
+
for level in hierarchy:
|
|
200
|
+
lvls.append(NonTopLevel(level.lvl_name, self.engine, level.lvl_attrs, level.pk, level.fk, level.lvl_member_attr))
|
|
201
|
+
lvls.append(TopLevel())
|
|
202
|
+
self._attach_relations_to_levels2(lvls)
|
|
203
|
+
more_hierarchies.append(Hierarchy(lvls, hierarchy.name))
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
with self._get_db_connection() as conn:
|
|
207
|
+
with conn.cursor() as cur:
|
|
208
|
+
cur.execute(lowest_levels_query(self.fact_table))
|
|
209
|
+
result: list[tuple[any, ...]] = cur.fetchall()
|
|
210
|
+
candidate_keys: list[str] = [x[1] for x in result if x[0] == more_hierarchies[0].lowest_level.name]
|
|
211
|
+
fact_table_fk: str = [x for x in candidate_keys if x not in self.processed_keys][0]
|
|
212
|
+
self.processed_keys.append(fact_table_fk)
|
|
213
|
+
|
|
214
|
+
return Dimension(dimension_name, more_hierarchies, fact_table_fk)
|
|
215
|
+
|
|
216
|
+
def _attach_relations_to_levels2(self, hierarchy: list[Level]) -> list[Level]:
|
|
217
|
+
length: int = len(hierarchy)
|
|
218
|
+
for i, level in enumerate(hierarchy):
|
|
219
|
+
child = max(0, i - 1)
|
|
220
|
+
parent = min(length - 1, i + 1)
|
|
221
|
+
level.child = hierarchy[child]
|
|
222
|
+
level.parent = hierarchy[parent]
|
|
223
|
+
return hierarchy
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _attach_relations_to_levels(self, hierarchy_levels: list[list[Level]]) -> list[list[Level]]:
|
|
227
|
+
for levels in hierarchy_levels:
|
|
228
|
+
length: int = len(levels)
|
|
229
|
+
for i, level in enumerate(levels):
|
|
230
|
+
child = max(0, i - 1)
|
|
231
|
+
parent = min(length - 1, i + 1)
|
|
232
|
+
level.child = levels[child]
|
|
233
|
+
level.parent = levels[parent]
|
|
234
|
+
return hierarchy_levels
|
|
235
|
+
|
|
236
|
+
def _find_dsd_nodes(self, graph: Graph) -> list[Node]:
|
|
237
|
+
return list(graph.subjects(predicate=None, object=self.QB.DataStructureDefinition))
|
|
238
|
+
|
|
239
|
+
def _extract_name(self, node: Node) -> str:
|
|
240
|
+
node_str: str = node.toPython()
|
|
241
|
+
return node_str[node_str.find("#") + 1:]
|
|
242
|
+
|
|
243
|
+
def _get_db_connection(self) -> psyconn | None:
|
|
244
|
+
engine = self.engine
|
|
245
|
+
if isinstance(engine, Postgres):
|
|
246
|
+
return psycopg2.connect(user=engine.user,
|
|
247
|
+
password=engine.password,
|
|
248
|
+
host=engine.host,
|
|
249
|
+
port=engine.port,
|
|
250
|
+
database=engine.dbname)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class LevelContainer:
|
|
255
|
+
def __init__(self) -> None:
|
|
256
|
+
self.lvl_name: str = ""
|
|
257
|
+
self.lvl_attrs: list[str] = []
|
|
258
|
+
self.lvl_member_attr: str = ""
|
|
259
|
+
self.pk: str = ""
|
|
260
|
+
self.fk: str = ""
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class HierarchyContainer:
|
|
264
|
+
def __init__(self) -> None:
|
|
265
|
+
self.name: str = ""
|
|
266
|
+
self.levels: list[LevelContainer] = []
|
|
267
|
+
|
|
268
|
+
def __iter__(self) -> Iterator[LevelContainer]:
|
|
269
|
+
for i in range(len(self.levels)):
|
|
270
|
+
yield self.levels[i]
|