pydatacube 0.1.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. pydatacube-0.1.7/LICENSE +24 -0
  2. pydatacube-0.1.7/PKG-INFO +27 -0
  3. pydatacube-0.1.7/README.md +6 -0
  4. pydatacube-0.1.7/pyproject.toml +34 -0
  5. pydatacube-0.1.7/setup.cfg +4 -0
  6. pydatacube-0.1.7/src/pyDataCube/EntryPoint.py +245 -0
  7. pydatacube-0.1.7/src/pyDataCube/Enums.py +9 -0
  8. pydatacube-0.1.7/src/pyDataCube/Interface.py +33 -0
  9. pydatacube-0.1.7/src/pyDataCube/__init__.py +2 -0
  10. pydatacube-0.1.7/src/pyDataCube/converter/__init__.py +0 -0
  11. pydatacube-0.1.7/src/pyDataCube/converter/export.py +10 -0
  12. pydatacube-0.1.7/src/pyDataCube/converter/importer.py +270 -0
  13. pydatacube-0.1.7/src/pyDataCube/converter/qb_integrity_constraints.py +23 -0
  14. pydatacube-0.1.7/src/pyDataCube/converter/validator.py +28 -0
  15. pydatacube-0.1.7/src/pyDataCube/cube/AggregateFunction.py +14 -0
  16. pydatacube-0.1.7/src/pyDataCube/cube/Attribute.py +135 -0
  17. pydatacube-0.1.7/src/pyDataCube/cube/Axis.py +29 -0
  18. pydatacube-0.1.7/src/pyDataCube/cube/BaseCube.py +216 -0
  19. pydatacube-0.1.7/src/pyDataCube/cube/BooleanConnective.py +7 -0
  20. pydatacube-0.1.7/src/pyDataCube/cube/Cube.py +82 -0
  21. pydatacube-0.1.7/src/pyDataCube/cube/CubeOperators.py +146 -0
  22. pydatacube-0.1.7/src/pyDataCube/cube/Cuboid.py +127 -0
  23. pydatacube-0.1.7/src/pyDataCube/cube/Dimension.py +68 -0
  24. pydatacube-0.1.7/src/pyDataCube/cube/Hierarchy.py +53 -0
  25. pydatacube-0.1.7/src/pyDataCube/cube/Level.py +43 -0
  26. pydatacube-0.1.7/src/pyDataCube/cube/LevelMember.py +193 -0
  27. pydatacube-0.1.7/src/pyDataCube/cube/LevelMemberType.py +6 -0
  28. pydatacube-0.1.7/src/pyDataCube/cube/Measure.py +63 -0
  29. pydatacube-0.1.7/src/pyDataCube/cube/NonTopLevel.py +111 -0
  30. pydatacube-0.1.7/src/pyDataCube/cube/Predicate.py +47 -0
  31. pydatacube-0.1.7/src/pyDataCube/cube/PredicateOperator.py +9 -0
  32. pydatacube-0.1.7/src/pyDataCube/cube/TopLevel.py +9 -0
  33. pydatacube-0.1.7/src/pyDataCube/cube/View.py +390 -0
  34. pydatacube-0.1.7/src/pyDataCube/cube/__init__.py +0 -0
  35. pydatacube-0.1.7/src/pyDataCube/engines/__init__.py +3 -0
  36. pydatacube-0.1.7/src/pyDataCube/engines/engine.py +4 -0
  37. pydatacube-0.1.7/src/pyDataCube/engines/postgres.py +14 -0
  38. pydatacube-0.1.7/src/pyDataCube/session/__init__.py +0 -0
  39. pydatacube-0.1.7/src/pyDataCube/session/cube_metadata.py +111 -0
  40. pydatacube-0.1.7/src/pyDataCube/session/infer_cube.py +224 -0
  41. pydatacube-0.1.7/src/pyDataCube/session/session.py +98 -0
  42. pydatacube-0.1.7/src/pyDataCube/session/sql_queries.py +106 -0
  43. pydatacube-0.1.7/src/pyDataCube/timers.py +87 -0
  44. pydatacube-0.1.7/src/pydatacube.egg-info/PKG-INFO +27 -0
  45. pydatacube-0.1.7/src/pydatacube.egg-info/SOURCES.txt +53 -0
  46. pydatacube-0.1.7/src/pydatacube.egg-info/dependency_links.txt +1 -0
  47. pydatacube-0.1.7/src/pydatacube.egg-info/requires.txt +6 -0
  48. pydatacube-0.1.7/src/pydatacube.egg-info/top_level.txt +1 -0
  49. pydatacube-0.1.7/test/test_cube.py +682 -0
  50. pydatacube-0.1.7/test/test_cube_operators.py +228 -0
  51. pydatacube-0.1.7/test/test_infer_cube.py +44 -0
  52. pydatacube-0.1.7/test/test_level_member.py +44 -0
  53. pydatacube-0.1.7/test/test_non_top_level.py +48 -0
  54. pydatacube-0.1.7/test/test_regular_dimension.py +60 -0
  55. pydatacube-0.1.7/test/test_top_level.py +23 -0
@@ -0,0 +1,24 @@
1
+ This is free and unencumbered software released into the public domain.
2
+
3
+ Anyone is free to copy, modify, publish, use, compile, sell, or
4
+ distribute this software, either in source code form or as a compiled
5
+ binary, for any purpose, commercial or non-commercial, and by any
6
+ means.
7
+
8
+ In jurisdictions that recognize copyright laws, the author or authors
9
+ of this software dedicate any and all copyright interest in the
10
+ software to the public domain. We make this dedication for the benefit
11
+ of the public at large and to the detriment of our heirs and
12
+ successors. We intend this dedication to be an overt act of
13
+ relinquishment in perpetuity of all present and future rights to this
14
+ software under copyright law.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19
+ IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20
+ OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21
+ ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22
+ OTHER DEALINGS IN THE SOFTWARE.
23
+
24
+ For more information, please refer to <https://unlicense.org>
@@ -0,0 +1,27 @@
1
+ Metadata-Version: 2.4
2
+ Name: pydatacube
3
+ Version: 0.1.7
4
+ Summary: Querying data cubes with Python using MDX inspired declarative syntax
5
+ Author-email: Sigmundur Vang <siva@cs.aau.dk>
6
+ License-Expression: Unlicense
7
+ Project-URL: Homepage, https://gitlab.com/sigmundur_phd/pycube
8
+ Project-URL: Source, https://gitlab.com/sigmundur_phd/pycube.git
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Operating System :: OS Independent
11
+ Requires-Python: >=3.8
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE
14
+ Requires-Dist: numpy
15
+ Requires-Dist: psycopg2
16
+ Requires-Dist: pandas
17
+ Requires-Dist: levenshtein
18
+ Requires-Dist: rdflib
19
+ Requires-Dist: sqlalchemy
20
+ Dynamic: license-file
21
+
22
+ # pyDataCube
23
+ ## Running the experiments
24
+ Commit 1c84c423dc439e842a5906a86235c3e7439f0c7e was used for the experiments.
25
+ The timer object that measured the time spent in the database was injected into pandas' `read_sql` method (and its auxiliary methods) in `io/sql.py`.
26
+ The modified `sql.py` file is included in the base of this repository as `pandas_sql.py`.
27
+ This modified `sql.py` was, together with the version associated with the above commit hash, used for gaining the experiments results in the journal paper
@@ -0,0 +1,6 @@
1
+ # pyDataCube
2
+ ## Running the experiments
3
+ Commit 1c84c423dc439e842a5906a86235c3e7439f0c7e was used for the experiments.
4
+ The timer object that measured the time spent in the database was injected into pandas' `read_sql` method (and its auxiliary methods) in `io/sql.py`.
5
+ The modified `sql.py` file is included in the base of this repository as `pandas_sql.py`.
6
+ This modified `sql.py` was, together with the version associated with the above commit hash, used for gaining the experiments results in the journal paper
@@ -0,0 +1,34 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "pydatacube"
7
+ version = "0.1.7"
8
+ description = "Querying data cubes with Python using MDX inspired declarative syntax"
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = "Unlicense"
12
+ license-files = ["LICENSE"]
13
+ dependencies = [
14
+ "numpy",
15
+ "psycopg2",
16
+ "pandas",
17
+ "levenshtein",
18
+ "rdflib",
19
+ "sqlalchemy"
20
+ ]
21
+
22
+ urls = { "Homepage" = "https://gitlab.com/sigmundur_phd/pycube", "Source" = "https://gitlab.com/sigmundur_phd/pycube.git" }
23
+
24
+ authors = [
25
+ { name = "Sigmundur Vang", email = "siva@cs.aau.dk" }
26
+ ]
27
+
28
+ classifiers = [
29
+ "Programming Language :: Python :: 3",
30
+ "Operating System :: OS Independent"
31
+ ]
32
+
33
+ [tool.setuptools.packages.find]
34
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,245 @@
1
+ from __future__ import annotations
2
+
3
+ from src.pyDataCube.engines.postgres import postgres
4
+ from src.pyDataCube.session.session import *
5
+
6
+ DATABASE_USER = "sigmundur"
7
+ DATABASE_PASSWORD = ""
8
+ DATABASE_HOST = "127.0.0.1"
9
+ DATABASE_PORT = "5432"
10
+ DATABASE_NAME = "ssb_snowflake_dev"
11
+
12
+ postgres_engine: Postgres = postgres(DATABASE_NAME, DATABASE_USER, DATABASE_PASSWORD, DATABASE_HOST, DATABASE_PORT)
13
+ postgres = create_session(postgres_engine)
14
+ view = postgres.load_view('lineorder')
15
+ print(view.day.year.attributes)
16
+ print(view.measures)
17
+
18
+ # # converter = QB4OLAPExport(view.cube, "./output.ttl")
19
+ # # converter.export()
20
+ # importer = QB4OLAPImporter("output.ttl", postgres_engine)
21
+ # session = importer.importer()
22
+ # view2: View = session.load_view("lineorder")
23
+ # # validator = Validator(view.cube._metadata)
24
+ # # result = validator.validate_qb()
25
+
26
+ mere_view = view.columns(view.day.month.mo_month.members()) \
27
+ .rows(view.part.brand1.b_brand1.members()) \
28
+ .pages(view.day.day.d_daynuminweek.members())
29
+ result100 = mere_view.output()
30
+
31
+ hej = 1
32
+ # view42 = view.columns(view.day.month.mo_month.members())
33
+ # output42 = view42.output()
34
+ #
35
+ # view.lo_quantity.set_aggregate_function("avg")
36
+ #
37
+ # view9 = view.columns(view.day.year.y_year[1997].children())\
38
+ # .rows(view.part.category.ca_category.members())\
39
+ # .pages([view.customer.city.ci_city["UNITED ST9"]])\
40
+ # .where((view.day.day.d_daynuminmonth <= 7)
41
+ # & ((view.supplier.nation.n_nation == "MOZAMBIQUE")
42
+ # | (view.supplier.nation.n_nation == "RUSSIA")))\
43
+ # .using(view.lo_quantity,
44
+ # lost_income=view.lo_extendedprice * view.lo_discount)
45
+ #
46
+ # urslit = view9.output()
47
+ #
48
+ # test = view.day.year.y_year[1997]
49
+ # test2 = test.children()
50
+ # view.day.month.mo_month.set_sort_key(view.day.month.mo_yearmonth)
51
+ # test3 = test.children()
52
+ #
53
+ # hej = 1
54
+ #
55
+ mere_view = view.columns(view.day.month.mo_month.members()) \
56
+ .rows(view.part.category.ca_category.members()) \
57
+ .pages(view.day.year.y_year.members()) \
58
+ .using(view.lo_quantity, view.lo_tax)
59
+ result_uden_order = mere_view.output()
60
+
61
+ hej = 1
62
+ # view.day.month.mo_month.set_sort_key(view.day.month.mo_yearmonthnum)
63
+ #
64
+ # mere_view = view.columns(view.day.month.mo_month.members()) \
65
+ # .rows(view.part.category.ca_category.members()) \
66
+ # .pages(view.day.year.y_year.members()) \
67
+ # .using(view.lo_quantity, view.lo_tax)
68
+ # result_med_order = mere_view.output()
69
+ #
70
+ # hej = 1
71
+ #
72
+
73
+ city = view.customer.city.ci_city.members()[0]
74
+
75
+ new_view = view \
76
+ .axis(0, view.day.year.y_year[1992].children())\
77
+ .axis(1, view.part.category.ca_category.members())\
78
+ .axis(2, [view.customer.city.ci_city.members()[0]])\
79
+ .where((view.day.day.d_daynuminmonth <= 7)
80
+ & ((view.supplier.nation.n_nation == "ARGENTINA")
81
+ | (view.supplier.nation.n_nation == "FRANCE")))\
82
+ .using(view.lo_revenue,
83
+ lost_income=
84
+ view.lo_extendedprice * view.lo_discount)
85
+
86
+ # new_view2 = view \
87
+ # .axis(0, view.day.year.y_year[1992].children()) \
88
+ # .axis(1, view.part.category.ca_category.members()) \
89
+ # .axis(2, [city]) \
90
+ # .where((view.day.day.d_daynuminmonth <= 7)
91
+ # & ((view.supplier.nation.n_nation == "CANADA")
92
+ # | (view.supplier.nation.n_nation == "JAPAN"))) \
93
+ # .using(view.lo_revenue,
94
+ # lost_income=
95
+ # view.lo_extendedprice * view.lo_discount)
96
+
97
+ article_result = new_view.output()
98
+ # check_result = new_view2.output()
99
+
100
+ # # view.delete_measure(view.lo_revenue)
101
+ #
102
+ hej = 1
103
+ #
104
+ # # result2 = view.output()
105
+ #
106
+ # view9 = view.where(
107
+ # (view.day.year.y_year == 1993)
108
+ # & (view.lo_discount > 0)
109
+ # & (view.lo_discount < 4)
110
+ # & (view.lo_quantity < 25)) \
111
+ # .using(revenue=view.lo_extendedprice * view.lo_discount)
112
+ #
113
+ # result2 = view9.output()
114
+ #
115
+ #
116
+ # hej = 1
117
+ #
118
+ # # test = view.output()
119
+ #
120
+ # # test10 = view.part.category.ca_category["MFGR#44"]["MFGR#4427"]
121
+ # # test11 = view.day.day.d_dayofweek.Thursday
122
+ # #
123
+ # # test1 = view.day.month.mo_month.members()
124
+ # # test1 = view.day.year.y_year[1992].January
125
+ # # test2 = [x for xs in [i.children() for i in test1] for x in xs]
126
+ #
127
+ # view10 = view.columns(view.day.month.mo_month.members()) \
128
+ # .rows(view.part.part.p_name.members()) \
129
+ # .using(view.lo_revenue)
130
+ # result10 = view10.output()
131
+ #
132
+ # view3 = view.columns(view.day.year.y_year.members()) \
133
+ # .rows(view.part.brand1.b_brand1.members()) \
134
+ # .where((view.part.category.ca_category == "MFGR#12") &
135
+ # (view.supplier.region.r_region == "AMERICA")) \
136
+ # .using(view.lo_revenue, revenue=view.lo_extendedprice * view.lo_discount)
137
+ # result = view3.output()
138
+ #
139
+ # test = view.columns(view.customer.city.ci_city.members()) \
140
+ # .rows(view.supplier.city.ci_city.members()) \
141
+ # .pages(view.day.year.y_year.members()) \
142
+ # .using(view.lo_revenue)
143
+ # result2 = test.output()
144
+ #
145
+ # view4 = view.columns(view.day.month.mo_month.members())
146
+ # result3 = view4.output()
147
+ #
148
+ # hej = 1
149
+ #
150
+ # # ## ssb query 2.1
151
+ # # view.columns(view.date.year.y_year.members()) \
152
+ # # .rows(view.part.brand1.b_brand1.members()) \
153
+ # # .where(view.part.category.ca_category = "MFGR#12" &
154
+ # # view.supplier.region.r_region = "AMERICA") \
155
+ # # .measures(view.lo_revenue)
156
+ #
157
+ #
158
+ # # print()
159
+ # # print("Measures: ", cube.measures())
160
+ # # print("Dimensions: ", cube.dimensions())
161
+ # # print("Date hierarchy: ", cube.date.hierarchies())
162
+ # # print("Supplier name dictionary: ", cube.supplier.supplier_name.__dict__)
163
+ # # print("Date dimension dictionary: ", cube.date.__dict__)
164
+ # # print("Date year level dictionary: ", cube.date.date_year.__dict__)
165
+ # # print("2022 Level member: ", cube.date.date_year["2022"]["January"])
166
+ #
167
+ # # print("Output of the cube (cube): ", cube.output())
168
+ # # print("Date dimension current level: ", cube._dimension_list[1].current_level)
169
+ # # c1 = cube._drill_down(cube.date)
170
+ # # print("Date dimension current level: ", c1._dimension_list[1].current_level)
171
+ # # print("c1 dictionary: ", c1.__dict__)
172
+ # # print("Output of the cube (c1): ", c1.output())
173
+ #
174
+ # # c2 = c1._drill_down(c1.date)
175
+ # # print("Cube (c2) dictionary: ", c2.date.__dict__)
176
+ # # c3 = c2._slice(c2.date, c2.date.date_year._2022._January)
177
+ # # print(c3._dimension_list[1].__dict__)
178
+ #
179
+ #
180
+ # # print(c2._dimension_list[1].current_level)
181
+ # # c3 = c2._roll_up(c2.date)
182
+ # # print(c3._dimension_list[1].current_level)
183
+ # # c4 = c3._roll_up(c3.date)
184
+ # # print(c4._dimension_list[1].current_level)
185
+ #
186
+ #
187
+ # # print("Date dimension dictionary: ", cube.date.__dict__)
188
+ # # cube.date._roll_up()
189
+ # # print("Date dimension dictionary: ", cube.date.__dict__)
190
+ #
191
+ # # print("2022 Level member dictionary: ", cube.date.date_year._2022.__dict__)
192
+ # # print("January Level member dictionary: ", cube.date.date_year._2022._January.__dict__)
193
+ # # print("Day 1 Level member dictionary: ", cube.date.date_year._2022._January._1.__dict__)
194
+ #
195
+ # # ## Supplier testing
196
+ # # print("Supplier hierarchy: ", cube.supplier.hierarchies())
197
+ # # print("Nation dictionary: ", cube.supplier.supplier_nation.__dict__)
198
+ # # print("Denmark dictionary: ", cube.supplier.supplier_nation._Denmark.__dict__)
199
+ #
200
+ #
201
+ # # cube.columns([cube.date.date_year._2022._January])
202
+ # # print("The output: ", cube.output())
203
+ #
204
+ #
205
+ # # print()
206
+ # # print(cube.store.store_address["Jyllandsgade 1"])
207
+ # # print(cube.store.store_address["Jyllandsgade 2"])
208
+ # #
209
+ # # cube.where(cube.supplier.supplier_nation == "Denmark" and cube.store.store_address == "Jyllandsgade 1")
210
+ #
211
+ #
212
+ #
213
+ # # conn = psycopg2.connect(
214
+ # # dbname="salesdb_snowflake_test",
215
+ # # user="sigmundur",
216
+ # # password=""
217
+ # # )
218
+ #
219
+ #
220
+ # # # Cube -> View syntactic suger not implemented
221
+ # # view._axes = []
222
+ # # view = view.axis(0, view.cube.date.date_month.members())
223
+ #
224
+ #
225
+ # # output = view.output()
226
+ # # print(output)
227
+ # # with conn:
228
+ # # with conn.cursor() as curs:
229
+ # # curs.execute(output)
230
+ # # print(curs.fetchall())
231
+ #
232
+ #
233
+ # def generate_data():
234
+ # supplier = 1221
235
+ # store = 4444
236
+ # product = 9012
237
+ # total_sales_price = np.random.randint(10, high=1000, size=721)
238
+ # unit_sales = np.random.randint(1, high=20, size=721)
239
+ # sale_date = [i for i in range(1, 722)]
240
+ #
241
+ # with open("/home/sigmundur/test/test.csv", "w") as out:
242
+ # csv_out = csv.writer(out)
243
+ # for i in range(len(sale_date)):
244
+ # csv_out.writerow((supplier, store, product, sale_date[i], total_sales_price[i], unit_sales[i]))
245
+ #
@@ -0,0 +1,9 @@
1
+ from enum import Enum
2
+
3
+
4
+ class Backend(Enum):
5
+ MONDRIAN = 1
6
+
7
+
8
+
9
+
@@ -0,0 +1,33 @@
1
+ from abc import ABC, abstractmethod
2
+
3
+
4
+ class Interface(ABC):
5
+ pass
6
+
7
+
8
+ class CubeInterface(Interface):
9
+ # @abstractmethod
10
+ # def drill_out(self, dimension_name):
11
+ # pass
12
+
13
+ @abstractmethod
14
+ def columns(self, column):
15
+ pass
16
+
17
+ @abstractmethod
18
+ def rows(self, row):
19
+ pass
20
+
21
+ @abstractmethod
22
+ def where(self, **kwargs):
23
+ pass
24
+
25
+
26
+ class DimensionInterface(Interface):
27
+ @abstractmethod
28
+ def roll_up(self):
29
+ pass
30
+
31
+ @abstractmethod
32
+ def drill_down(self):
33
+ pass
@@ -0,0 +1,2 @@
1
+ from . import engines
2
+ from .session.session import create_session
File without changes
@@ -0,0 +1,10 @@
1
+ from ..cube.BaseCube import BaseCube
2
+
3
+
4
+ class QB4OLAPExport:
5
+ def __init__(self, cube: BaseCube, filename: str) -> None:
6
+ self.cube = cube
7
+ self.filename = filename
8
+
9
+ def export(self):
10
+ self.cube._metadata.serialize(destination=self.filename)
@@ -0,0 +1,270 @@
1
+ from typing import Iterator
2
+
3
+ import psycopg2
4
+ from psycopg2.extensions import connection as psyconn
5
+ from rdflib import Graph, Namespace, Node
6
+
7
+ from ..cube.Attribute import Attribute
8
+ from ..cube.Axis import Axis
9
+ from ..cube.BaseCube import BaseCube
10
+ from ..cube.Dimension import Dimension
11
+ from ..cube.Hierarchy import Hierarchy
12
+ from ..cube.Level import Level
13
+ from ..cube.LevelMember import LevelMember
14
+ from ..cube.TopLevel import TopLevel
15
+ from ..engines.postgres import Postgres
16
+ from ..engines.engine import Engine
17
+ from ..cube.AggregateFunction import AggregateFunction
18
+ from ..cube.Measure import Measure
19
+ from ..cube.NonTopLevel import NonTopLevel
20
+ from ..cube.View import View
21
+ from ..session.session import Session
22
+ from ..session.sql_queries import get_non_key_columns_query, get_pk_and_fk_columns_query, lowest_levels_query
23
+
24
+
25
+ class QB4OLAPImporter:
26
+ def __init__(self, filename: str, engine: Engine) -> None:
27
+ self.filename: str = filename
28
+ self.engine: Engine = engine
29
+ self.fact_table: str = ""
30
+ self.measures: list[Measure] = []
31
+ self.dimensions: list[Dimension] = []
32
+ self.views: list[View] = []
33
+ self.QB: Namespace = Namespace("http://purl.org/linked-data/cube#")
34
+ self.RDF: Namespace = Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
35
+ self.QB4O: Namespace = Namespace("http://purl.org/qb4olap/cubes#")
36
+ self.processed_dimensions: dict[Node, list[Node]] = {}
37
+ self.processed_keys: list[str] = []
38
+
39
+ def importer(self) -> Session:
40
+ g = Graph()
41
+ g.parse(self.filename, format="ttl")
42
+
43
+ dsd_nodes: list[Node] = self._find_dsd_nodes(g)
44
+ if not dsd_nodes:
45
+ raise Exception(f"No data structure definition found in {self.filename}")
46
+ else:
47
+ for dsd in dsd_nodes:
48
+ self._process_dsd(g, dsd)
49
+
50
+ return Session(self.views, self.engine)
51
+
52
+ def _process_dsd(self, g: Graph, dsd: Node) -> None:
53
+ self.fact_table = self._extract_name(dsd)
54
+ for _, _, component_specification in g.triples((dsd, self.QB.component, None)):
55
+ self._process_component_specification(g, component_specification)
56
+
57
+ for dimension in self.dimensions:
58
+ dimension.metadata = g
59
+
60
+ cube: BaseCube = BaseCube(self.fact_table, self.dimensions, self.measures, self.fact_table, g, self.engine)
61
+ cube.base_cube = cube
62
+ axes: list[Axis] = []
63
+ for dimension in self.dimensions:
64
+ hierarchy: Hierarchy = dimension.hierarchies[0]
65
+ level: NonTopLevel = hierarchy.lowest_level
66
+ attribute: Attribute = level.level_member_attr
67
+ level_members: list[LevelMember] = attribute.members()
68
+ axes.append(Axis(dimension, hierarchy, level, attribute, level_members))
69
+
70
+ measures: list[Measure] = cube.measure_list[0] if cube.measure_list else []
71
+ predicates: None = None
72
+ self.views.append(View(cube=cube, name=cube.name))
73
+
74
+ def _process_component_specification(self, g: Graph, comp_spec: Node) -> None:
75
+ for _, comp_prop, Comp_Prop in g.triples((comp_spec, None, None)):
76
+ match comp_prop:
77
+ case self.QB.measure:
78
+ self.measures.append(self._process_measure(g, comp_spec))
79
+ case self.QB4O.level:
80
+ self.dimensions.append(self._process_dimension(g, Comp_Prop))
81
+ case self.QB4O.hasAggregateFunction:
82
+ continue
83
+ case _:
84
+ print(f"Only considering {self.QB4O.measure} and {self.QB4O.level} properties in DataStructureDefinition. Found {_}")
85
+
86
+ def _process_measure(self, g: Graph, comp_spec: Node) -> Measure:
87
+ name: str = ""
88
+ sql: str = ""
89
+ agg_func: AggregateFunction | None = None
90
+ for _, comp_prop, Comp_Prop in g.triples((comp_spec, None, None)):
91
+ match comp_prop:
92
+ case self.QB.measure:
93
+ measure_str: str = Comp_Prop.toPython()
94
+ name = measure_str[measure_str.find("#") + 1:]
95
+ sql = f"{self.fact_table}.{name}"
96
+ case self.QB4O.hasAggregateFunction:
97
+ try:
98
+ agg_func = self._process_aggregate_function(Comp_Prop)
99
+ except NotImplementedError as e:
100
+ raise e
101
+ except Exception as e:
102
+ raise Exception(f"Unknown triple ({comp_spec, comp_prop, Comp_Prop})") from e
103
+ case _:
104
+ raise Exception(f"Unknown component property {comp_prop} leading Component Property {Comp_Prop} leading to component specification {comp_spec} (triple ({comp_spec}, {comp_prop}, {Comp_Prop}) is invalid)")
105
+ if not (name and sql and agg_func):
106
+ raise Exception(f"Incomplete measure specification {comp_spec}")
107
+ else:
108
+ return Measure(name, agg_func, sql)
109
+
110
+ def _process_aggregate_function(self, agg_func: Node) -> AggregateFunction:
111
+ func: str = agg_func.toPython()
112
+ name: str = func[func.find("#") + 1:]
113
+ match name:
114
+ case "sum":
115
+ return AggregateFunction("SUM", lambda x, y: x + y)
116
+ case "min":
117
+ return AggregateFunction("MIN", lambda x, y: x if x < y else y)
118
+ case "max":
119
+ return AggregateFunction("MAX", lambda x, y: x if x > y else y)
120
+ case "count":
121
+ return AggregateFunction("COUNT", lambda acc, x: acc + 1)
122
+ case "avg":
123
+ return AggregateFunction("AVG", lambda x, y: x)
124
+ case _:
125
+ raise Exception(f"Unknown aggregate function {agg_func}")
126
+
127
+ def _get_keys(self, lvl_name: str) -> tuple[str, str]:
128
+ with self._get_db_connection() as conn:
129
+ with conn.cursor() as cur:
130
+ cur.execute(get_pk_and_fk_columns_query(lvl_name))
131
+ pk: str
132
+ fk: str
133
+ pk, fk = "", ""
134
+ for t in cur.fetchall():
135
+ if t[1] == 'PRIMARY KEY':
136
+ pk = t[0]
137
+ else:
138
+ fk = t[0]
139
+ return pk, fk
140
+
141
+ def _get_level_member_attribute(self, lvl_name: str, lvl_attrs: list[str]) -> str:
142
+ with self._get_db_connection() as conn:
143
+ with conn.cursor() as cur:
144
+ cur.execute(get_non_key_columns_query(lvl_name))
145
+ all_attrs = list(map(lambda x: x[0], cur.fetchall()))
146
+
147
+ lvl_member_attrs: list[str] = [x for x in all_attrs if x not in lvl_attrs]
148
+ if len(lvl_member_attrs) > 1:
149
+ raise Exception(f"Found multiple level member attributes: {lvl_member_attrs} for level '{lvl_name}'")
150
+ if len(lvl_member_attrs) == 0:
151
+ raise Exception(f"Found no level member attribute for level '{lvl_name}'")
152
+ return lvl_member_attrs[0]
153
+
154
+ def _get_next_level(self, g: Graph, Comp_Prop: Node) -> list[Node]:
155
+ return [x[2] for x in g.triples((Comp_Prop, self.QB4O.parentLevel, None))]
156
+
157
+ def _process_dimension(self, g: Graph, Comp_Prop: Node) -> Dimension:
158
+ dim_props: list[Node] = list(g.objects(Comp_Prop, self.QB4O.inDimension))
159
+ if len(dim_props) > 1:
160
+ chosen_dimension: Node = [x for x in dim_props if x not in self.processed_dimensions.setdefault(Comp_Prop, [])][0]
161
+ self.processed_dimensions[Comp_Prop].append(chosen_dimension)
162
+ else:
163
+ chosen_dimension: Node = dim_props[0]
164
+ dimension_name: str = self._extract_name(chosen_dimension)
165
+
166
+ hierarchies: list[HierarchyContainer] = []
167
+ hierarchy_counter: int = 0
168
+ def traverse(path: list[Node]) -> None:
169
+ current_node = path[-1]
170
+ next_levels = self._get_next_level(g, current_node)
171
+
172
+ if not next_levels:
173
+ hierarchy_container = HierarchyContainer()
174
+ nonlocal hierarchy_counter
175
+ hierarchy_counter += 1
176
+ hierarchy_container.name = f"h{hierarchy_counter}"
177
+
178
+ for node in path:
179
+ container = LevelContainer()
180
+ container.lvl_name = self._extract_name(node)
181
+ container.lvl_attrs = list(map(lambda x: self._extract_name(x[2]),
182
+ g.triples((node, self.QB4O.hasAttribute, None))))
183
+ container.lvl_member_attr = self._get_level_member_attribute(container.lvl_name,
184
+ container.lvl_attrs)
185
+ container.pk, container.fk = self._get_keys(container.lvl_name)
186
+ hierarchy_container.levels.append(container)
187
+
188
+ hierarchies.append(hierarchy_container)
189
+ return
190
+
191
+ for next_node in next_levels:
192
+ traverse(path + [next_node])
193
+
194
+ traverse([Comp_Prop])
195
+
196
+ more_hierarchies: list[Hierarchy] = []
197
+ for hierarchy in hierarchies:
198
+ lvls: list[Level] = []
199
+ for level in hierarchy:
200
+ lvls.append(NonTopLevel(level.lvl_name, self.engine, level.lvl_attrs, level.pk, level.fk, level.lvl_member_attr))
201
+ lvls.append(TopLevel())
202
+ self._attach_relations_to_levels2(lvls)
203
+ more_hierarchies.append(Hierarchy(lvls, hierarchy.name))
204
+
205
+
206
+ with self._get_db_connection() as conn:
207
+ with conn.cursor() as cur:
208
+ cur.execute(lowest_levels_query(self.fact_table))
209
+ result: list[tuple[any, ...]] = cur.fetchall()
210
+ candidate_keys: list[str] = [x[1] for x in result if x[0] == more_hierarchies[0].lowest_level.name]
211
+ fact_table_fk: str = [x for x in candidate_keys if x not in self.processed_keys][0]
212
+ self.processed_keys.append(fact_table_fk)
213
+
214
+ return Dimension(dimension_name, more_hierarchies, fact_table_fk)
215
+
216
+ def _attach_relations_to_levels2(self, hierarchy: list[Level]) -> list[Level]:
217
+ length: int = len(hierarchy)
218
+ for i, level in enumerate(hierarchy):
219
+ child = max(0, i - 1)
220
+ parent = min(length - 1, i + 1)
221
+ level.child = hierarchy[child]
222
+ level.parent = hierarchy[parent]
223
+ return hierarchy
224
+
225
+
226
+ def _attach_relations_to_levels(self, hierarchy_levels: list[list[Level]]) -> list[list[Level]]:
227
+ for levels in hierarchy_levels:
228
+ length: int = len(levels)
229
+ for i, level in enumerate(levels):
230
+ child = max(0, i - 1)
231
+ parent = min(length - 1, i + 1)
232
+ level.child = levels[child]
233
+ level.parent = levels[parent]
234
+ return hierarchy_levels
235
+
236
+ def _find_dsd_nodes(self, graph: Graph) -> list[Node]:
237
+ return list(graph.subjects(predicate=None, object=self.QB.DataStructureDefinition))
238
+
239
+ def _extract_name(self, node: Node) -> str:
240
+ node_str: str = node.toPython()
241
+ return node_str[node_str.find("#") + 1:]
242
+
243
+ def _get_db_connection(self) -> psyconn | None:
244
+ engine = self.engine
245
+ if isinstance(engine, Postgres):
246
+ return psycopg2.connect(user=engine.user,
247
+ password=engine.password,
248
+ host=engine.host,
249
+ port=engine.port,
250
+ database=engine.dbname)
251
+
252
+
253
+
254
+ class LevelContainer:
255
+ def __init__(self) -> None:
256
+ self.lvl_name: str = ""
257
+ self.lvl_attrs: list[str] = []
258
+ self.lvl_member_attr: str = ""
259
+ self.pk: str = ""
260
+ self.fk: str = ""
261
+
262
+
263
+ class HierarchyContainer:
264
+ def __init__(self) -> None:
265
+ self.name: str = ""
266
+ self.levels: list[LevelContainer] = []
267
+
268
+ def __iter__(self) -> Iterator[LevelContainer]:
269
+ for i in range(len(self.levels)):
270
+ yield self.levels[i]