pydpm_xl 0.2.9__py3-none-any.whl → 0.2.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_dpm/__init__.py +1 -1
- py_dpm/api/dpm_xl/ast_generator.py +107 -47
- py_dpm/dpm/migration.py +40 -45
- py_dpm/dpm/models.py +22 -2
- py_dpm/dpm_xl/ast/operands.py +41 -32
- {pydpm_xl-0.2.9.dist-info → pydpm_xl-0.2.11.dist-info}/METADATA +1 -1
- {pydpm_xl-0.2.9.dist-info → pydpm_xl-0.2.11.dist-info}/RECORD +11 -11
- {pydpm_xl-0.2.9.dist-info → pydpm_xl-0.2.11.dist-info}/WHEEL +1 -1
- {pydpm_xl-0.2.9.dist-info → pydpm_xl-0.2.11.dist-info}/entry_points.txt +0 -0
- {pydpm_xl-0.2.9.dist-info → pydpm_xl-0.2.11.dist-info}/licenses/LICENSE +0 -0
- {pydpm_xl-0.2.9.dist-info → pydpm_xl-0.2.11.dist-info}/top_level.txt +0 -0
py_dpm/__init__.py
CHANGED
|
@@ -41,7 +41,7 @@ Available packages:
|
|
|
41
41
|
- pydpm.api: Main APIs for migration, syntax, and semantic analysis
|
|
42
42
|
"""
|
|
43
43
|
|
|
44
|
-
__version__ = "0.2.
|
|
44
|
+
__version__ = "0.2.11"
|
|
45
45
|
__author__ = "MeaningfulData S.L."
|
|
46
46
|
__email__ = "info@meaningfuldata.eu"
|
|
47
47
|
__license__ = "GPL-3.0-or-later"
|
|
@@ -984,6 +984,7 @@ class ASTGeneratorAPI:
|
|
|
984
984
|
"""
|
|
985
985
|
from py_dpm.dpm.utils import get_engine
|
|
986
986
|
from py_dpm.api.dpm import DataDictionaryAPI
|
|
987
|
+
from py_dpm.api.dpm_xl.operation_scopes import OperationScopesAPI
|
|
987
988
|
|
|
988
989
|
# Initialize database connection
|
|
989
990
|
engine = get_engine(database_path=self.database_path, connection_url=self.connection_url)
|
|
@@ -1017,6 +1018,12 @@ class ASTGeneratorAPI:
|
|
|
1017
1018
|
connection_url=self.connection_url
|
|
1018
1019
|
)
|
|
1019
1020
|
|
|
1021
|
+
# Initialize OperationScopesAPI once for all expressions (performance optimization)
|
|
1022
|
+
scopes_api = OperationScopesAPI(
|
|
1023
|
+
database_path=self.database_path,
|
|
1024
|
+
connection_url=self.connection_url
|
|
1025
|
+
)
|
|
1026
|
+
|
|
1020
1027
|
# Primary module info will be determined from the first expression or module_code
|
|
1021
1028
|
primary_module_info = None
|
|
1022
1029
|
namespace = None
|
|
@@ -1035,6 +1042,21 @@ class ASTGeneratorAPI:
|
|
|
1035
1042
|
complete_ast = complete_result["ast"]
|
|
1036
1043
|
context = complete_result.get("context") or table_context
|
|
1037
1044
|
|
|
1045
|
+
# Get tables with modules for this expression FIRST (reuse scopes_api from outer scope)
|
|
1046
|
+
# This is done before _get_primary_module_info to pass precomputed values
|
|
1047
|
+
tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
|
|
1048
|
+
expression=expression,
|
|
1049
|
+
release_id=release_id
|
|
1050
|
+
)
|
|
1051
|
+
|
|
1052
|
+
# Calculate scope_result once (avoid duplicate calls in other methods)
|
|
1053
|
+
scope_result = scopes_api.calculate_scopes_from_expression(
|
|
1054
|
+
expression=expression,
|
|
1055
|
+
release_id=release_id,
|
|
1056
|
+
read_only=True
|
|
1057
|
+
)
|
|
1058
|
+
all_tables_with_modules.extend(tables_with_modules)
|
|
1059
|
+
|
|
1038
1060
|
# Get primary module info from first expression (or use module_code)
|
|
1039
1061
|
if primary_module_info is None:
|
|
1040
1062
|
primary_module_info = self._get_primary_module_info(
|
|
@@ -1042,6 +1064,9 @@ class ASTGeneratorAPI:
|
|
|
1042
1064
|
primary_module_vid=primary_module_vid,
|
|
1043
1065
|
release_id=release_id,
|
|
1044
1066
|
module_code=module_code,
|
|
1067
|
+
# Performance optimization: pass precomputed values
|
|
1068
|
+
tables_with_modules=tables_with_modules,
|
|
1069
|
+
scopes_api=scopes_api,
|
|
1045
1070
|
)
|
|
1046
1071
|
namespace = primary_module_info.get("module_uri", "default_module")
|
|
1047
1072
|
|
|
@@ -1066,18 +1091,6 @@ class ASTGeneratorAPI:
|
|
|
1066
1091
|
# Clean extra fields from data entries
|
|
1067
1092
|
self._clean_ast_data_entries(ast_with_coords)
|
|
1068
1093
|
|
|
1069
|
-
# Get tables with modules for this expression
|
|
1070
|
-
from py_dpm.api.dpm_xl.operation_scopes import OperationScopesAPI
|
|
1071
|
-
scopes_api = OperationScopesAPI(
|
|
1072
|
-
database_path=self.database_path,
|
|
1073
|
-
connection_url=self.connection_url
|
|
1074
|
-
)
|
|
1075
|
-
tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
|
|
1076
|
-
expression=expression,
|
|
1077
|
-
release_id=release_id
|
|
1078
|
-
)
|
|
1079
|
-
all_tables_with_modules.extend(tables_with_modules)
|
|
1080
|
-
|
|
1081
1094
|
# Build mapping of table_code -> module_vid
|
|
1082
1095
|
# Prefer the module VID that matches the detected primary module
|
|
1083
1096
|
table_to_module = {}
|
|
@@ -1179,6 +1192,10 @@ class ASTGeneratorAPI:
|
|
|
1179
1192
|
operation_code=operation_code,
|
|
1180
1193
|
release_id=release_id,
|
|
1181
1194
|
preferred_module_dependencies=preferred_module_dependencies,
|
|
1195
|
+
# Performance optimization: pass precomputed values to avoid redundant work
|
|
1196
|
+
tables_with_modules=tables_with_modules,
|
|
1197
|
+
scopes_api=scopes_api,
|
|
1198
|
+
scope_result=scope_result,
|
|
1182
1199
|
)
|
|
1183
1200
|
|
|
1184
1201
|
# Merge dependency modules (avoid table duplicates)
|
|
@@ -1313,6 +1330,8 @@ class ASTGeneratorAPI:
|
|
|
1313
1330
|
primary_module_vid: Optional[int],
|
|
1314
1331
|
release_id: Optional[int],
|
|
1315
1332
|
module_code: Optional[str] = None,
|
|
1333
|
+
tables_with_modules: Optional[List[Dict[str, Any]]] = None,
|
|
1334
|
+
scopes_api: Optional[Any] = None,
|
|
1316
1335
|
) -> Dict[str, Any]:
|
|
1317
1336
|
"""
|
|
1318
1337
|
Detect and return metadata for the primary module from the expression.
|
|
@@ -1323,6 +1342,10 @@ class ASTGeneratorAPI:
|
|
|
1323
1342
|
release_id: Optional release ID for filtering
|
|
1324
1343
|
module_code: Optional module code (e.g., "FINREP9") - takes precedence over
|
|
1325
1344
|
primary_module_vid if provided
|
|
1345
|
+
tables_with_modules: Optional precomputed tables with module metadata
|
|
1346
|
+
(performance optimization to avoid redundant database queries)
|
|
1347
|
+
scopes_api: Optional precomputed OperationScopesAPI instance
|
|
1348
|
+
(performance optimization to reuse database connections)
|
|
1326
1349
|
|
|
1327
1350
|
Returns:
|
|
1328
1351
|
Dict with module_uri, module_code, module_version, framework_code,
|
|
@@ -1341,20 +1364,28 @@ class ASTGeneratorAPI:
|
|
|
1341
1364
|
"module_vid": None,
|
|
1342
1365
|
}
|
|
1343
1366
|
|
|
1367
|
+
# Track if we created the scopes_api locally (need to close it)
|
|
1368
|
+
local_scopes_api = False
|
|
1369
|
+
|
|
1344
1370
|
try:
|
|
1345
|
-
scopes_api
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1371
|
+
# Reuse provided scopes_api or create a new one
|
|
1372
|
+
if scopes_api is None:
|
|
1373
|
+
scopes_api = OperationScopesAPI(
|
|
1374
|
+
database_path=self.database_path,
|
|
1375
|
+
connection_url=self.connection_url
|
|
1376
|
+
)
|
|
1377
|
+
local_scopes_api = True
|
|
1349
1378
|
|
|
1350
|
-
#
|
|
1351
|
-
tables_with_modules
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1379
|
+
# Reuse provided tables_with_modules or fetch if not available
|
|
1380
|
+
if tables_with_modules is None:
|
|
1381
|
+
tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
|
|
1382
|
+
expression=expression,
|
|
1383
|
+
release_id=release_id
|
|
1384
|
+
)
|
|
1355
1385
|
|
|
1356
1386
|
if not tables_with_modules:
|
|
1357
|
-
|
|
1387
|
+
if local_scopes_api:
|
|
1388
|
+
scopes_api.close()
|
|
1358
1389
|
return default_info
|
|
1359
1390
|
|
|
1360
1391
|
# Determine primary module
|
|
@@ -1408,7 +1439,8 @@ class ASTGeneratorAPI:
|
|
|
1408
1439
|
to_date = module.get("to_reference_date", to_date)
|
|
1409
1440
|
break
|
|
1410
1441
|
|
|
1411
|
-
|
|
1442
|
+
if local_scopes_api:
|
|
1443
|
+
scopes_api.close()
|
|
1412
1444
|
|
|
1413
1445
|
return {
|
|
1414
1446
|
"module_uri": module_uri or "default_module",
|
|
@@ -1864,6 +1896,9 @@ class ASTGeneratorAPI:
|
|
|
1864
1896
|
operation_code: str,
|
|
1865
1897
|
release_id: Optional[int] = None,
|
|
1866
1898
|
preferred_module_dependencies: Optional[List[str]] = None,
|
|
1899
|
+
tables_with_modules: Optional[List[Dict[str, Any]]] = None,
|
|
1900
|
+
scopes_api: Optional[Any] = None,
|
|
1901
|
+
scope_result: Optional[Any] = None,
|
|
1867
1902
|
) -> tuple:
|
|
1868
1903
|
"""
|
|
1869
1904
|
Detect cross-module dependencies for a single expression.
|
|
@@ -1879,6 +1914,12 @@ class ASTGeneratorAPI:
|
|
|
1879
1914
|
release_id: Optional release ID for filtering
|
|
1880
1915
|
preferred_module_dependencies: Optional list of module codes to prefer when
|
|
1881
1916
|
a table belongs to multiple modules
|
|
1917
|
+
tables_with_modules: Optional precomputed tables with module metadata
|
|
1918
|
+
(performance optimization to avoid redundant database queries)
|
|
1919
|
+
scopes_api: Optional precomputed OperationScopesAPI instance
|
|
1920
|
+
(performance optimization to reuse database connections)
|
|
1921
|
+
scope_result: Optional precomputed scope result from calculate_scopes_from_expression
|
|
1922
|
+
(performance optimization to avoid redundant computation)
|
|
1882
1923
|
|
|
1883
1924
|
Returns:
|
|
1884
1925
|
Tuple of (dependency_modules, cross_instance_dependencies)
|
|
@@ -1889,24 +1930,28 @@ class ASTGeneratorAPI:
|
|
|
1889
1930
|
from py_dpm.dpm.queries.explorer_queries import ExplorerQuery
|
|
1890
1931
|
import logging
|
|
1891
1932
|
|
|
1892
|
-
scopes_api
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1933
|
+
# Reuse provided scopes_api or create a new one
|
|
1934
|
+
if scopes_api is None:
|
|
1935
|
+
scopes_api = OperationScopesAPI(
|
|
1936
|
+
database_path=self.database_path,
|
|
1937
|
+
connection_url=self.connection_url
|
|
1938
|
+
)
|
|
1896
1939
|
|
|
1897
1940
|
try:
|
|
1898
|
-
#
|
|
1899
|
-
tables_with_modules
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1941
|
+
# Reuse provided tables_with_modules or fetch if not available
|
|
1942
|
+
if tables_with_modules is None:
|
|
1943
|
+
tables_with_modules = scopes_api.get_tables_with_metadata_from_expression(
|
|
1944
|
+
expression=expression,
|
|
1945
|
+
release_id=release_id
|
|
1946
|
+
)
|
|
1903
1947
|
|
|
1904
|
-
#
|
|
1905
|
-
scope_result
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1948
|
+
# Reuse provided scope_result or compute if not available
|
|
1949
|
+
if scope_result is None:
|
|
1950
|
+
scope_result = scopes_api.calculate_scopes_from_expression(
|
|
1951
|
+
expression=expression,
|
|
1952
|
+
release_id=release_id,
|
|
1953
|
+
read_only=True
|
|
1954
|
+
)
|
|
1910
1955
|
|
|
1911
1956
|
if scope_result.has_error or not scope_result.is_cross_module:
|
|
1912
1957
|
return {}, []
|
|
@@ -2161,12 +2206,11 @@ class ASTGeneratorAPI:
|
|
|
2161
2206
|
entry["x"] = x_index
|
|
2162
2207
|
|
|
2163
2208
|
# Calculate y coordinate (column position)
|
|
2209
|
+
# Only add y if there are multiple columns
|
|
2164
2210
|
if cols and col_code in cols:
|
|
2165
2211
|
y_index = cols.index(col_code) + 1
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
# Default to 1 if no column info
|
|
2169
|
-
entry["y"] = 1
|
|
2212
|
+
if len(cols) > 1:
|
|
2213
|
+
entry["y"] = y_index
|
|
2170
2214
|
|
|
2171
2215
|
# Calculate z coordinate (sheet position)
|
|
2172
2216
|
if sheets and sheet_code in sheets:
|
|
@@ -2193,13 +2237,16 @@ class ASTGeneratorAPI:
|
|
|
2193
2237
|
Remove extra fields from data entries in the AST.
|
|
2194
2238
|
|
|
2195
2239
|
Keeps only the fields required by the engine:
|
|
2196
|
-
- datapoint, operand_reference_id
|
|
2240
|
+
- datapoint, operand_reference_id (always)
|
|
2241
|
+
- x and row (only if multiple rows - rows are variable)
|
|
2242
|
+
- y and column (only if multiple columns - columns are variable)
|
|
2243
|
+
- z and sheet (only if multiple sheets - sheets are variable)
|
|
2197
2244
|
|
|
2198
2245
|
Removes internal/debug fields:
|
|
2199
|
-
- data_type, cell_code, table_code, table_vid
|
|
2246
|
+
- data_type, cell_code, table_code, table_vid
|
|
2200
2247
|
"""
|
|
2201
|
-
#
|
|
2202
|
-
|
|
2248
|
+
# Base fields to always keep in data entries
|
|
2249
|
+
BASE_FIELDS = {"datapoint", "operand_reference_id"}
|
|
2203
2250
|
|
|
2204
2251
|
def clean_node(node):
|
|
2205
2252
|
if isinstance(node, dict):
|
|
@@ -2207,9 +2254,22 @@ class ASTGeneratorAPI:
|
|
|
2207
2254
|
if node.get("class_name") == "VarID" and "data" in node:
|
|
2208
2255
|
cleaned_data = []
|
|
2209
2256
|
for data_entry in node["data"]:
|
|
2257
|
+
# Build allowed fields based on which coordinates are present
|
|
2258
|
+
# Only keep row/column/sheet if the corresponding x/y/z coordinate exists
|
|
2259
|
+
allowed = set(BASE_FIELDS)
|
|
2260
|
+
if "x" in data_entry:
|
|
2261
|
+
allowed.add("x")
|
|
2262
|
+
allowed.add("row")
|
|
2263
|
+
if "y" in data_entry:
|
|
2264
|
+
allowed.add("y")
|
|
2265
|
+
allowed.add("column")
|
|
2266
|
+
if "z" in data_entry:
|
|
2267
|
+
allowed.add("z")
|
|
2268
|
+
allowed.add("sheet")
|
|
2269
|
+
|
|
2210
2270
|
# Keep only allowed fields
|
|
2211
2271
|
cleaned_entry = {
|
|
2212
|
-
k: v for k, v in data_entry.items() if k in
|
|
2272
|
+
k: v for k, v in data_entry.items() if k in allowed
|
|
2213
2273
|
}
|
|
2214
2274
|
cleaned_data.append(cleaned_entry)
|
|
2215
2275
|
node["data"] = cleaned_data
|
py_dpm/dpm/migration.py
CHANGED
|
@@ -103,14 +103,16 @@ def _extract_with_pyodbc(access_file):
|
|
|
103
103
|
import pyodbc
|
|
104
104
|
except ImportError:
|
|
105
105
|
raise Exception("pyodbc not available")
|
|
106
|
-
|
|
106
|
+
|
|
107
|
+
import decimal
|
|
108
|
+
|
|
107
109
|
# Try different Access drivers
|
|
108
110
|
drivers_to_try = [
|
|
109
111
|
r'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};',
|
|
110
112
|
r'DRIVER={Microsoft Access Driver (*.mdb)};',
|
|
111
113
|
r'DRIVER={MDBTools};'
|
|
112
114
|
]
|
|
113
|
-
|
|
115
|
+
|
|
114
116
|
conn = None
|
|
115
117
|
for driver in drivers_to_try:
|
|
116
118
|
try:
|
|
@@ -120,10 +122,10 @@ def _extract_with_pyodbc(access_file):
|
|
|
120
122
|
break
|
|
121
123
|
except pyodbc.Error:
|
|
122
124
|
continue
|
|
123
|
-
|
|
125
|
+
|
|
124
126
|
if not conn:
|
|
125
127
|
raise Exception("No suitable ODBC driver found for Access database")
|
|
126
|
-
|
|
128
|
+
|
|
127
129
|
try:
|
|
128
130
|
# Get all table names
|
|
129
131
|
cursor = conn.cursor()
|
|
@@ -132,63 +134,56 @@ def _extract_with_pyodbc(access_file):
|
|
|
132
134
|
table_name = table_info.table_name
|
|
133
135
|
if not table_name.startswith('MSys'): # Skip system tables
|
|
134
136
|
tables.append(table_name)
|
|
135
|
-
|
|
137
|
+
|
|
136
138
|
data = {}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
+
|
|
139
140
|
# Extract each table
|
|
140
141
|
for table_name in tables:
|
|
141
142
|
print(table_name)
|
|
142
143
|
try:
|
|
143
144
|
cursor.execute(f"SELECT * FROM [{table_name}]")
|
|
144
|
-
|
|
145
|
+
|
|
146
|
+
# Get column metadata from cursor.description
|
|
147
|
+
# Each entry is: (name, type_code, display_size, internal_size, precision, scale, null_ok)
|
|
148
|
+
# type_code is a Python type (str, int, float, decimal.Decimal, etc.)
|
|
149
|
+
column_info = []
|
|
150
|
+
for col_desc in cursor.description:
|
|
151
|
+
col_name = col_desc[0]
|
|
152
|
+
col_type = col_desc[1] # Python type from ODBC metadata
|
|
153
|
+
column_info.append((col_name, col_type))
|
|
154
|
+
|
|
155
|
+
columns = [info[0] for info in column_info]
|
|
145
156
|
rows = cursor.fetchall()
|
|
146
|
-
|
|
157
|
+
|
|
147
158
|
if rows:
|
|
148
159
|
# Convert to DataFrame
|
|
149
160
|
df = pd.DataFrame([list(row) for row in rows], columns=columns)
|
|
150
161
|
|
|
151
|
-
#
|
|
152
|
-
#
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
#
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
# Test numeric conversion
|
|
169
|
-
numeric_series = pd.to_numeric(df[column], errors='coerce')
|
|
170
|
-
|
|
171
|
-
if not has_leading_zeros and not numeric_series.isna().all():
|
|
172
|
-
numeric_columns.append(column)
|
|
173
|
-
except Exception:
|
|
174
|
-
continue
|
|
175
|
-
|
|
176
|
-
# Convert only the identified numeric columns
|
|
177
|
-
for col in numeric_columns:
|
|
178
|
-
try:
|
|
179
|
-
df[col] = pd.to_numeric(df[col], errors='coerce')
|
|
180
|
-
except (ValueError, TypeError):
|
|
181
|
-
# Keep as string if conversion fails
|
|
182
|
-
pass
|
|
183
|
-
|
|
162
|
+
# Use the actual column types from Access schema metadata
|
|
163
|
+
# instead of inferring from data values (fixes Windows vs Linux inconsistency)
|
|
164
|
+
numeric_types = (int, float, decimal.Decimal)
|
|
165
|
+
|
|
166
|
+
for col_name, col_type in column_info:
|
|
167
|
+
if col_type in numeric_types:
|
|
168
|
+
# Column is defined as numeric in Access schema - convert to numeric
|
|
169
|
+
try:
|
|
170
|
+
df[col_name] = pd.to_numeric(df[col_name], errors='coerce')
|
|
171
|
+
except (ValueError, TypeError):
|
|
172
|
+
pass
|
|
173
|
+
else:
|
|
174
|
+
# Column is defined as text/other in Access schema - keep as string
|
|
175
|
+
df[col_name] = df[col_name].astype(object)
|
|
176
|
+
mask = df[col_name].notna()
|
|
177
|
+
df.loc[mask, col_name] = df.loc[mask, col_name].astype(str)
|
|
178
|
+
|
|
184
179
|
data[table_name] = df
|
|
185
|
-
|
|
180
|
+
|
|
186
181
|
except Exception as e:
|
|
187
182
|
print(f"Error processing table {table_name}: {e}", file=sys.stderr)
|
|
188
183
|
continue
|
|
189
|
-
|
|
184
|
+
|
|
190
185
|
return data
|
|
191
|
-
|
|
186
|
+
|
|
192
187
|
finally:
|
|
193
188
|
conn.close()
|
|
194
189
|
|
py_dpm/dpm/models.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
|
-
from typing import List
|
|
2
|
+
from typing import Dict, Hashable, List, Tuple
|
|
3
3
|
from sqlalchemy import (
|
|
4
4
|
Boolean,
|
|
5
5
|
Column,
|
|
@@ -36,6 +36,11 @@ class SerializationMixin:
|
|
|
36
36
|
Base = declarative_base(cls=SerializationMixin)
|
|
37
37
|
|
|
38
38
|
|
|
39
|
+
def _get_engine_cache_key(session) -> Hashable:
|
|
40
|
+
bind = session.get_bind()
|
|
41
|
+
return getattr(bind, "url", repr(bind))
|
|
42
|
+
|
|
43
|
+
|
|
39
44
|
def _read_sql_with_connection(sql, session):
|
|
40
45
|
"""
|
|
41
46
|
Execute pd.read_sql with proper connection handling to avoid pandas warnings.
|
|
@@ -2319,6 +2324,11 @@ class ViewDatapoints(Base):
|
|
|
2319
2324
|
context_id = Column(Integer)
|
|
2320
2325
|
variable_vid = Column(String)
|
|
2321
2326
|
|
|
2327
|
+
_TABLE_DATA_CACHE: Dict[
|
|
2328
|
+
Tuple[Hashable, str, Tuple[str, ...] | None, Tuple[str, ...] | None, Tuple[str, ...] | None, int | None],
|
|
2329
|
+
pd.DataFrame,
|
|
2330
|
+
] = {}
|
|
2331
|
+
|
|
2322
2332
|
@classmethod
|
|
2323
2333
|
def _create_base_query_with_aliases(cls, session):
|
|
2324
2334
|
"""
|
|
@@ -2552,7 +2562,16 @@ class ViewDatapoints(Base):
|
|
|
2552
2562
|
def get_table_data(
|
|
2553
2563
|
cls, session, table, rows=None, columns=None, sheets=None, release_id=None
|
|
2554
2564
|
):
|
|
2555
|
-
|
|
2565
|
+
engine_key = _get_engine_cache_key(session)
|
|
2566
|
+
rows_key = tuple(rows) if rows is not None else None
|
|
2567
|
+
columns_key = tuple(columns) if columns is not None else None
|
|
2568
|
+
sheets_key = tuple(sheets) if sheets is not None else None
|
|
2569
|
+
cache_key = (engine_key, table, rows_key, columns_key, sheets_key, release_id)
|
|
2570
|
+
|
|
2571
|
+
cached = cls._TABLE_DATA_CACHE.get(cache_key)
|
|
2572
|
+
if cached is not None:
|
|
2573
|
+
return cached
|
|
2574
|
+
|
|
2556
2575
|
query, aliases = cls._create_base_query_with_aliases(session)
|
|
2557
2576
|
|
|
2558
2577
|
# Add column selections
|
|
@@ -2669,6 +2688,7 @@ class ViewDatapoints(Base):
|
|
|
2669
2688
|
data = _check_ranges_values_are_present(data, "column_code", columns)
|
|
2670
2689
|
data = _check_ranges_values_are_present(data, "sheet_code", sheets)
|
|
2671
2690
|
|
|
2691
|
+
cls._TABLE_DATA_CACHE[cache_key] = data
|
|
2672
2692
|
return data
|
|
2673
2693
|
|
|
2674
2694
|
@classmethod
|
py_dpm/dpm_xl/ast/operands.py
CHANGED
|
@@ -2,6 +2,7 @@ from abc import ABC
|
|
|
2
2
|
|
|
3
3
|
import pandas as pd
|
|
4
4
|
import warnings
|
|
5
|
+
from typing import Dict, Hashable, Tuple
|
|
5
6
|
|
|
6
7
|
# Suppress pandas UserWarning about SQLAlchemy connection types
|
|
7
8
|
warnings.filterwarnings("ignore", message=".*pandas only supports SQLAlchemy.*")
|
|
@@ -43,6 +44,9 @@ from py_dpm.dpm_xl.utils.data_handlers import filter_all_data
|
|
|
43
44
|
operand_elements = ["table", "rows", "cols", "sheets", "default", "interval"]
|
|
44
45
|
|
|
45
46
|
|
|
47
|
+
_HEADERS_CACHE: Dict[Tuple[Hashable, int, Tuple[str, ...]], pd.DataFrame] = {}
|
|
48
|
+
|
|
49
|
+
|
|
46
50
|
def _create_operand_label(node):
|
|
47
51
|
label = generate_new_label()
|
|
48
52
|
node.label = label
|
|
@@ -185,42 +189,47 @@ class OperandsChecking(ASTTemplate, ABC):
|
|
|
185
189
|
if len(table_codes) == 0:
|
|
186
190
|
return
|
|
187
191
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
192
|
+
engine = self.session.get_bind()
|
|
193
|
+
engine_key: Hashable = getattr(engine, "url", repr(engine))
|
|
194
|
+
cache_key = (engine_key, self.release_id, tuple(sorted(table_codes)))
|
|
195
|
+
|
|
196
|
+
df_headers = _HEADERS_CACHE.get(cache_key)
|
|
197
|
+
if df_headers is None:
|
|
198
|
+
query = (
|
|
199
|
+
self.session.query(
|
|
200
|
+
TableVersion.code.label("Code"),
|
|
201
|
+
TableVersion.startreleaseid.label("StartReleaseID"),
|
|
202
|
+
TableVersion.endreleaseid.label("EndReleaseID"),
|
|
203
|
+
Header.direction.label("Direction"),
|
|
204
|
+
Table.hasopenrows.label("HasOpenRows"),
|
|
205
|
+
Table.hasopencolumns.label("HasOpenColumns"),
|
|
206
|
+
Table.hasopensheets.label("HasOpenSheets"),
|
|
207
|
+
)
|
|
208
|
+
.join(Table, Table.tableid == TableVersion.tableid)
|
|
209
|
+
.join(
|
|
210
|
+
TableVersionHeader,
|
|
211
|
+
TableVersion.tablevid == TableVersionHeader.tablevid,
|
|
212
|
+
)
|
|
213
|
+
.join(Header, Header.headerid == TableVersionHeader.headerid)
|
|
214
|
+
.filter(TableVersion.code.in_(table_codes))
|
|
215
|
+
.distinct()
|
|
202
216
|
)
|
|
203
|
-
.join(Header, Header.headerid == TableVersionHeader.headerid)
|
|
204
|
-
.filter(TableVersion.code.in_(table_codes))
|
|
205
|
-
.distinct()
|
|
206
|
-
)
|
|
207
217
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
)
|
|
218
|
+
query = filter_by_release(
|
|
219
|
+
query,
|
|
220
|
+
start_col=TableVersion.startreleaseid,
|
|
221
|
+
end_col=TableVersion.endreleaseid,
|
|
222
|
+
release_id=self.release_id,
|
|
223
|
+
)
|
|
215
224
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
)
|
|
225
|
+
from py_dpm.dpm.models import (
|
|
226
|
+
_compile_query_for_pandas,
|
|
227
|
+
_read_sql_with_connection,
|
|
228
|
+
)
|
|
221
229
|
|
|
222
|
-
|
|
223
|
-
|
|
230
|
+
compiled_query = _compile_query_for_pandas(query.statement, self.session)
|
|
231
|
+
df_headers = _read_sql_with_connection(compiled_query, self.session)
|
|
232
|
+
_HEADERS_CACHE[cache_key] = df_headers
|
|
224
233
|
|
|
225
234
|
for table in table_codes:
|
|
226
235
|
table_headers = df_headers[df_headers["Code"] == table]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
py_dpm/__init__.py,sha256=
|
|
1
|
+
py_dpm/__init__.py,sha256=mCQT4E9AM2a8Gg_V4tbxaQkfJ_CB9uUJR0fH7DR9uVI,1859
|
|
2
2
|
py_dpm/api/__init__.py,sha256=bPWo4KWW99GmGkQbq0yQ3_tbTxvidEVVrWmcDXWz-xA,828
|
|
3
3
|
py_dpm/api/dpm/__init__.py,sha256=HQflgiRbs1eDi3KTadNhxS1NoaG6PGQDVMvFnuIEfXo,506
|
|
4
4
|
py_dpm/api/dpm/data_dictionary.py,sha256=q6w_5bFdc6WPd5Z601PpDaCcnIw39CnI4wdby3GJmFU,29893
|
|
@@ -7,7 +7,7 @@ py_dpm/api/dpm/hierarchical_queries.py,sha256=X4AbpsWy3iItOTVIdVbtaTmRgOHPf0Y64I
|
|
|
7
7
|
py_dpm/api/dpm/instance.py,sha256=v3DWzdaM5gPCecLjwjZ49FGfqZzUR3dPC0U8zGwdttk,3795
|
|
8
8
|
py_dpm/api/dpm/migration.py,sha256=9FT7zzz4QdUIRR6MD01gMODBtfq9HH_RF4hRgZqMcZc,2404
|
|
9
9
|
py_dpm/api/dpm_xl/__init__.py,sha256=SOsCtwxuCkFQIdYoTcU6tVJdK1j96dPF4ZQpr8okLPc,576
|
|
10
|
-
py_dpm/api/dpm_xl/ast_generator.py,sha256=
|
|
10
|
+
py_dpm/api/dpm_xl/ast_generator.py,sha256=53Hq1I2Uey6VqNmehU-6vSjyUnfU56GhcJKZuNZAEFw,98334
|
|
11
11
|
py_dpm/api/dpm_xl/complete_ast.py,sha256=jjn9tsns4X7EN4RV5mFh5DRxxZfBOuX2b1wic4oB_ls,3967
|
|
12
12
|
py_dpm/api/dpm_xl/operation_scopes.py,sha256=v2t3f2-AiF39hspNtpf_PA94T69JqbymFK9a5hpckRs,48831
|
|
13
13
|
py_dpm/api/dpm_xl/semantic.py,sha256=Ddmh2Wj_iXIpQZ4jCjqOI-6ddFCquaO9RTu6W9i1Rts,13968
|
|
@@ -16,8 +16,8 @@ py_dpm/cli/__init__.py,sha256=UrfGHoQ0sZLjWfA0hoOoI4iTrn-bjr2f9Q8wDWd5nMo,133
|
|
|
16
16
|
py_dpm/cli/main.py,sha256=LJ7JBk7lyWXe7ZYxnbxmohM1Dbha4sIdQzSTYKd9ZNo,22457
|
|
17
17
|
py_dpm/cli/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
18
|
py_dpm/dpm/__init__.py,sha256=moagUo5Gxf24-Tl9FL_3n2wmVoD_oXtpC-YIGktH_rc,212
|
|
19
|
-
py_dpm/dpm/migration.py,sha256=
|
|
20
|
-
py_dpm/dpm/models.py,sha256=
|
|
19
|
+
py_dpm/dpm/migration.py,sha256=I_OX1xS2eE3L2V1RAPyHOqFn3Z54NWE0ozLAJdQrLcc,13810
|
|
20
|
+
py_dpm/dpm/models.py,sha256=YyOYXzN4tIT2rO6-PP-53C6f7skatgnz7VY_5SnOsLQ,136414
|
|
21
21
|
py_dpm/dpm/utils.py,sha256=3w06_kKiFRPqHvRUfN_0nNQIpppfidFybdDge97qzI0,14918
|
|
22
22
|
py_dpm/dpm/queries/base.py,sha256=EddMeJMwtp63DyyIFO7_XxGvdlCtJQWWpeOVImlKp4I,3648
|
|
23
23
|
py_dpm/dpm/queries/basic_objects.py,sha256=JOXC235lMDfVENrFAhZAl7_nqePJ4RrwJhFF0WDyk0M,955
|
|
@@ -35,7 +35,7 @@ py_dpm/dpm_xl/ast/ml_generation.py,sha256=Lw_1Btln2x1ewD9xH-2Ea4NJJP3PIqFoivWASX
|
|
|
35
35
|
py_dpm/dpm_xl/ast/module_analyzer.py,sha256=ZnldoYn-s41UMiJpcAV6hjIwH6fssZeOpc564epngg8,2872
|
|
36
36
|
py_dpm/dpm_xl/ast/module_dependencies.py,sha256=tbCqoDcE1n1lJOjtbpD3rNPkXrLk-k2rM5zyVwmsNpc,8355
|
|
37
37
|
py_dpm/dpm_xl/ast/nodes.py,sha256=5ob8MsCW0fPZgz9yP_6IgVTH2SGeoTk5VncJuQ2SgrE,25035
|
|
38
|
-
py_dpm/dpm_xl/ast/operands.py,sha256=
|
|
38
|
+
py_dpm/dpm_xl/ast/operands.py,sha256=0Yye5z6l0BBh9nrMlkVVNy3fYgbeUzZYLmP5Jvzr95M,21606
|
|
39
39
|
py_dpm/dpm_xl/ast/template.py,sha256=QhYm7Jh_a-ws3kSmf0hqXFLzB_quO9GgKcmcFe22_fg,3045
|
|
40
40
|
py_dpm/dpm_xl/ast/visitor.py,sha256=yL9UpPMQlq8ToHR8COyFYpuSChnDRjnkQHbCyYX0tsY,509
|
|
41
41
|
py_dpm/dpm_xl/ast/where_clause.py,sha256=g3cslQ8vmlm0doqQ_ghjXzhzItc_xlC_bQ9odn87FGk,328
|
|
@@ -76,9 +76,9 @@ py_dpm/exceptions/exceptions.py,sha256=6S3p-_i5O1oStvSMixt_JQG0xwTeSfBcdzrwL8yBy
|
|
|
76
76
|
py_dpm/exceptions/messages.py,sha256=UwY6QIK8c-POcDCc9HYbZFGArCIYAanUGNh2LNKPx3U,7534
|
|
77
77
|
py_dpm/instance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
78
78
|
py_dpm/instance/instance.py,sha256=gRSg2dh1nEa0Srx9yKcN3bxiYidvZyRU_jsTNaKkP5I,10882
|
|
79
|
-
pydpm_xl-0.2.
|
|
80
|
-
pydpm_xl-0.2.
|
|
81
|
-
pydpm_xl-0.2.
|
|
82
|
-
pydpm_xl-0.2.
|
|
83
|
-
pydpm_xl-0.2.
|
|
84
|
-
pydpm_xl-0.2.
|
|
79
|
+
pydpm_xl-0.2.11.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
80
|
+
pydpm_xl-0.2.11.dist-info/METADATA,sha256=JXXtaUl9tMD7TSrGR5G9KWu_d1gTwutLIVpWYBJY18g,9154
|
|
81
|
+
pydpm_xl-0.2.11.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
82
|
+
pydpm_xl-0.2.11.dist-info/entry_points.txt,sha256=6DDmBfw-AjtgvMHgq_I730i_LAAs_7-N3C95HD_bRr4,47
|
|
83
|
+
pydpm_xl-0.2.11.dist-info/top_level.txt,sha256=495PvWZRoKl2NvbQU25W7dqWIBHqY-mFMPt83uxPpcM,7
|
|
84
|
+
pydpm_xl-0.2.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|