ShinySC 0.7.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ShinySC/__init__.py +10 -0
- ShinySC/main.py +687 -0
- shinysc-0.7.1.dist-info/METADATA +51 -0
- shinysc-0.7.1.dist-info/RECORD +7 -0
- shinysc-0.7.1.dist-info/WHEEL +5 -0
- shinysc-0.7.1.dist-info/licenses/LICENSE.txt +674 -0
- shinysc-0.7.1.dist-info/top_level.txt +2 -0
ShinySC/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
|
|
2
|
+
from .main import describe
|
|
3
|
+
from .main import make_url
|
|
4
|
+
from .main import help
|
|
5
|
+
from .main import simple_metadata
|
|
6
|
+
from .main import full_metadata
|
|
7
|
+
from .main import search
|
|
8
|
+
from .main import update_list
|
|
9
|
+
|
|
10
|
+
__all__ = ["describe", "make_url","help","simple_metadata","full_metadata","search","update_list"]
|
ShinySC/main.py
ADDED
|
@@ -0,0 +1,687 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import json
|
|
3
|
+
import requests
|
|
4
|
+
import urllib.parse
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
|
|
7
|
+
_cached_metadata = None
|
|
8
|
+
_cached_cube_list = None
|
|
9
|
+
|
|
10
|
+
_codes = requests.get('https://www150.statcan.gc.ca/t1/wds/rest/getCodeSets')
|
|
11
|
+
_codes = _codes.json()
|
|
12
|
+
|
|
13
|
+
def _fmt_id(id):
|
|
14
|
+
|
|
15
|
+
id = str(id)
|
|
16
|
+
|
|
17
|
+
if id.count('-') == 3:
|
|
18
|
+
a,b,c,d = id.split('-')
|
|
19
|
+
return a+b+c
|
|
20
|
+
|
|
21
|
+
if id.count('-') == 2:
|
|
22
|
+
a,b,c = id.split('-')
|
|
23
|
+
return a+b+c
|
|
24
|
+
|
|
25
|
+
if len(id) == 10:
|
|
26
|
+
return id[:8]
|
|
27
|
+
|
|
28
|
+
return id
|
|
29
|
+
|
|
30
|
+
def _vali_date(date_text):
|
|
31
|
+
try:
|
|
32
|
+
datetime.strptime(date_text, '%Y-%m-%d')
|
|
33
|
+
return True
|
|
34
|
+
|
|
35
|
+
except ValueError:
|
|
36
|
+
return False
|
|
37
|
+
|
|
38
|
+
def _cube_list(id='',lang='en'):
|
|
39
|
+
"""
|
|
40
|
+
Retrieves information about a cube (table).
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
id (int or str): The product ID (PID)
|
|
44
|
+
lang (str): The language ('en' or 'fr')
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
global _cached_cube_list
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
if _cached_cube_list == None:
|
|
51
|
+
url = "https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesList"
|
|
52
|
+
resp = requests.get(url)
|
|
53
|
+
table = resp.json()
|
|
54
|
+
_cached_cube_list = table
|
|
55
|
+
|
|
56
|
+
else:
|
|
57
|
+
table = _cached_cube_list
|
|
58
|
+
|
|
59
|
+
except Exception as e:
|
|
60
|
+
print('Failed to find "getAllCubesList" on WDS')
|
|
61
|
+
|
|
62
|
+
if id == '':
|
|
63
|
+
return table
|
|
64
|
+
|
|
65
|
+
for i in table:
|
|
66
|
+
if i['productId'] == id: return i
|
|
67
|
+
else:
|
|
68
|
+
print(f'Table {id} not found.')
|
|
69
|
+
|
|
70
|
+
def _remove_lang(obj,language):
|
|
71
|
+
|
|
72
|
+
language = 'En' if language == 'en' else 'Fr'
|
|
73
|
+
|
|
74
|
+
if isinstance(obj, dict):
|
|
75
|
+
|
|
76
|
+
obj = {k: _remove_lang(v,language) for k, v in obj.items() if not k.endswith(language)}
|
|
77
|
+
|
|
78
|
+
for k in obj.keys():
|
|
79
|
+
if isinstance(obj[k],list):
|
|
80
|
+
if len(obj[k]) > 0:
|
|
81
|
+
if isinstance(obj[k][0],dict):
|
|
82
|
+
obj[k] = [_remove_lang(i,language) for i in obj[k]]
|
|
83
|
+
return obj
|
|
84
|
+
|
|
85
|
+
else:
|
|
86
|
+
return obj
|
|
87
|
+
|
|
88
|
+
def _search_json(obj: dict={}, term: str='', mode: str='AND'):
|
|
89
|
+
"""
|
|
90
|
+
Recursively search JSON-like structures.
|
|
91
|
+
Keep entries if:
|
|
92
|
+
- key contains term
|
|
93
|
+
- OR value contains term
|
|
94
|
+
- OR nested match
|
|
95
|
+
- OR sibling key contains "code" when any match occurs in the same dict
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
mode = mode.upper()
|
|
99
|
+
|
|
100
|
+
if mode == "OR":
|
|
101
|
+
terms = [t.strip() for t in term.split(",") if t.strip()]
|
|
102
|
+
pattern = re.compile("|".join(map(re.escape, terms)), re.IGNORECASE | re.DOTALL)
|
|
103
|
+
|
|
104
|
+
if mode == "AND":
|
|
105
|
+
query = [t.strip() for t in term.split(",") if t.strip()]
|
|
106
|
+
regex = "".join(
|
|
107
|
+
rf"(?=.*(?:^|\W){re.escape(t)}(?:\W|$))"
|
|
108
|
+
for t in query
|
|
109
|
+
)
|
|
110
|
+
pattern = re.compile(regex, re.IGNORECASE | re.DOTALL)
|
|
111
|
+
|
|
112
|
+
# Case 1: dict
|
|
113
|
+
if isinstance(obj, dict):
|
|
114
|
+
matches = {} # matches found in this dict
|
|
115
|
+
matched_in_dict = False # flag so we can keep sibling "code" keys
|
|
116
|
+
|
|
117
|
+
# First pass: detect matches
|
|
118
|
+
child_results = {}
|
|
119
|
+
|
|
120
|
+
for k, v in obj.items():
|
|
121
|
+
|
|
122
|
+
key_match = bool(pattern.search(k))
|
|
123
|
+
value_match = isinstance(v, str) and bool(pattern.search(v))
|
|
124
|
+
|
|
125
|
+
nested = None
|
|
126
|
+
|
|
127
|
+
if isinstance(v, (dict, list)):
|
|
128
|
+
nested = _search_json(v, term, mode)
|
|
129
|
+
|
|
130
|
+
# If this entry matches directly or via nested
|
|
131
|
+
if key_match or value_match or nested:
|
|
132
|
+
matched_in_dict = True
|
|
133
|
+
child_results[k] = nested if nested is not None else v
|
|
134
|
+
|
|
135
|
+
# Second pass: if any match happened, also keep sibling "code" keys
|
|
136
|
+
if matched_in_dict:
|
|
137
|
+
for k, v in obj.items():
|
|
138
|
+
if "Code" in k and k not in child_results:
|
|
139
|
+
print(k,v)
|
|
140
|
+
child_results[k] = v
|
|
141
|
+
|
|
142
|
+
return child_results if child_results else None
|
|
143
|
+
|
|
144
|
+
# Case 2: list
|
|
145
|
+
elif isinstance(obj, list):
|
|
146
|
+
results = []
|
|
147
|
+
for item in obj:
|
|
148
|
+
nested = _search_json(item, term, mode)
|
|
149
|
+
#value_match = isinstance(item, str) and term in item
|
|
150
|
+
value_match = isinstance(item, str) and bool(pattern.search(item))
|
|
151
|
+
|
|
152
|
+
if nested is not None or value_match:
|
|
153
|
+
results.append(nested if nested is not None else item)
|
|
154
|
+
|
|
155
|
+
return results if results else None
|
|
156
|
+
|
|
157
|
+
# Case 3: primitive
|
|
158
|
+
else:
|
|
159
|
+
#if isinstance(obj, str) and term in obj:
|
|
160
|
+
print(obj)
|
|
161
|
+
try:
|
|
162
|
+
if isinstance(obj, str) and bool(re.search(obj)):
|
|
163
|
+
return obj
|
|
164
|
+
except:
|
|
165
|
+
pass
|
|
166
|
+
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
def _parse_dim(x,full=True):
|
|
170
|
+
|
|
171
|
+
tmp = {}
|
|
172
|
+
sel = []
|
|
173
|
+
|
|
174
|
+
if full:
|
|
175
|
+
for i in x:
|
|
176
|
+
try:
|
|
177
|
+
tmp[i['dimensionNameEn']] = i['member']
|
|
178
|
+
j = []
|
|
179
|
+
for k in i['member']:
|
|
180
|
+
j.append(k['memberId'])
|
|
181
|
+
except:
|
|
182
|
+
tmp[i['dimensionNameEn']] = i['hasUom']
|
|
183
|
+
|
|
184
|
+
sel.append(j)
|
|
185
|
+
else:
|
|
186
|
+
pass
|
|
187
|
+
|
|
188
|
+
return tmp,sel
|
|
189
|
+
|
|
190
|
+
def _parse_filters(filters,id,lang='en'):
|
|
191
|
+
"""
|
|
192
|
+
Takes user-defined filters as a dictionary and
|
|
193
|
+
transforms it to use in the StatCan URL.
|
|
194
|
+
|
|
195
|
+
Args:
|
|
196
|
+
filters (dict): User-defined filters
|
|
197
|
+
"""
|
|
198
|
+
#Iterate over dict; put it in same order metadata has them;
|
|
199
|
+
#Then convert selected items into their numeric entries as nested list
|
|
200
|
+
#--> Convert dimensionName to dimensionPositionId; this lets us put in right order
|
|
201
|
+
#-->---> Convert members' memberNameEn(Fr) to memberId
|
|
202
|
+
#There will be some "easy" filters, like '503' to select all CMAs in geography
|
|
203
|
+
#...though I guess that would be applied after the table is downloaded
|
|
204
|
+
|
|
205
|
+
tmp = full_metadata(id)
|
|
206
|
+
dim = tmp['dimension']
|
|
207
|
+
|
|
208
|
+
new_filters = []
|
|
209
|
+
|
|
210
|
+
for i in range(0,len(dim)):
|
|
211
|
+
|
|
212
|
+
dim_name = dim[i][f'dimensionName{lang.capitalize()}']
|
|
213
|
+
|
|
214
|
+
if dim_name in filters:
|
|
215
|
+
|
|
216
|
+
selected_names = filters[dim_name]
|
|
217
|
+
selected_ids = []
|
|
218
|
+
|
|
219
|
+
for j in dim[i]['member']:
|
|
220
|
+
|
|
221
|
+
if j[f'memberName{lang.capitalize()}'] in selected_names:
|
|
222
|
+
selected_ids.append(j['memberId'])
|
|
223
|
+
|
|
224
|
+
new_filters.append(selected_ids)
|
|
225
|
+
|
|
226
|
+
else:
|
|
227
|
+
#Select all members if not specified in filters
|
|
228
|
+
selected_ids = []
|
|
229
|
+
for j in dim[i]['member']:
|
|
230
|
+
selected_ids.append(j['memberId'])
|
|
231
|
+
new_filters.append(selected_ids)
|
|
232
|
+
|
|
233
|
+
raw = json.dumps(new_filters, separators=(',',':'))
|
|
234
|
+
enc = urllib.parse.quote(raw)
|
|
235
|
+
|
|
236
|
+
return enc
|
|
237
|
+
|
|
238
|
+
def full_metadata(id, timeout=30, lang='en', special=[]):
|
|
239
|
+
"""
|
|
240
|
+
Retrieves metadata for a cube (table) from Statistics Canada WDS in JSON format.
|
|
241
|
+
|
|
242
|
+
:param id (str): productId
|
|
243
|
+
:param timeout (int): Timeout in seconds for the HTTP request.
|
|
244
|
+
:param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
|
|
245
|
+
"""
|
|
246
|
+
|
|
247
|
+
if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
|
|
248
|
+
|
|
249
|
+
endpoint = "https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata"
|
|
250
|
+
|
|
251
|
+
# Prepare payload as list of dicts, per documentation example. :contentReference[oaicite:2]{index=2}
|
|
252
|
+
payload = [
|
|
253
|
+
{
|
|
254
|
+
"productId": str(id)
|
|
255
|
+
}
|
|
256
|
+
]
|
|
257
|
+
|
|
258
|
+
headers = {
|
|
259
|
+
"Content-Type": "application/json"
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
try:
|
|
263
|
+
response = requests.post(endpoint, headers=headers, json=payload, timeout=timeout)
|
|
264
|
+
response.raise_for_status()
|
|
265
|
+
data = response.json()
|
|
266
|
+
|
|
267
|
+
# Check for status
|
|
268
|
+
if data[0]["status"] != "SUCCESS":
|
|
269
|
+
raise RuntimeError(f"Request returned non-SUCCESS status: {data}")
|
|
270
|
+
|
|
271
|
+
if lang == 'en':
|
|
272
|
+
return _remove_lang(data[0]['object'],'Fr')
|
|
273
|
+
|
|
274
|
+
elif lang == 'fr':
|
|
275
|
+
return _remove_lang(data[0]['object'],'En')
|
|
276
|
+
|
|
277
|
+
else:
|
|
278
|
+
return data[0]['object']
|
|
279
|
+
|
|
280
|
+
except requests.RequestException as e:
|
|
281
|
+
raise RuntimeError(f"HTTP request failed: {e}") from e
|
|
282
|
+
|
|
283
|
+
except json.JSONDecodeError as e:
|
|
284
|
+
raise RuntimeError(f"Failed to parse JSON response: {e}") from e
|
|
285
|
+
|
|
286
|
+
def simple_metadata(id, lang='en'):
|
|
287
|
+
"""
|
|
288
|
+
Retrieves and displays simplified metadata for a cube (table).\n
|
|
289
|
+
This will only show productId, cubeTitle, cubeEndDate, cubeStartDate, and a simplified dimension.
|
|
290
|
+
|
|
291
|
+
:param id (str): productId
|
|
292
|
+
:param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
|
|
296
|
+
|
|
297
|
+
meta = full_metadata(id,30,lang)
|
|
298
|
+
|
|
299
|
+
keep = ['productId',f'cubeTitle{lang.capitalize}','cubeEndDate','cubeStartDate','dimension']
|
|
300
|
+
|
|
301
|
+
obj = {k:v for k,v in meta.items() if k in keep}
|
|
302
|
+
|
|
303
|
+
return obj
|
|
304
|
+
|
|
305
|
+
def help():
|
|
306
|
+
print("""
|
|
307
|
+
---------------------------------------------------------
|
|
308
|
+
|
|
309
|
+
describe(): provides key information about a table to help you build a custom query
|
|
310
|
+
make_url(): builds a custom URL to download data from StatCan WDS
|
|
311
|
+
list_tables (): ists all active tables available on StatCan WDS
|
|
312
|
+
help(): displays this help message
|
|
313
|
+
simple_metadata(): retrieves simplified metadata for a table
|
|
314
|
+
full_metadata(): retrieves full metadata for a table
|
|
315
|
+
search (): earches for tables matching your criteria
|
|
316
|
+
update_list(): checks for recently updated tables
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
If you already know the productId and filters for the table
|
|
320
|
+
you want, simply call:
|
|
321
|
+
|
|
322
|
+
url = ShinySC.make_url(id={id}, filters={...})
|
|
323
|
+
|
|
324
|
+
Addtionally, you can specify date ranges or number of recent periods:
|
|
325
|
+
|
|
326
|
+
url = ShinySC.make_url(id={id}, periods=5)
|
|
327
|
+
url = ShinySC.make_url(id={id}, start='2020-01-01',end='2022-01-01')
|
|
328
|
+
|
|
329
|
+
If you don't know what table you want, you can programmatically
|
|
330
|
+
search for tables that match your criteria using these criteria:
|
|
331
|
+
|
|
332
|
+
ShinySC.search(query,last_updated,dates,status)
|
|
333
|
+
|
|
334
|
+
query: comma-separated terms to search for in table names, descriptions, and surveys
|
|
335
|
+
last_updated: 'YYYY-MM-DD' to search tables updated on or after this date
|
|
336
|
+
dates: list of two strings [start_date,end_date] to filter tables by date range
|
|
337
|
+
status: 'active' or 'archived' tables (default is 'active')
|
|
338
|
+
|
|
339
|
+
Call ShinySC.describe(productId) to see available dimensionsthat can be used for filtering.
|
|
340
|
+
|
|
341
|
+
Call ShinySC.update_list(date='YYYY-MM-DD') to see recently updated tables for a specfiic date.
|
|
342
|
+
Add in a productId to see if that specific table was updated on that date.
|
|
343
|
+
|
|
344
|
+
ShinySC.update_list(id=35100003,date='2023-01-01')
|
|
345
|
+
|
|
346
|
+
This library makes use of these endpoints:
|
|
347
|
+
https://www150.statcan.gc.ca/t1/wds/rest/getCodeSets
|
|
348
|
+
https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesList
|
|
349
|
+
https://www150.statcan.gc.ca/t1/wds/rest/getCubeMetadata
|
|
350
|
+
https://www150.statcan.gc.ca/t1/wds/rest/getChangedCubeList
|
|
351
|
+
|
|
352
|
+
---------------------------------------------------------
|
|
353
|
+
""")
|
|
354
|
+
|
|
355
|
+
def describe(id, lang='en'):
|
|
356
|
+
"""
|
|
357
|
+
Quickly get key information to help you build a custom query.
|
|
358
|
+
|
|
359
|
+
:param id (str): productId
|
|
360
|
+
:param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
|
|
361
|
+
"""
|
|
362
|
+
|
|
363
|
+
if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
|
|
364
|
+
|
|
365
|
+
attributes = {}
|
|
366
|
+
|
|
367
|
+
md = full_metadata(id,30,lang)
|
|
368
|
+
|
|
369
|
+
attributes['name'] = md[f'cubeTitle{lang.capitalize()}']
|
|
370
|
+
attributes['dataPoints'] = md['nbDatapointsCube']
|
|
371
|
+
attributes['productId'] = id
|
|
372
|
+
attributes['status'] = 'Active' if md['archiveStatusCode'] == '2' else 'Archived'
|
|
373
|
+
attributes['dataDateRange'] = [md['cubeStartDate'],md['cubeEndDate']]
|
|
374
|
+
attributes['lastUpdated'] = md['releaseTime']
|
|
375
|
+
attributes['subject'] = md['subjectCode']
|
|
376
|
+
attributes['dimensions'] = [
|
|
377
|
+
{i[f'dimensionName{lang.capitalize()}']:[j[f'memberName{lang.capitalize()}'] for j in i['member']]} for i in md['dimension']
|
|
378
|
+
]
|
|
379
|
+
|
|
380
|
+
return attributes
|
|
381
|
+
|
|
382
|
+
def make_url(id: int='',periods: int='',start: str='',end: str='',filters={},lang: str='en'):
|
|
383
|
+
"""
|
|
384
|
+
Downloads a table from Statistics Canada using custom filters.
|
|
385
|
+
Default language is English ('en')
|
|
386
|
+
Returns a URL that can be downloaded with your favourite library.
|
|
387
|
+
|
|
388
|
+
Args:
|
|
389
|
+
:param id (int, str): productId
|
|
390
|
+
:param periods (int): number of recent periods you wish to download (default is 1)
|
|
391
|
+
:param start (str): desired start date for data series (YYYY-MM-DD)
|
|
392
|
+
:param end (str): desired end date for data series (YYYY-MM-DD)
|
|
393
|
+
:param full (bool): download the full table or not (default True)
|
|
394
|
+
:param filters (dict): filters you wish to apply ('attribute':[...],...)
|
|
395
|
+
:param lang (str): which langauge you wish to get data in ('en'[default] or 'fr')
|
|
396
|
+
"""
|
|
397
|
+
global _cached_metadata
|
|
398
|
+
full = False
|
|
399
|
+
|
|
400
|
+
#Validate inputs
|
|
401
|
+
if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
|
|
402
|
+
if lang not in ['en','fr']: raise ValueError('Language must be "en" or "fr".')
|
|
403
|
+
if (periods != '') & (not isinstance(periods,int)): raise TypeError('Must be integer number of periods to download.')
|
|
404
|
+
|
|
405
|
+
if (start != '') & (not(_vali_date(start))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
|
|
406
|
+
if (start > datetime.now().strftime('%Y-%m-%d')): raise ValueError('Invalid start date. Cannot be in the future.')
|
|
407
|
+
if (start > end) & (end != ''): raise ValueError('Start date cannot be after end date.')
|
|
408
|
+
|
|
409
|
+
if (end != '') & (not(_vali_date(end))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
|
|
410
|
+
if (end < start) & (start != ''): raise ValueError('End date cannot be after start date.')
|
|
411
|
+
|
|
412
|
+
if not isinstance(filters,dict): raise ValueError('Filters must be a dictionary of filterName:[values,...].')
|
|
413
|
+
|
|
414
|
+
try:
|
|
415
|
+
if _cached_metadata == None:
|
|
416
|
+
md = full_metadata(id,30,lang)
|
|
417
|
+
|
|
418
|
+
else:
|
|
419
|
+
md = _cached_metadata
|
|
420
|
+
except:
|
|
421
|
+
print('Invalid productId.')
|
|
422
|
+
return ''
|
|
423
|
+
|
|
424
|
+
#tablename = md['cubeTitleEn']
|
|
425
|
+
|
|
426
|
+
archived = md['archiveStatusCode']
|
|
427
|
+
lastUpdated = md['releaseTime']
|
|
428
|
+
|
|
429
|
+
if archived == '1': print(f'ADVISORY: This table has been archived and does not get updated. Last updated: {lastUpdated}')
|
|
430
|
+
|
|
431
|
+
if filters == {} and not full and periods == '' and start == '' and end == '':
|
|
432
|
+
print('ADVISORY: No filters specified. Generating URL for the full table. Full tables can be very large.')
|
|
433
|
+
full = True
|
|
434
|
+
|
|
435
|
+
if full:
|
|
436
|
+
dim = md['dimension']
|
|
437
|
+
dim,selected = _parse_dim(dim,full)
|
|
438
|
+
raw = json.dumps(selected, separators=(',',':'))
|
|
439
|
+
filters = urllib.parse.quote(raw)
|
|
440
|
+
|
|
441
|
+
else:
|
|
442
|
+
filters = _parse_filters(filters,id,lang)
|
|
443
|
+
|
|
444
|
+
#Date filter stuff
|
|
445
|
+
if (periods == '') & (start == '') & (end == ''):
|
|
446
|
+
print('ADVISORY: No date range or periods specified.\nDefaulting to all periods, which may be a large amount of data.')
|
|
447
|
+
start = md['cubeStartDate']
|
|
448
|
+
end = md['cubeEndDate']
|
|
449
|
+
|
|
450
|
+
url = f'https://www150.statcan.gc.ca/t1/tbl1/en/dtl!downloadDbLoadingData-nonTraduit.action?pid={id}01&latestN={periods}&startDate={start}&endDate={end}&csvLocale={lang}&selectedMembers={filters}&checkedLevels='
|
|
451
|
+
|
|
452
|
+
return url
|
|
453
|
+
|
|
454
|
+
def search(query='',last_updated='',data_dates=[],status='active',mode='AND',lang='en'):
|
|
455
|
+
"""
|
|
456
|
+
Docstring for find_tables
|
|
457
|
+
|
|
458
|
+
:param query (str): Comma-delimited terms to search for in table names, descriptions, and surveys
|
|
459
|
+
:param status (str): Active (default) or Archived
|
|
460
|
+
:param last_updated (str): Only return tables updated after this date (YYYY-MM-DD)
|
|
461
|
+
:param data_dates ([str,str]): Only return tables with data in this date range (YYYY-MM-DD)
|
|
462
|
+
:param mode (str): 'AND'[default] or 'OR' search mode for query terms
|
|
463
|
+
:param lang (str): Language ('en' or 'fr')
|
|
464
|
+
"""
|
|
465
|
+
|
|
466
|
+
global _codes
|
|
467
|
+
|
|
468
|
+
lang = lang.lower()
|
|
469
|
+
mode = mode.upper()
|
|
470
|
+
|
|
471
|
+
#Validate inputs
|
|
472
|
+
if query == '': raise ValueError('Query cannot be empty.')
|
|
473
|
+
if query.count(',') >= 10: raise ValueError('Maximum of 10 search terms allowed.')
|
|
474
|
+
if query.count(',,') > 0: raise ValueError('Empty search terms are not allowed.')
|
|
475
|
+
if query.startswith(',') or query.endswith(','): raise ValueError('Empty search terms are not allowed.')
|
|
476
|
+
|
|
477
|
+
if lang not in ['en','fr']: raise ValueError('Language must be "en" or "fr".')
|
|
478
|
+
if status.lower() not in ['active','archived']: raise ValueError('Status must be "active" or "archived".')
|
|
479
|
+
if mode.upper() not in ['AND','OR']: raise ValueError('Mode must be "AND" or "OR".')
|
|
480
|
+
|
|
481
|
+
if (last_updated != '') & (not(_vali_date(last_updated))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
|
|
482
|
+
|
|
483
|
+
if data_dates != []:
|
|
484
|
+
|
|
485
|
+
if len(data_dates) == 2:
|
|
486
|
+
|
|
487
|
+
start = data_dates[0]
|
|
488
|
+
end = data_dates[1]
|
|
489
|
+
|
|
490
|
+
if (start != '') & (not(_vali_date(start))): raise ValueError('Starting data must be a valid date in the format YYYY-MM-DD.')
|
|
491
|
+
if (start > datetime.now().strftime('%Y-%m-%d')): raise ValueError('Invalid start date. Cannot be in the future.')
|
|
492
|
+
if (start > end) & (end != ''): raise ValueError('Start date cannot be after end date.')
|
|
493
|
+
|
|
494
|
+
if (end != '') & (not(_vali_date(end))): raise ValueError('End data must be a valid date in the format YYYY-MM-DD.')
|
|
495
|
+
if (end < start) & (start != ''): raise ValueError('End date cannot be after start date.')
|
|
496
|
+
|
|
497
|
+
status = status.lower()
|
|
498
|
+
mode = mode.upper()
|
|
499
|
+
lang = lang.lower()
|
|
500
|
+
|
|
501
|
+
if status == 'active':
|
|
502
|
+
status = '2'
|
|
503
|
+
elif status == 'archived':
|
|
504
|
+
status = '1'
|
|
505
|
+
else:
|
|
506
|
+
raise ValueError('Status must be "active" or "archived".')
|
|
507
|
+
|
|
508
|
+
local_codes = _remove_lang(_codes,'Fr' if lang == 'en' else 'En')
|
|
509
|
+
local_codes = _search_json(local_codes,query,mode)
|
|
510
|
+
|
|
511
|
+
parsed_local_codes = {}
|
|
512
|
+
|
|
513
|
+
if local_codes != None:
|
|
514
|
+
|
|
515
|
+
for k in local_codes['object'].keys():
|
|
516
|
+
parsed_local_codes[k] = []
|
|
517
|
+
for i in local_codes['object'][k]:
|
|
518
|
+
try:
|
|
519
|
+
parsed_local_codes[k].append(i[f'{k}Code'])
|
|
520
|
+
except:
|
|
521
|
+
pass
|
|
522
|
+
|
|
523
|
+
tables = _cube_list()
|
|
524
|
+
filtered_tables = []
|
|
525
|
+
|
|
526
|
+
if mode == "OR":
|
|
527
|
+
terms = [t.strip() for t in query.split(",") if t.strip()]
|
|
528
|
+
pattern = re.compile("|".join(map(re.escape, terms)), re.IGNORECASE | re.DOTALL)
|
|
529
|
+
|
|
530
|
+
if mode == "AND":
|
|
531
|
+
query = [t.strip() for t in query.split(",") if t.strip()]
|
|
532
|
+
regex = "".join(
|
|
533
|
+
rf"(?=.*(?:^|\W){re.escape(t)}(?:\W|$))"
|
|
534
|
+
for t in query
|
|
535
|
+
)
|
|
536
|
+
pattern = re.compile(regex, re.IGNORECASE | re.DOTALL)
|
|
537
|
+
|
|
538
|
+
for t in tables:
|
|
539
|
+
|
|
540
|
+
cubeTitle = f'cubeTitle{lang.capitalize()}'
|
|
541
|
+
released = t['releaseTime'][:10]
|
|
542
|
+
startDate = t['cubeStartDate']
|
|
543
|
+
endDate = t['cubeEndDate']
|
|
544
|
+
|
|
545
|
+
if pattern.search(t[cubeTitle]):
|
|
546
|
+
if t['archived'] == status:
|
|
547
|
+
|
|
548
|
+
# prevent duplicates
|
|
549
|
+
existing_ids = {list(d.keys())[0] for d in filtered_tables}
|
|
550
|
+
if t['productId'] in existing_ids:
|
|
551
|
+
return # or continue if in a loop
|
|
552
|
+
|
|
553
|
+
if last_updated:
|
|
554
|
+
released_date = datetime.strptime(released, "%Y-%m-%d").date()
|
|
555
|
+
last_updated_date = datetime.strptime(last_updated, "%Y-%m-%d").date()
|
|
556
|
+
|
|
557
|
+
if released_date >= last_updated_date:
|
|
558
|
+
filtered_tables.append({t['productId']: t[cubeTitle]})
|
|
559
|
+
else:
|
|
560
|
+
filtered_tables.append({t['productId']: t[cubeTitle]})
|
|
561
|
+
|
|
562
|
+
for k in parsed_local_codes.keys():
|
|
563
|
+
|
|
564
|
+
if (t['productId'] not in filtered_tables) & (t['archived'] == status):
|
|
565
|
+
if (k == 'subject') & (len(parsed_local_codes[k]) > 0) & ('subjectCode' in t):
|
|
566
|
+
try:
|
|
567
|
+
filtered_tables.append({t['productId']:t[f'cubeTitle{lang.capitalize()}']}) if set(parsed_local_codes[k]).intersection(set(t['subjectCode'])) else None
|
|
568
|
+
except:
|
|
569
|
+
pass
|
|
570
|
+
|
|
571
|
+
if (k == 'survey') & (len(parsed_local_codes[k]) > 0) & (t['surveyCode'] != None):
|
|
572
|
+
try:
|
|
573
|
+
filtered_tables.append({t['productId']:t[f'cubeTitle{lang.capitalize()}']}) if set(parsed_local_codes[k]).intersection(set(t['surveyCode'])) else None
|
|
574
|
+
except:
|
|
575
|
+
pass
|
|
576
|
+
|
|
577
|
+
print(f'Found {len(filtered_tables)} tables matching search criteria, "{query}".\n')
|
|
578
|
+
|
|
579
|
+
return filtered_tables
|
|
580
|
+
|
|
581
|
+
def update_list(id='',date=''):
|
|
582
|
+
"""
|
|
583
|
+
:param id (str): Product ID of the table to check
|
|
584
|
+
:param date (str): Date in the format YYYY-MM-DD to check for changes
|
|
585
|
+
"""
|
|
586
|
+
|
|
587
|
+
#if they input just a date in the id field, treat it as date
|
|
588
|
+
if (id.count('-') == 2) & (date == '') & (_vali_date(id)):
|
|
589
|
+
date = id
|
|
590
|
+
id = ''
|
|
591
|
+
|
|
592
|
+
if (id != '') & ((len(str(id)) != 8) | (any(c.isalpha() for c in str(id)))): raise ValueError('productId must be 8 digits.')
|
|
593
|
+
if (date != '') & (not(_vali_date(date))): raise ValueError('Must be a valid date in the format YYYY-MM-DD.')
|
|
594
|
+
|
|
595
|
+
#If no date provided, use today's date
|
|
596
|
+
if date == '':
|
|
597
|
+
date = datetime.now().strftime('%Y-%m-%d')
|
|
598
|
+
|
|
599
|
+
url = f"https://www150.statcan.gc.ca/t1/wds/rest/getChangedCubeList/{date}"
|
|
600
|
+
resp = requests.get(url)
|
|
601
|
+
tables = resp.json()
|
|
602
|
+
|
|
603
|
+
if id != '':
|
|
604
|
+
for i in tables['object']:
|
|
605
|
+
if i['productId'] == int(id):
|
|
606
|
+
return True
|
|
607
|
+
else:
|
|
608
|
+
return False
|
|
609
|
+
|
|
610
|
+
elif id == '':
|
|
611
|
+
return tables['object']
|
|
612
|
+
|
|
613
|
+
def list_tables(lang='en'):
|
|
614
|
+
|
|
615
|
+
"""
|
|
616
|
+
:param lang (str): Language of the table to return ('en' or 'fr')
|
|
617
|
+
"""
|
|
618
|
+
|
|
619
|
+
lang = lang.lower()
|
|
620
|
+
if lang not in ['en','fr']: raise ValueError('Language must be "en" or "fr".')
|
|
621
|
+
|
|
622
|
+
url = "https://www150.statcan.gc.ca/t1/wds/rest/getAllCubesList"
|
|
623
|
+
resp = requests.get(url)
|
|
624
|
+
tables = resp.json()
|
|
625
|
+
|
|
626
|
+
df = pd.DataFrame(tables)
|
|
627
|
+
|
|
628
|
+
if lang == 'en':
|
|
629
|
+
df = df[[x for x in df.columns if x if 'Fr' not in x]]
|
|
630
|
+
elif lang == 'fr':
|
|
631
|
+
df = df[[x for x in df.columns if x if 'En' not in x]]
|
|
632
|
+
|
|
633
|
+
df = df[df['archived'] == '2']
|
|
634
|
+
|
|
635
|
+
archived_tables = df[df['archived'] == '1']
|
|
636
|
+
active_tables = df[df['archived'] == '2']
|
|
637
|
+
|
|
638
|
+
card_css = "border:1px solid black;margin:10px;padding:10px;background:white;max-width:200px;border-radius:3px;color:black;"
|
|
639
|
+
n = 2
|
|
640
|
+
|
|
641
|
+
count = 0
|
|
642
|
+
|
|
643
|
+
for p in tables:
|
|
644
|
+
|
|
645
|
+
id = p['productId']
|
|
646
|
+
endDate = datetime.fromisoformat(p['cubeEndDate'].replace("Z", "+00:00")).replace(tzinfo=None)
|
|
647
|
+
|
|
648
|
+
if p['archived'] == '2':
|
|
649
|
+
|
|
650
|
+
metadata = full_metadata(id)
|
|
651
|
+
num_dimensions = len(metadata['dimension'])
|
|
652
|
+
data = metadata['dimension']
|
|
653
|
+
|
|
654
|
+
tablename = p['cubeTitleEn']
|
|
655
|
+
dim = p['dimensions']
|
|
656
|
+
dim,selected = _parse_dim(dim)
|
|
657
|
+
|
|
658
|
+
raw = json.dumps(selected, separators=(',',':'))
|
|
659
|
+
enc = urllib.parse.quote(raw)
|
|
660
|
+
|
|
661
|
+
num_param = len(data)
|
|
662
|
+
|
|
663
|
+
attributes = {}
|
|
664
|
+
attributes['Table Name'] = tablename
|
|
665
|
+
|
|
666
|
+
for i in range(0,num_param):
|
|
667
|
+
|
|
668
|
+
sub_size = len(data[i]['member'])
|
|
669
|
+
top_name = data[i]['dimensionNameEn']
|
|
670
|
+
|
|
671
|
+
for j in range(0,sub_size):
|
|
672
|
+
|
|
673
|
+
#print(data[i]['member'][j])
|
|
674
|
+
|
|
675
|
+
name = data[i]['member'][j]['memberNameEn']
|
|
676
|
+
classification = data[i]['member'][j]['classificationCode']
|
|
677
|
+
|
|
678
|
+
if top_name in attributes:
|
|
679
|
+
attributes[top_name].append(name)
|
|
680
|
+
else:
|
|
681
|
+
attributes[top_name] = [name]
|
|
682
|
+
|
|
683
|
+
print(id,' - ',tablename,'\n',attributes)
|
|
684
|
+
#get_table(id=id,n=n,enc=enc)#region_type='503')
|
|
685
|
+
|
|
686
|
+
count += 1
|
|
687
|
+
|